tpeetz/kontor
1
0
Fork 0
Code Issues 6 Pull Requests 2 Actions Packages Projects Releases 2 Wiki Activity

fix exporting and importing from file

Browse Source
This commit is contained in:
Thomas Peetz
2025-01-21 16:46:12 +01:00
parent 4c330a1138
commit 4e884fdbe5
10 changed files with 35114 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/kontor-cli/kontor/controllers/database.py
+15 -9
View File
@@ -1,6 +1,5 @@
import mariadb
from cement import Controller, ex
from kontor_schema import KontorDB
class Database(Controller):
@@ -30,9 +29,9 @@ class Database(Controller):
}
if self.app.pargs.db_file is not None:
data['db_file'] = self.app.pargs.db_file
kontor_db = KontorDB(self.app.engine)
db = self.app.kontor_db
self.app.log.info(f"export DB to {data['db_file']} as {data['export_type']}")
results = kontor_db.export_db(data['export_type'], data['db_file'])
results = db.export_db(data['export_type'], data['db_file'])
data['results'] = results
self.app.render(data, 'export_db.jinja2')
@@ -43,18 +42,25 @@ class Database(Controller):
(['-f', '--file'],
{'help': 'file to read data',
'action': 'store',
'dest': 'db_file'})
'dest': 'db_file'}),
(['-d', '--delete-first'],
{'help': 'delete existing entries before import',
'action': 'store_true',
'dest': 'delete_first'})
],
)
def import_cmd(self):
data = {
'db_file': 'data.json',
'data_type': 'JSON',
'delete_first': False,
}
if self.app.pargs.db_file is not None:
data['db_file'] = self.app.pargs.db_file
kontor_db = KontorDB(self.app.engine)
kontor_db.import_db(data['db_file'], self.app.pargs.dry_run)
if self.app.pargs.delete_first is not None:
data['delete_first'] = self.app.pargs.delete_first
db = self.app.kontor_db
db.import_db(data['db_file'], data['delete_first'])
@ex(
help='check the db schema against MetaDataTable and MetaDataColumn'
@@ -72,13 +78,13 @@ class Database(Controller):
cursor.execute("SHOW TABLES")
for (tablename,) in cursor.fetchall():
table_list.append(tablename)
kontor_db = KontorDB(self.app.engine)
table_names = kontor_db.get_table_names()
db = self.app.kontor_db
table_names = db.get_table_names()
for table in table_list:
if table not in table_names:
self.app.log.info(f"{table} is not stored in MetaDataTable")
continue
meta_data = kontor_db.get_columns(table)
meta_data = db.get_columns(table)
field_info = self.get_table_field_info(cursor, table)
for column in field_info:
if column not in meta_data:
python/kontor-cli/kontor/controllers/media.py
+11 -11
View File
@@ -16,14 +16,14 @@ class Media(Controller):
help='update title for mediafiles',
)
def update_title(self):
kontor_db = KontorDB(self.app.engine)
updates = kontor_db.get_update_list()
db = self.app.kontor_db
updates = db.get_update_list()
self.app.log.info(f"found {len(updates)} links for update")
for file_id, url in updates.items():
link = VideoLink(url)
title = link.get_title()
if title is not None:
kontor_db.update_entry('media_file', file_id, {'title': title, 'review': 0,})
db.update_entry('media_file', file_id, {'title': title, 'review': 0,})
@ex(
label='download',
@@ -41,21 +41,21 @@ class Media(Controller):
}
if self.app.pargs.media_dir is not None:
data['media_dir'] = self.app.pargs.media_dir
kontor_db = KontorDB(self.app.engine)
downloads = kontor_db.get_download_list()
db = self.app.kontor_db
downloads = db.get_download_list()
self.app.log.info(f"found {len(downloads)} links for download")
for file_id, url in downloads.items():
link = VideoLink(url)
file_name = link.download(download_dir=data['media_dir'])
if file_name is None:
kontor_db.update_entry('media_file', file_id, {'file_name': None, 'should_download': 1})
db.update_entry('media_file', file_id, {'file_name': None, 'should_download': 1})
else:
download_file = Path(file_name)
download_file.with_name(f"{file_id}{download_file.suffix}")
link.file_name = download_file.name
link.should_download = 0
link.cloud_link = download_file.absolute()
kontor_db.update_entry('media_file', file_id,
db.update_entry('media_file', file_id,
{
'file_name': download_file.name,
'should_download': 0,
@@ -80,8 +80,8 @@ class Media(Controller):
if self.app.pargs.dry_run:
print(f"add url {data['link_url']} to database")
else:
kontor_db = KontorDB(self.app.engine)
result = kontor_db.add_link(self.app.pargs.link)
db = self.app.kontor_db
result = db.add_link(self.app.pargs.link)
self.log.info(result)
else:
print("no url was given.")
@@ -101,5 +101,5 @@ class Media(Controller):
}
if self.app.pargs.media_dir is not None:
data['media_dir'] = self.app.pargs.media_dir
kontor_db = KontorDB(self.app.engine)
kontor_db.check_files()
db = self.app.kontor_db
db.check_files()
python/kontor-cli/kontor/main.py
+3 -1
View File
@@ -1,7 +1,7 @@
from cement import App, TestApp, init_defaults
from cement.core.exc import CaughtSignal
from kontor_schema.base import Base
from kontor_schema import Base, KontorDB
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
@@ -35,6 +35,8 @@ def extend_sqlalchemy(app):
Base.metadata.create_all(bind=engine, checkfirst=True)
__session__ = sessionmaker(bind=engine)
app.extend('engine', engine)
kontor_db = KontorDB(engine, app.log)
app.extend('kontor_db', kontor_db)
class Kontor(App):
python/kontor-gui/.gitignore
+1
View File
@@ -6,3 +6,4 @@ include/
lib/
lib64/
lib64
env/
python/kontor-gui/data.json
+34939
View File
File diff suppressed because it is too large Load Diff
python/kontor-gui/gui/comic_window.py
+54
View File
@@ -0,0 +1,54 @@
from PySide6.QtWidgets import QMainWindow, QWidget, QVBoxLayout, QTabWidget, QMenu, QTableView
from gui.model_config import KontorModelConfig
from gui.table_model import KontorTableModel
class ComicWindow(QWidget):
def __init__(self, main_window):
super().__init__()
self.statusBar = main_window.statusBar
self._main_window = main_window
self.data_views = list()
# self._create_menubar()
self._init_gui()
def _init_gui(self):
self.central_widget = QWidget()
layout = QVBoxLayout()
#self.central_widget.setLayout(parent_layout)
self.tabs = QTabWidget()
self.tabs.addTab(self.generate_data_tab("comic"), "Comics")
self.tabs.addTab(self.generate_data_tab("publisher"), "Publisher")
self.tabs.currentChanged.connect(self._tab_changed)
# label.setAlignment(Qt.AlignmentFlag.AlignCenter)
layout.addWidget(self.tabs)
self.setLayout(layout)
# self.setCentralWidget(self.central_widget)
# def _create_menubar(self):
# menu_bar = self.menuBar()
# comic_menu = QMenu("Comic")
# menu_bar.addMenu(comic_menu)
def refresh(self):
self.data_views[self.tabs.currentIndex()].refresh()
def _tab_changed(self, tab_index):
self.data_views[tab_index].refresh()
def generate_data_tab(self, table_name):
data_tab = QWidget()
table_config = KontorModelConfig(self._main_window.kontor_db, self, table_name)
model = KontorTableModel(table_config)
layout = QVBoxLayout()
self.data_views.append(model)
data_tab.setLayout(layout)
table_view = QTableView()
table_view.setModel(model)
layout.addLayout(table_config.get_filter_layout())
layout.addWidget(table_view)
model.refresh()
return data_tab
python/kontor-gui/gui/main_window.py
+13 -1
View File
@@ -4,6 +4,7 @@ from PySide6.QtWidgets import QLabel, QMainWindow
from sqlalchemy import Engine
from kontor_schema import KontorDB
from .comic_window import ComicWindow
from .progress import ProgressUpdate
from .dialogs import ExportKontorDialog, ImportKontorDialog
from .model_config import KontorModelConfig
@@ -34,6 +35,8 @@ class MainWindow(QMainWindow):
self.filter = {}
self.kontor_db = KontorDB(engine, log)
self.log = log
self.comic_window = ComicWindow(self)
self.central_widget = QWidget()
parent_layout = QVBoxLayout()
self.central_widget.setLayout(parent_layout)
@@ -52,6 +55,8 @@ class MainWindow(QMainWindow):
self.newAction = QAction("&New", self)
self.aboutAction = QAction("&Über...", self)
self.aboutAction.triggered.connect(self.about)
self.showComicWindow = QAction("Show/Hide &Comic Window", self)
self.showComicWindow.triggered.connect(self.show_comic_window)
self.importAction = QAction(self.import_icon, "&Import", self)
self.importAction.triggered.connect(self.import_from_file)
self.exportAction = QAction(self.export_icon, "&Export", self)
@@ -80,6 +85,7 @@ class MainWindow(QMainWindow):
kontor_menu.addAction(self.importAction)
kontor_menu.addAction(self.exportAction)
comic_menu = QMenu("&Comic")
comic_menu.addAction(self.showComicWindow)
tysc_menu = QMenu("&TradeYourSportCards")
media_file_menu = QMenu("&MediaFile")
media_file_menu.addAction(self.updateTitleAction)
@@ -110,12 +116,18 @@ class MainWindow(QMainWindow):
def about(self):
QMessageBox.about(self.central_widget, "Über Kontor", f"Python: 3.11\nKontor: 0.1.0")
def show_comic_window(self):
if self.comic_window.isHidden():
self.comic_window.show()
else:
self.comic_window.hide()
def import_from_file(self):
import_dlg = ImportKontorDialog(self)
if import_dlg.exec():
print(f"import DB from file {import_dlg.file_name}")
else:
print("no nothing for import")
print("do nothing for import")
pass
def export_to_file(self):
python/kontor-gui/kontor.py
+1 -2
View File
@@ -1,7 +1,6 @@
"""
PyQT6 GUI for Kontor
PySide6 GUI for Kontor
"""
import logging
import sys
import logging.config
from pathlib import Path
python/kontor-schema/kontor_schema/__init__.py
+75 -50
View File
@@ -3,12 +3,15 @@ import re
import subprocess
import uuid
from datetime import datetime
from logging import Logger
from pathlib import Path
from sqlalchemy import Engine
import mariadb
from sqlalchemy import Engine, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import sessionmaker
from .base import Base, BaseMixin
from .admin import User, Token, Role, AuthorizationMatrix, ModuleData, MailAccount, Mail
from .bookshelf import Article, Book, Author, BookshelfPublisher, ArticleAuthor, BookAuthor
from .comic import Comic, Artist, Publisher, Issue, StoryArc, TradePaperback, Volume, ComicWork, WorkType
@@ -19,44 +22,45 @@ from .media import MediaFile, MediaArticle, MediaVideo
class KontorDB:
def __init__(self, db_engine: Engine):
def __init__(self, db_engine: Engine, log: Logger):
self.engine = db_engine
self.registry = {}
self.init_registry()
self.log = log
def init_registry(self):
self.registry['card'] = Card
self.registry['card_set'] = CardSet
self.registry['sport'] = Sport
self.registry['team'] = Team
self.registry['field_position'] = FieldPosition
self.registry['rooster'] = Rooster
self.registry['player'] = Player
self.registry['vendor'] = Vendor
self.registry['artist'] = Artist
self.registry['publisher'] = Publisher
self.registry['comic'] = Comic
self.registry['issue'] = Issue
self.registry['story_arc'] = StoryArc
self.registry['trade_paperback'] = TradePaperback
self.registry['volume'] = Volume
self.registry['comic_work'] = ComicWork
self.registry['worktype'] = WorkType
self.registry['article'] = Article
self.registry['book'] = Book
self.registry['author'] = Author
self.registry['bookshelf_publisher'] = BookshelfPublisher
self.registry['article_author'] = ArticleAuthor
self.registry['book_author'] = BookAuthor
self.registry['media_file'] = MediaFile
self.registry['media_article'] = MediaArticle
self.registry['media_video'] = MediaVideo
self.registry[MetaDataTable.__tablename__] = MetaDataTable
self.registry[Card.__tablename__] = Card
self.registry[CardSet.__tablename__] = CardSet
self.registry[Rooster.__tablename__] = Rooster
self.registry[Team.__tablename__] = Team
self.registry[FieldPosition.__tablename__] = FieldPosition
self.registry[Player.__tablename__] = Player
self.registry[Vendor.__tablename__] = Vendor
self.registry[Sport.__tablename__] = Sport
self.registry[Issue.__tablename__] = Issue
self.registry[TradePaperback.__tablename__] = TradePaperback
self.registry[StoryArc.__tablename__] = StoryArc
self.registry[Volume.__tablename__] = Volume
self.registry[ComicWork.__tablename__] = ComicWork
self.registry[Artist.__tablename__] = Artist
self.registry[Comic.__tablename__] = Comic
self.registry[Publisher.__tablename__] = Publisher
self.registry[WorkType.__tablename__] = WorkType
self.registry[ArticleAuthor.__tablename__] = ArticleAuthor
self.registry[BookAuthor.__tablename__] = BookAuthor
self.registry[BookshelfPublisher.__tablename__] = BookshelfPublisher
self.registry[Article.__tablename__] = Article
self.registry[Book.__tablename__] = Book
self.registry[Author.__tablename__] = Author
self.registry[MediaFile.__tablename__] = MediaFile
self.registry[MediaArticle.__tablename__] = MediaArticle
self.registry[MediaVideo.__tablename__] = MediaVideo
self.registry[MetaDataColumn.__tablename__] = MetaDataColumn
self.registry[User.__tablename__] = User
self.registry[Token.__tablename__] = Token
self.registry[Role.__tablename__] = Role
self.registry[MetaDataTable.__tablename__] = MetaDataTable
self.registry[AuthorizationMatrix.__tablename__] = AuthorizationMatrix
self.registry[Token.__tablename__] = Token
self.registry[User.__tablename__] = User
self.registry[Role.__tablename__] = Role
self.registry[ModuleData.__tablename__] = ModuleData
self.registry[MailAccount.__tablename__] = MailAccount
self.registry[Mail.__tablename__] = Mail
@@ -65,7 +69,7 @@ class KontorDB:
result = []
__session__ = sessionmaker(self.engine)
with __session__() as session:
tables = session.query(MetaDataTable).all()
tables = session.scalars(select(MetaDataTable)).all()
result = [table.table_name for table in tables]
return result
@@ -123,9 +127,9 @@ class KontorDB:
with __session__() as session:
entries = []
if len(filters) == 0:
entries = session.query(table).all()
entries = session.scalars(select(table)).all()
else:
entries = session.query(table).filter_by(**filters)
entries = session.scalars(select(table).filter_by(**filters)).all()
for entry in entries:
row = []
for order in columns.keys():
@@ -149,7 +153,7 @@ class KontorDB:
if table in self.registry:
model = self.registry[table]
else:
print(f"table {table} is not registered")
self.log.info(f"table {table} is not registered")
continue
__session__ = sessionmaker(self.engine)
with __session__() as session:
@@ -182,13 +186,16 @@ class KontorDB:
export_file = Path(export_file_name)
case "SQLite":
export_file = Path(export_file_name)
self.log.info("%d tables exported", len(results))
return results
def import_db(self, import_file_name: str, dry_run: bool) -> dict:
def import_db(self, import_file_name: str, delete_first: bool) -> dict:
result = {}
if delete_first:
self.delete_entries()
import_file = Path(import_file_name)
if not import_file.exists():
print(f"File {import_file_name} does not exist. Do nothing.")
self.log.info("File %s does not exist. Do nothing.", import_file_name)
return result
match import_file.suffix:
case '.json':
@@ -196,8 +203,8 @@ class KontorDB:
with open(import_file_name, 'r') as json_file:
json_load = json.load(json_file)
for table in json_load:
print(f"{table}: {len(json_load[table])}")
result[table] = self.import_table(table, json_load[table], dry_run)
self.log.info("%s: %d", table, len(json_load[table]))
result[table] = self.import_table(table, json_load[table])
case '.yml':
print("read yaml file")
case '.yaml':
@@ -206,30 +213,36 @@ class KontorDB:
print("read sqlite file")
return result
def import_table(self, table_name, items, dry_run: bool) -> dict:
def import_table(self, table_name: str, items:list) -> dict:
result = {}
updated = []
added = []
remaining = []
existing_ids = self.get_ids(table_name)
self.log.info("found %d existing ids for table %s", len(existing_ids), table_name)
for item in items:
current_id = item['id']
# print(f"import item: {item}")
found_item = None
__session__ = sessionmaker(self.engine)
with __session__() as session:
found_item = session.query(self.registry[table_name]).get(current_id)
found_item = session.get(self.registry[table_name], current_id)
# print(f"found item: {found_item}")
if found_item is not None:
changed = self.update_entry(table_name, current_id, item)
updated.append(item)
if changed:
print(f"{current_id} has changed")
self.log.info("%s has changed", current_id)
updated.append(item)
existing_ids.remove(current_id)
else:
self.add_entry(table_name, item)
added.append(item)
try:
self.add_entry(table_name, item)
added.append(item)
except IntegrityError as error:
self.log.info("Could not add item, due to: %s", error.detail)
if len(existing_ids) > 0:
print("remaining items")
print(f"remaining items: {existing_ids}")
remaining.extend(existing_ids)
result['updated'] = updated
result['added'] = added
@@ -246,16 +259,18 @@ class KontorDB:
return existing_ids
def add_entry(self, table_name: str, update_item: dict):
self.log.info("add entry to table %s with %s", table_name, update_item)
__session__ = sessionmaker(self.engine)
with __session__() as session:
add_item = self.registry[table_name]()
for key in update_item.keys():
update_value = update_item[key]
setattr(add_item, key, update_value)
session.add(add_item)
session.commit()
session.add(add_item)
session.commit()
def update_entry(self, table_name, current_id, update_item: dict) -> bool:
self.log.info("update entry to table %s", table_name)
__session__ = sessionmaker(self.engine)
with __session__() as session:
existing_item = session.query(self.registry[table_name]).get(current_id)
@@ -266,11 +281,11 @@ class KontorDB:
if type(existing_value) is not type(update_value):
existing_value = str(existing_value)
if existing_value != update_value:
print(f"{key} has changed: {existing_value} != {update_value}")
self.log.info("%s has changed: %s != %s", key, existing_value, update_value)
setattr(existing_item, key, update_value)
session.commit()
changed = True
print(f"update {key} with {update_value}")
self.log.info("update %s with %s", key, update_value)
return changed
def add_link(self, link: str) -> dict:
@@ -317,3 +332,13 @@ class KontorDB:
continue
download_list[link.id] = url
return download_list
def delete_entries(self):
for (table_name, table) in self.registry.items():
self.log.info("delete entries from table %s", table_name)
__session__ = sessionmaker(self.engine)
with __session__() as session:
items = session.query(table).all()
for item in items:
session.delete(item)
session.commit()