fix exporting and importing from file

python/kontor-cli/kontor/controllers/database.py

@@ -1,6 +1,5 @@
 import mariadb
 from cement import Controller, ex
-from kontor_schema import KontorDB
 
 
 class Database(Controller):
@@ -30,9 +29,9 @@ class Database(Controller):
         }
         if self.app.pargs.db_file is not None:
             data['db_file'] = self.app.pargs.db_file
-        kontor_db = KontorDB(self.app.engine)
+        db = self.app.kontor_db
         self.app.log.info(f"export DB to {data['db_file']} as {data['export_type']}")
-        results = kontor_db.export_db(data['export_type'], data['db_file'])
+        results = db.export_db(data['export_type'], data['db_file'])
         data['results'] = results
         self.app.render(data, 'export_db.jinja2')
 
@@ -43,18 +42,25 @@ class Database(Controller):
             (['-f', '--file'],
              {'help': 'file to read data',
               'action': 'store',
-              'dest': 'db_file'})
+              'dest': 'db_file'}),
+            (['-d', '--delete-first'],
+             {'help': 'delete existing entries before import',
+              'action': 'store_true',
+              'dest': 'delete_first'})
         ],
     )
     def import_cmd(self):
         data = {
             'db_file': 'data.json',
             'data_type': 'JSON',
+            'delete_first': False,
         }
         if self.app.pargs.db_file is not None:
             data['db_file'] = self.app.pargs.db_file
-        kontor_db = KontorDB(self.app.engine)
+        if self.app.pargs.delete_first is not None:
-        kontor_db.import_db(data['db_file'], self.app.pargs.dry_run)
+            data['delete_first'] = self.app.pargs.delete_first
+        db = self.app.kontor_db
+        db.import_db(data['db_file'], data['delete_first'])
 
     @ex(
         help='check the db schema against MetaDataTable and MetaDataColumn'
@@ -72,13 +78,13 @@ class Database(Controller):
         cursor.execute("SHOW TABLES")
         for (tablename,) in cursor.fetchall():
             table_list.append(tablename)
-        kontor_db = KontorDB(self.app.engine)
+        db = self.app.kontor_db
-        table_names = kontor_db.get_table_names()
+        table_names = db.get_table_names()
         for table in table_list:
             if table not in table_names:
                 self.app.log.info(f"{table} is not stored in MetaDataTable")
                 continue
-            meta_data = kontor_db.get_columns(table)
+            meta_data = db.get_columns(table)
             field_info = self.get_table_field_info(cursor, table)
             for column in field_info:
                 if column not in meta_data:

python/kontor-cli/kontor/controllers/media.py

@@ -16,14 +16,14 @@ class Media(Controller):
         help='update title for mediafiles',
     )
     def update_title(self):
-        kontor_db = KontorDB(self.app.engine)
+        db = self.app.kontor_db
-        updates = kontor_db.get_update_list()
+        updates = db.get_update_list()
         self.app.log.info(f"found {len(updates)} links for update")
         for file_id, url in updates.items():
             link = VideoLink(url)
             title = link.get_title()
             if title is not None:
-                kontor_db.update_entry('media_file', file_id, {'title': title, 'review': 0,})
+                db.update_entry('media_file', file_id, {'title': title, 'review': 0,})
 
     @ex(
         label='download',
@@ -41,21 +41,21 @@ class Media(Controller):
         }
         if self.app.pargs.media_dir is not None:
             data['media_dir'] = self.app.pargs.media_dir
-        kontor_db = KontorDB(self.app.engine)
+        db = self.app.kontor_db
-        downloads = kontor_db.get_download_list()
+        downloads = db.get_download_list()
         self.app.log.info(f"found {len(downloads)} links for download")
         for file_id, url in downloads.items():
             link = VideoLink(url)
             file_name = link.download(download_dir=data['media_dir'])
             if file_name is None:
-                kontor_db.update_entry('media_file', file_id, {'file_name': None, 'should_download': 1})
+                db.update_entry('media_file', file_id, {'file_name': None, 'should_download': 1})
             else:
                 download_file = Path(file_name)
                 download_file.with_name(f"{file_id}{download_file.suffix}")
                 link.file_name = download_file.name
                 link.should_download = 0
                 link.cloud_link = download_file.absolute()
-                kontor_db.update_entry('media_file', file_id,
+                db.update_entry('media_file', file_id,
                                        {
                                            'file_name': download_file.name,
                                            'should_download': 0,
@@ -80,8 +80,8 @@ class Media(Controller):
             if self.app.pargs.dry_run:
                 print(f"add url {data['link_url']} to database")
             else:
-                kontor_db = KontorDB(self.app.engine)
+                db = self.app.kontor_db
-                result = kontor_db.add_link(self.app.pargs.link)
+                result = db.add_link(self.app.pargs.link)
                 self.log.info(result)
         else:
             print("no url was given.")
@@ -101,5 +101,5 @@ class Media(Controller):
         }
         if self.app.pargs.media_dir is not None:
             data['media_dir'] = self.app.pargs.media_dir
-        kontor_db = KontorDB(self.app.engine)
+        db = self.app.kontor_db
-        kontor_db.check_files()
+        db.check_files()

python/kontor-cli/kontor/main.py

@@ -1,7 +1,7 @@
 
 from cement import App, TestApp, init_defaults
 from cement.core.exc import CaughtSignal
-from kontor_schema.base import Base
+from kontor_schema import Base, KontorDB
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 
@@ -35,6 +35,8 @@ def extend_sqlalchemy(app):
     Base.metadata.create_all(bind=engine, checkfirst=True)
     __session__ = sessionmaker(bind=engine)
     app.extend('engine', engine)
+    kontor_db = KontorDB(engine, app.log)
+    app.extend('kontor_db', kontor_db)
 
 
 class Kontor(App):

python/kontor-gui/.gitignore

@@ -6,3 +6,4 @@ include/
 lib/
 lib64/
 lib64
+env/

python/kontor-gui/gui/comic_window.py

@@ -0,0 +1,54 @@
+from PySide6.QtWidgets import QMainWindow, QWidget, QVBoxLayout, QTabWidget, QMenu, QTableView
+
+from gui.model_config import KontorModelConfig
+from gui.table_model import KontorTableModel
+
+
+class ComicWindow(QWidget):
+    def __init__(self, main_window):
+        super().__init__()
+        self.statusBar = main_window.statusBar
+        self._main_window = main_window
+        self.data_views = list()
+        # self._create_menubar()
+        self._init_gui()
+
+    def _init_gui(self):
+        self.central_widget = QWidget()
+        layout = QVBoxLayout()
+        #self.central_widget.setLayout(parent_layout)
+        self.tabs = QTabWidget()
+        self.tabs.addTab(self.generate_data_tab("comic"), "Comics")
+        self.tabs.addTab(self.generate_data_tab("publisher"), "Publisher")
+        self.tabs.currentChanged.connect(self._tab_changed)
+        # label.setAlignment(Qt.AlignmentFlag.AlignCenter)
+        layout.addWidget(self.tabs)
+        self.setLayout(layout)
+        # self.setCentralWidget(self.central_widget)
+
+    # def _create_menubar(self):
+    #    menu_bar = self.menuBar()
+    #    comic_menu = QMenu("Comic")
+    #    menu_bar.addMenu(comic_menu)
+
+    def refresh(self):
+        self.data_views[self.tabs.currentIndex()].refresh()
+
+    def _tab_changed(self, tab_index):
+        self.data_views[tab_index].refresh()
+
+    def generate_data_tab(self, table_name):
+        data_tab = QWidget()
+
+        table_config = KontorModelConfig(self._main_window.kontor_db, self, table_name)
+        model = KontorTableModel(table_config)
+        layout = QVBoxLayout()
+        self.data_views.append(model)
+        data_tab.setLayout(layout)
+        table_view = QTableView()
+        table_view.setModel(model)
+        layout.addLayout(table_config.get_filter_layout())
+        layout.addWidget(table_view)
+        model.refresh()
+        return data_tab
+

python/kontor-gui/gui/main_window.py

@@ -4,6 +4,7 @@ from PySide6.QtWidgets import QLabel, QMainWindow
 from sqlalchemy import Engine
 from kontor_schema import KontorDB
 
+from .comic_window import ComicWindow
 from .progress import ProgressUpdate
 from .dialogs import ExportKontorDialog, ImportKontorDialog
 from .model_config import KontorModelConfig
@@ -34,6 +35,8 @@ class MainWindow(QMainWindow):
         self.filter = {}
         self.kontor_db = KontorDB(engine, log)
         self.log = log
+
+        self.comic_window = ComicWindow(self)
         self.central_widget = QWidget()
         parent_layout = QVBoxLayout()
         self.central_widget.setLayout(parent_layout)
@@ -52,6 +55,8 @@ class MainWindow(QMainWindow):
         self.newAction = QAction("&New", self)
         self.aboutAction = QAction("&Über...", self)
         self.aboutAction.triggered.connect(self.about)
+        self.showComicWindow = QAction("Show/Hide &Comic Window", self)
+        self.showComicWindow.triggered.connect(self.show_comic_window)
         self.importAction = QAction(self.import_icon, "&Import", self)
         self.importAction.triggered.connect(self.import_from_file)
         self.exportAction = QAction(self.export_icon, "&Export", self)
@@ -80,6 +85,7 @@ class MainWindow(QMainWindow):
         kontor_menu.addAction(self.importAction)
         kontor_menu.addAction(self.exportAction)
         comic_menu = QMenu("&Comic")
+        comic_menu.addAction(self.showComicWindow)
         tysc_menu = QMenu("&TradeYourSportCards")
         media_file_menu = QMenu("&MediaFile")
         media_file_menu.addAction(self.updateTitleAction)
@@ -110,12 +116,18 @@ class MainWindow(QMainWindow):
     def about(self):
         QMessageBox.about(self.central_widget, "Über Kontor", f"Python: 3.11\nKontor: 0.1.0")
 
+    def show_comic_window(self):
+        if self.comic_window.isHidden():
+            self.comic_window.show()
+        else:
+            self.comic_window.hide()
+
     def import_from_file(self):
         import_dlg = ImportKontorDialog(self)
         if import_dlg.exec():
             print(f"import DB from file {import_dlg.file_name}")
         else:
-            print("no nothing for import")
+            print("do nothing for import")
         pass
 
     def export_to_file(self):

python/kontor-gui/kontor.py

@@ -1,7 +1,6 @@
 """
-PyQT6 GUI for Kontor
+PySide6 GUI for Kontor
 """
-import logging
 import sys
 import logging.config
 from pathlib import Path

python/kontor-schema/kontor_schema/__init__.py

@@ -3,12 +3,15 @@ import re
 import subprocess
 import uuid
 from datetime import datetime
+from logging import Logger
 from pathlib import Path
 
-from sqlalchemy import Engine
+import mariadb
+from sqlalchemy import Engine, select
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import sessionmaker
 
+from .base import Base, BaseMixin
 from .admin import User, Token, Role, AuthorizationMatrix, ModuleData, MailAccount, Mail
 from .bookshelf import Article, Book, Author, BookshelfPublisher, ArticleAuthor, BookAuthor
 from .comic import Comic, Artist, Publisher, Issue, StoryArc, TradePaperback, Volume, ComicWork, WorkType
@@ -19,44 +22,45 @@ from .media import MediaFile, MediaArticle, MediaVideo
 
 class KontorDB:
 
-    def __init__(self, db_engine: Engine):
+    def __init__(self, db_engine: Engine, log: Logger):
         self.engine = db_engine
         self.registry = {}
         self.init_registry()
+        self.log = log
 
     def init_registry(self):
-        self.registry['card'] = Card
+        self.registry[Card.__tablename__] = Card
-        self.registry['card_set'] = CardSet
+        self.registry[CardSet.__tablename__] = CardSet
-        self.registry['sport'] = Sport
+        self.registry[Rooster.__tablename__] = Rooster
-        self.registry['team'] = Team
+        self.registry[Team.__tablename__] = Team
-        self.registry['field_position'] = FieldPosition
+        self.registry[FieldPosition.__tablename__] = FieldPosition
-        self.registry['rooster'] = Rooster
+        self.registry[Player.__tablename__] = Player
-        self.registry['player'] = Player
+        self.registry[Vendor.__tablename__] = Vendor
-        self.registry['vendor'] = Vendor
+        self.registry[Sport.__tablename__] = Sport
-        self.registry['artist'] = Artist
+        self.registry[Issue.__tablename__] = Issue
-        self.registry['publisher'] = Publisher
+        self.registry[TradePaperback.__tablename__] = TradePaperback
-        self.registry['comic'] = Comic
+        self.registry[StoryArc.__tablename__] = StoryArc
-        self.registry['issue'] = Issue
+        self.registry[Volume.__tablename__] = Volume
-        self.registry['story_arc'] = StoryArc
+        self.registry[ComicWork.__tablename__] = ComicWork
-        self.registry['trade_paperback'] = TradePaperback
+        self.registry[Artist.__tablename__] = Artist
-        self.registry['volume'] = Volume
+        self.registry[Comic.__tablename__] = Comic
-        self.registry['comic_work'] = ComicWork
+        self.registry[Publisher.__tablename__] = Publisher
-        self.registry['worktype'] = WorkType
+        self.registry[WorkType.__tablename__] = WorkType
-        self.registry['article'] = Article
+        self.registry[ArticleAuthor.__tablename__] = ArticleAuthor
-        self.registry['book'] = Book
+        self.registry[BookAuthor.__tablename__] = BookAuthor
-        self.registry['author'] = Author
+        self.registry[BookshelfPublisher.__tablename__] = BookshelfPublisher
-        self.registry['bookshelf_publisher'] = BookshelfPublisher
+        self.registry[Article.__tablename__] = Article
-        self.registry['article_author'] = ArticleAuthor
+        self.registry[Book.__tablename__] = Book
-        self.registry['book_author'] = BookAuthor
+        self.registry[Author.__tablename__] = Author
-        self.registry['media_file'] = MediaFile
+        self.registry[MediaFile.__tablename__] = MediaFile
-        self.registry['media_article'] = MediaArticle
+        self.registry[MediaArticle.__tablename__] = MediaArticle
-        self.registry['media_video'] = MediaVideo
+        self.registry[MediaVideo.__tablename__] = MediaVideo
-        self.registry[MetaDataTable.__tablename__] = MetaDataTable
         self.registry[MetaDataColumn.__tablename__] = MetaDataColumn
-        self.registry[User.__tablename__] = User
+        self.registry[MetaDataTable.__tablename__] = MetaDataTable
-        self.registry[Token.__tablename__] = Token
-        self.registry[Role.__tablename__] = Role
         self.registry[AuthorizationMatrix.__tablename__] = AuthorizationMatrix
+        self.registry[Token.__tablename__] = Token
+        self.registry[User.__tablename__] = User
+        self.registry[Role.__tablename__] = Role
         self.registry[ModuleData.__tablename__] = ModuleData
         self.registry[MailAccount.__tablename__] = MailAccount
         self.registry[Mail.__tablename__] = Mail
@@ -65,7 +69,7 @@ class KontorDB:
         result = []
         __session__ = sessionmaker(self.engine)
         with __session__() as session:
-            tables = session.query(MetaDataTable).all()
+            tables = session.scalars(select(MetaDataTable)).all()
             result = [table.table_name for table in tables]
         return result
 
@@ -123,9 +127,9 @@ class KontorDB:
         with __session__() as session:
             entries = []
             if len(filters) == 0:
-                entries = session.query(table).all()
+                entries = session.scalars(select(table)).all()
             else:
-                entries = session.query(table).filter_by(**filters)
+                entries = session.scalars(select(table).filter_by(**filters)).all()
             for entry in entries:
                 row = []
                 for order in columns.keys():
@@ -149,7 +153,7 @@ class KontorDB:
             if table in self.registry:
                 model = self.registry[table]
             else:
-                print(f"table {table} is not registered")
+                self.log.info(f"table {table} is not registered")
                 continue
             __session__ = sessionmaker(self.engine)
             with __session__() as session:
@@ -182,13 +186,16 @@ class KontorDB:
                 export_file = Path(export_file_name)
             case "SQLite":
                 export_file = Path(export_file_name)
+        self.log.info("%d tables exported", len(results))
         return results
 
-    def import_db(self, import_file_name: str, dry_run: bool) -> dict:
+    def import_db(self, import_file_name: str, delete_first: bool) -> dict:
         result = {}
+        if delete_first:
+            self.delete_entries()
         import_file = Path(import_file_name)
         if not import_file.exists():
-            print(f"File {import_file_name} does not exist. Do nothing.")
+            self.log.info("File %s does not exist. Do nothing.", import_file_name)
             return result
         match import_file.suffix:
             case '.json':
@@ -196,8 +203,8 @@ class KontorDB:
                 with open(import_file_name, 'r') as json_file:
                     json_load = json.load(json_file)
                 for table in json_load:
-                    print(f"{table}: {len(json_load[table])}")
+                    self.log.info("%s: %d", table, len(json_load[table]))
-                    result[table] = self.import_table(table, json_load[table], dry_run)
+                    result[table] = self.import_table(table, json_load[table])
             case '.yml':
                 print("read yaml file")
             case '.yaml':
@@ -206,30 +213,36 @@ class KontorDB:
                 print("read sqlite file")
         return result
 
-    def import_table(self, table_name, items, dry_run: bool) -> dict:
+    def import_table(self, table_name: str, items:list) -> dict:
         result = {}
         updated = []
         added = []
         remaining = []
         existing_ids = self.get_ids(table_name)
+        self.log.info("found %d existing ids for table %s", len(existing_ids), table_name)
         for item in items:
             current_id = item['id']
+            # print(f"import item: {item}")
             found_item = None
             __session__ = sessionmaker(self.engine)
             with __session__() as session:
-                found_item = session.query(self.registry[table_name]).get(current_id)
+                found_item = session.get(self.registry[table_name], current_id)
+                # print(f"found item: {found_item}")
             if found_item is not None:
                 changed = self.update_entry(table_name, current_id, item)
                 updated.append(item)
                 if changed:
-                    print(f"{current_id} has changed")
+                    self.log.info("%s has changed", current_id)
                     updated.append(item)
                 existing_ids.remove(current_id)
             else:
-                self.add_entry(table_name, item)
+                try:
-                added.append(item)
+                    self.add_entry(table_name, item)
+                    added.append(item)
+                except IntegrityError as error:
+                    self.log.info("Could not add item, due to: %s", error.detail)
         if len(existing_ids) > 0:
-            print("remaining items")
+            print(f"remaining items: {existing_ids}")
             remaining.extend(existing_ids)
         result['updated'] = updated
         result['added'] = added
@@ -246,16 +259,18 @@ class KontorDB:
         return existing_ids
 
     def add_entry(self, table_name: str, update_item: dict):
+        self.log.info("add entry to table %s with %s", table_name, update_item)
         __session__ = sessionmaker(self.engine)
         with __session__() as session:
             add_item = self.registry[table_name]()
             for key in update_item.keys():
                 update_value = update_item[key]
                 setattr(add_item, key, update_value)
-                session.add(add_item)
+            session.add(add_item)
-                session.commit()
+            session.commit()
 
     def update_entry(self, table_name, current_id, update_item: dict) -> bool:
+        self.log.info("update entry to table %s", table_name)
         __session__ = sessionmaker(self.engine)
         with __session__() as session:
             existing_item = session.query(self.registry[table_name]).get(current_id)
@@ -266,11 +281,11 @@ class KontorDB:
                 if type(existing_value) is not type(update_value):
                     existing_value = str(existing_value)
                 if existing_value != update_value:
-                    print(f"{key} has changed: {existing_value} != {update_value}")
+                    self.log.info("%s has changed: %s != %s", key, existing_value, update_value)
                     setattr(existing_item, key, update_value)
                     session.commit()
                     changed = True
-                    print(f"update {key} with {update_value}")
+                    self.log.info("update %s with %s", key, update_value)
         return changed
 
     def add_link(self, link: str) -> dict:
@@ -317,3 +332,13 @@ class KontorDB:
                     continue
                 download_list[link.id] = url
         return download_list
+
+    def delete_entries(self):
+        for (table_name, table) in self.registry.items():
+            self.log.info("delete entries from table %s", table_name)
+            __session__ = sessionmaker(self.engine)
+            with __session__() as session:
+                items = session.query(table).all()
+                for item in items:
+                    session.delete(item)
+                session.commit()

springboot/src/main/java/de/thpeetz/kontor/admin/SetupModuleAdmin.java

@@ -166,11 +166,11 @@ public class SetupModuleAdmin implements ApplicationListener<ContextRefreshedEve
         metaDataService.getColumn(artistTable, "version", "version", "LONG", null, 4, Boolean.FALSE, "", Boolean.FALSE, null);
         metaDataService.getColumn(artistTable, "name", "name", "TEXT", "UNIQUE", 5, Boolean.FALSE, "", Boolean.FALSE, null);
         MetaDataTable publisherTable = metaDataService.getTable("publisher");
-        metaDataService.getColumn(publisherTable, "id", "identifier", "TEXT", "PRIMARY KEY", 1, Boolean.FALSE, "", Boolean.FALSE, null);
+        metaDataService.getColumn(publisherTable, "id", "identifier", "TEXT", "PRIMARY KEY", 1, Boolean.TRUE, "", Boolean.FALSE, null);
         metaDataService.getColumn(publisherTable, "created_date", "created", "TIMESTAMP", null, 2, Boolean.FALSE, "", Boolean.FALSE, null);
         metaDataService.getColumn(publisherTable, "last_modified_date", "modified", "TIMESTAMP", null, 3, Boolean.FALSE, "", Boolean.FALSE, null);
         metaDataService.getColumn(publisherTable, "version", "version", "LONG", null, 4, Boolean.FALSE, "", Boolean.FALSE, null);
-        metaDataService.getColumn(publisherTable, "name", "name", "TEXT", "UNIQUE", 5, Boolean.FALSE, "", Boolean.FALSE, null);
+        metaDataService.getColumn(publisherTable, "name", "name", "TEXT", "UNIQUE", 5, Boolean.TRUE, "", Boolean.FALSE, null);
         MetaDataTable comicTable = metaDataService.getTable("comic");
         metaDataService.getColumn(comicTable, "id", "identifier", "TEXT", "PRIMARY KEY", 1, Boolean.TRUE, "ID", Boolean.FALSE, null);
         metaDataService.getColumn(comicTable, "created_date", "created", "TIMESTAMP", null, 2, Boolean.FALSE, "", Boolean.FALSE, null);