fix exporting and importing from file

python/kontor-schema/kontor_schema/__init__.py

@@ -3,12 +3,15 @@ import re
 import subprocess
 import uuid
 from datetime import datetime
+from logging import Logger
 from pathlib import Path
 
-from sqlalchemy import Engine
+import mariadb
+from sqlalchemy import Engine, select
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import sessionmaker
 
+from .base import Base, BaseMixin
 from .admin import User, Token, Role, AuthorizationMatrix, ModuleData, MailAccount, Mail
 from .bookshelf import Article, Book, Author, BookshelfPublisher, ArticleAuthor, BookAuthor
 from .comic import Comic, Artist, Publisher, Issue, StoryArc, TradePaperback, Volume, ComicWork, WorkType
@@ -19,44 +22,45 @@ from .media import MediaFile, MediaArticle, MediaVideo
 
 class KontorDB:
 
-    def __init__(self, db_engine: Engine):
+    def __init__(self, db_engine: Engine, log: Logger):
         self.engine = db_engine
         self.registry = {}
         self.init_registry()
+        self.log = log
 
     def init_registry(self):
-        self.registry['card'] = Card
-        self.registry['card_set'] = CardSet
-        self.registry['sport'] = Sport
-        self.registry['team'] = Team
-        self.registry['field_position'] = FieldPosition
-        self.registry['rooster'] = Rooster
-        self.registry['player'] = Player
-        self.registry['vendor'] = Vendor
-        self.registry['artist'] = Artist
-        self.registry['publisher'] = Publisher
-        self.registry['comic'] = Comic
-        self.registry['issue'] = Issue
-        self.registry['story_arc'] = StoryArc
-        self.registry['trade_paperback'] = TradePaperback
-        self.registry['volume'] = Volume
-        self.registry['comic_work'] = ComicWork
-        self.registry['worktype'] = WorkType
-        self.registry['article'] = Article
-        self.registry['book'] = Book
-        self.registry['author'] = Author
-        self.registry['bookshelf_publisher'] = BookshelfPublisher
-        self.registry['article_author'] = ArticleAuthor
-        self.registry['book_author'] = BookAuthor
-        self.registry['media_file'] = MediaFile
-        self.registry['media_article'] = MediaArticle
-        self.registry['media_video'] = MediaVideo
-        self.registry[MetaDataTable.__tablename__] = MetaDataTable
+        self.registry[Card.__tablename__] = Card
+        self.registry[CardSet.__tablename__] = CardSet
+        self.registry[Rooster.__tablename__] = Rooster
+        self.registry[Team.__tablename__] = Team
+        self.registry[FieldPosition.__tablename__] = FieldPosition
+        self.registry[Player.__tablename__] = Player
+        self.registry[Vendor.__tablename__] = Vendor
+        self.registry[Sport.__tablename__] = Sport
+        self.registry[Issue.__tablename__] = Issue
+        self.registry[TradePaperback.__tablename__] = TradePaperback
+        self.registry[StoryArc.__tablename__] = StoryArc
+        self.registry[Volume.__tablename__] = Volume
+        self.registry[ComicWork.__tablename__] = ComicWork
+        self.registry[Artist.__tablename__] = Artist
+        self.registry[Comic.__tablename__] = Comic
+        self.registry[Publisher.__tablename__] = Publisher
+        self.registry[WorkType.__tablename__] = WorkType
+        self.registry[ArticleAuthor.__tablename__] = ArticleAuthor
+        self.registry[BookAuthor.__tablename__] = BookAuthor
+        self.registry[BookshelfPublisher.__tablename__] = BookshelfPublisher
+        self.registry[Article.__tablename__] = Article
+        self.registry[Book.__tablename__] = Book
+        self.registry[Author.__tablename__] = Author
+        self.registry[MediaFile.__tablename__] = MediaFile
+        self.registry[MediaArticle.__tablename__] = MediaArticle
+        self.registry[MediaVideo.__tablename__] = MediaVideo
         self.registry[MetaDataColumn.__tablename__] = MetaDataColumn
-        self.registry[User.__tablename__] = User
-        self.registry[Token.__tablename__] = Token
-        self.registry[Role.__tablename__] = Role
+        self.registry[MetaDataTable.__tablename__] = MetaDataTable
         self.registry[AuthorizationMatrix.__tablename__] = AuthorizationMatrix
+        self.registry[Token.__tablename__] = Token
+        self.registry[User.__tablename__] = User
+        self.registry[Role.__tablename__] = Role
         self.registry[ModuleData.__tablename__] = ModuleData
         self.registry[MailAccount.__tablename__] = MailAccount
         self.registry[Mail.__tablename__] = Mail
@@ -65,7 +69,7 @@ class KontorDB:
         result = []
         __session__ = sessionmaker(self.engine)
         with __session__() as session:
-            tables = session.query(MetaDataTable).all()
+            tables = session.scalars(select(MetaDataTable)).all()
             result = [table.table_name for table in tables]
         return result
 
@@ -123,9 +127,9 @@ class KontorDB:
         with __session__() as session:
             entries = []
             if len(filters) == 0:
-                entries = session.query(table).all()
+                entries = session.scalars(select(table)).all()
             else:
-                entries = session.query(table).filter_by(**filters)
+                entries = session.scalars(select(table).filter_by(**filters)).all()
             for entry in entries:
                 row = []
                 for order in columns.keys():
@@ -149,7 +153,7 @@ class KontorDB:
             if table in self.registry:
                 model = self.registry[table]
             else:
-                print(f"table {table} is not registered")
+                self.log.info(f"table {table} is not registered")
                 continue
             __session__ = sessionmaker(self.engine)
             with __session__() as session:
@@ -182,13 +186,16 @@ class KontorDB:
                 export_file = Path(export_file_name)
             case "SQLite":
                 export_file = Path(export_file_name)
+        self.log.info("%d tables exported", len(results))
         return results
 
-    def import_db(self, import_file_name: str, dry_run: bool) -> dict:
+    def import_db(self, import_file_name: str, delete_first: bool) -> dict:
         result = {}
+        if delete_first:
+            self.delete_entries()
         import_file = Path(import_file_name)
         if not import_file.exists():
-            print(f"File {import_file_name} does not exist. Do nothing.")
+            self.log.info("File %s does not exist. Do nothing.", import_file_name)
             return result
         match import_file.suffix:
             case '.json':
@@ -196,8 +203,8 @@ class KontorDB:
                 with open(import_file_name, 'r') as json_file:
                     json_load = json.load(json_file)
                 for table in json_load:
-                    print(f"{table}: {len(json_load[table])}")
-                    result[table] = self.import_table(table, json_load[table], dry_run)
+                    self.log.info("%s: %d", table, len(json_load[table]))
+                    result[table] = self.import_table(table, json_load[table])
             case '.yml':
                 print("read yaml file")
             case '.yaml':
@@ -206,30 +213,36 @@ class KontorDB:
                 print("read sqlite file")
         return result
 
-    def import_table(self, table_name, items, dry_run: bool) -> dict:
+    def import_table(self, table_name: str, items:list) -> dict:
         result = {}
         updated = []
         added = []
         remaining = []
         existing_ids = self.get_ids(table_name)
+        self.log.info("found %d existing ids for table %s", len(existing_ids), table_name)
         for item in items:
             current_id = item['id']
+            # print(f"import item: {item}")
             found_item = None
             __session__ = sessionmaker(self.engine)
             with __session__() as session:
-                found_item = session.query(self.registry[table_name]).get(current_id)
+                found_item = session.get(self.registry[table_name], current_id)
+                # print(f"found item: {found_item}")
             if found_item is not None:
                 changed = self.update_entry(table_name, current_id, item)
                 updated.append(item)
                 if changed:
-                    print(f"{current_id} has changed")
+                    self.log.info("%s has changed", current_id)
                     updated.append(item)
                 existing_ids.remove(current_id)
             else:
-                self.add_entry(table_name, item)
-                added.append(item)
+                try:
+                    self.add_entry(table_name, item)
+                    added.append(item)
+                except IntegrityError as error:
+                    self.log.info("Could not add item, due to: %s", error.detail)
         if len(existing_ids) > 0:
-            print("remaining items")
+            print(f"remaining items: {existing_ids}")
             remaining.extend(existing_ids)
         result['updated'] = updated
         result['added'] = added
@@ -246,16 +259,18 @@ class KontorDB:
         return existing_ids
 
     def add_entry(self, table_name: str, update_item: dict):
+        self.log.info("add entry to table %s with %s", table_name, update_item)
         __session__ = sessionmaker(self.engine)
         with __session__() as session:
             add_item = self.registry[table_name]()
             for key in update_item.keys():
                 update_value = update_item[key]
                 setattr(add_item, key, update_value)
-                session.add(add_item)
-                session.commit()
+            session.add(add_item)
+            session.commit()
 
     def update_entry(self, table_name, current_id, update_item: dict) -> bool:
+        self.log.info("update entry to table %s", table_name)
         __session__ = sessionmaker(self.engine)
         with __session__() as session:
             existing_item = session.query(self.registry[table_name]).get(current_id)
@@ -266,11 +281,11 @@ class KontorDB:
                 if type(existing_value) is not type(update_value):
                     existing_value = str(existing_value)
                 if existing_value != update_value:
-                    print(f"{key} has changed: {existing_value} != {update_value}")
+                    self.log.info("%s has changed: %s != %s", key, existing_value, update_value)
                     setattr(existing_item, key, update_value)
                     session.commit()
                     changed = True
-                    print(f"update {key} with {update_value}")
+                    self.log.info("update %s with %s", key, update_value)
         return changed
 
     def add_link(self, link: str) -> dict:
@@ -317,3 +332,13 @@ class KontorDB:
                     continue
                 download_list[link.id] = url
         return download_list
+
+    def delete_entries(self):
+        for (table_name, table) in self.registry.items():
+            self.log.info("delete entries from table %s", table_name)
+            __session__ = sessionmaker(self.engine)
+            with __session__() as session:
+                items = session.query(table).all()
+                for item in items:
+                    session.delete(item)
+                session.commit()