use sqlalchemy session only as short as possible

python/Dockerfile

@@ -1,6 +1,10 @@
-FROM python:3.9-alpine
+#FROM python:3.9-alpine
+FROM python:3.11-bookworm
 LABEL MAINTAINER="Thomas Peetz <thomas.peetz@thpeetz.de>"
 ENV PS1="\[\e[0;33m\]|> kontor <| \[\e[1;35m\]\W\[\e[0m\] \[\e[0m\]# "
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update && apt-get install -y apt-utils libmariadb3 libmariadb-dev
+RUN rm -rf /var/lib/apt/lists/*
 
 WORKDIR /src
 COPY . /src

python/kontor/controllers/clibase.py

@@ -40,6 +40,6 @@ class CliBase(Controller):
     )
     def gui(self):
         application = QApplication([])
-        window = MainWindow(self.app.session, self.app.log)
+        window = MainWindow(self.app.session, self.app.engine, self.app.log)
         window.show()
         application.exec()

python/kontor/controllers/database.py

@@ -26,9 +26,8 @@ class Database(Controller):
         }
         if self.app.pargs.db_file is not None:
             data['db_file'] = self.app.pargs.db_file
-        kontor_db = KontorDB(self.app.session, self.app.log)
+        kontor_db = KontorDB(self.app.session, self.app.engine, self.app.log)
-        table_list = kontor_db.get_table_names()
+        kontor_db.export_db(data['export_type'], data['db_file'])
-        kontor_db.export_db(data['export_type'], data['db_file'], table_list)
         self.app.render(data, 'command1.jinja2')
 
     @ex(
@@ -48,6 +47,6 @@ class Database(Controller):
         }
         if self.app.pargs.db_file is not None:
             data['db_file'] = self.app.pargs.db_file
-        kontor_db = KontorDB(self.app.session, self.app.log)
+        kontor_db = KontorDB(self.app.session, self.app.engine, self.app.log)
         self.app.render(data, 'import.jinja2')
         kontor_db.import_db(data['db_file'], self.app.pargs.dry_run)

python/kontor/controllers/media.py

@@ -58,7 +58,7 @@ class Media(Controller):
             data['link_url'] = self.app.pargs.link
             if self.app.pargs.dry_run:
                 print(f"add url {data['link_url']} to database")
-            kontor_db = KontorDB(self.app.session, self.app.log)
+            kontor_db = KontorDB(self.app.session, self.app.engine, self.app.log)
             kontor_db.add_link(self.app.pargs.link, self.app.pargs.dry_run)
 
         else:

python/kontor/database/__init__.py

@@ -1,7 +1,11 @@
 import json
+import uuid
 from datetime import datetime
 from pathlib import Path
 
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import sessionmaker
+
 from .base import Base
 from .bookshelf import Article, Book, Author, BookshelfPublisher, ArticleAuthor, BookAuthor
 from .comic import Comic, Artist, Publisher, Issue, StoryArc, TradePaperback, Volume, ComicWork, WorkType
@@ -12,8 +16,9 @@ from .media import MediaFile, MediaArticle, MediaVideo
 
 class KontorDB:
 
-    def __init__(self, db_session, log):
+    def __init__(self, db_session, db_engine, log):
         self.session = db_session
+        self.engine = db_engine
         self.log = log
         self.registry = {}
         self.init_registry()
@@ -47,8 +52,11 @@ class KontorDB:
         self.registry['media_video'] = MediaVideo
 
     def get_table_names(self) -> list:
-        tables = self.session.query(MetaDataTable).all()
+        result = []
-        result = [table.table_name for table in tables]
+        __session__ = sessionmaker(self.engine)
+        with __session__() as session:
+            tables = self.session.query(MetaDataTable).all()
+            result = [table.table_name for table in tables]
         return result
 
     def get_column_meta_data(self, table_name: str, view_only=True) -> dict:
@@ -76,41 +84,46 @@ class KontorDB:
 
     def get_filters(self, table_name):
         _filter_map = {}
-        for (_, column) in (self.session.query(MetaDataTable, MetaDataColumn).
+        __session__ = sessionmaker(self.engine)
-                filter(MetaDataTable.id == MetaDataColumn.table_id).
+        with __session__() as session:
-                filter(MetaDataTable.table_name == table_name).
+            for (_, column) in (session.query(MetaDataTable, MetaDataColumn).
-                filter(MetaDataColumn.show_filter == 1).all()):
+                    filter(MetaDataTable.id == MetaDataColumn.table_id).
-            _filter_map[column.column_name] = {'label': column.filter_label, 'widget': None}
+                    filter(MetaDataTable.table_name == table_name).
-        self.log.info(f"retrieved {len(_filter_map)} filters: {_filter_map}")
+                    filter(MetaDataColumn.show_filter == 1).all()):
+                _filter_map[column.column_name] = {'label': column.filter_label, 'widget': None}
+            self.log.debug(f"retrieved {len(_filter_map)} filters: {_filter_map}")
         return _filter_map
 
     def data(self, table, columns: dict, filters) -> list:
         data = []
-        entries = []
+        __session__ = sessionmaker(self.engine)
-        if len(filters) == 0:
+        with __session__() as session:
-            entries = self.session.query(table).all()
+            entries = []
-        else:
+            if len(filters) == 0:
-            entries = self.session.query(table).filter_by(**filters)
+                entries = session.query(table).all()
-        for entry in entries:
+            else:
-            row = []
+                entries = session.query(table).filter_by(**filters)
-            for order in columns.keys():
+            for entry in entries:
-                column_name = columns[order]['column']
+                row = []
-                if str(column_name).endswith("_id"):
+                for order in columns.keys():
-                    ref_table = column_name[:-3]
+                    column_name = columns[order]['column']
-                    # print(f"{ref_table=}")
+                    if str(column_name).endswith("_id"):
-                    ref = getattr(entry, ref_table)
+                        ref_table = column_name[:-3]
-                    value = getattr(ref, "name")
+                        # print(f"{ref_table=}")
-                    # print(f"{value=}")
+                        ref = getattr(entry, ref_table)
-                    row.append(value)
+                        value = getattr(ref, "name")
-                else:
+                        # print(f"{value=}")
-                    row.append(getattr(entry, column_name))
+                        row.append(value)
-            # print(repr(row))
+                    else:
-            data.append(row)
+                        row.append(getattr(entry, column_name))
+                # print(repr(row))
+                data.append(row)
         return data
 
-    def export_db(self, export_type: str, export_file_name: str, export_table_list: list):
+    def export_db(self, export_type: str, export_file_name: str):
         self.log.info(f"export DB to {export_file_name} as {export_type}")
         db = {}
+        export_table_list = self.get_table_names()
         for table in export_table_list:
             columns = self.get_column_meta_data(table, view_only=False)
             if table in self.registry:
@@ -118,27 +131,29 @@ class KontorDB:
             else:
                 print(f"table {table} is not registered")
                 continue
-            rows = self.session.query(model).all()
+            __session__ = sessionmaker(self.engine)
-            entries = []
+            with __session__() as session:
-            self.log.debug(f"found {len(rows)} entries")
+                rows = session.query(model).all()
-            self.log.debug(f"found {len(columns)} columns")
+                entries = []
-            for row in rows:
+                self.log.debug(f"found {len(rows)} entries")
-                # print(row)
+                self.log.debug(f"found {len(columns)} columns")
-                entry = {}
+                for row in rows:
-                for order in columns:
+                    # print(row)
-                    # print(columns[order])
+                    entry = {}
-                    column_name = columns[order]['column']
+                    for order in columns:
-                    # print(f"get value {column_name} from {row} of table {table}")
+                        # print(columns[order])
-                    try:
+                        column_name = columns[order]['column']
-                        value = getattr(row, column_name)
+                        # print(f"get value {column_name} from {row} of table {table}")
-                        if isinstance(value, datetime):
+                        try:
-                            entry[column_name] = str(value)
+                            value = getattr(row, column_name)
-                        else:
+                            if isinstance(value, datetime):
-                            entry[column_name] = value
+                                entry[column_name] = str(value)
-                    except AttributeError as error:
+                            else:
-                        self.log.debug("could not get value")
+                                entry[column_name] = value
-                entries.append(entry)
+                        except AttributeError as error:
-            db[table] = entries
+                            self.log.debug("could not get value")
+                    entries.append(entry)
+                db[table] = entries
         export_file = Path(export_file_name)
         match export_type:
             case "JSON":
@@ -187,17 +202,31 @@ class KontorDB:
                 if changed:
                     print(f"{current_id} has changed")
                 existing_ids.remove(current_id)
+            else:
+                self.log.info("item to import not found in database, add new one...")
         if len(existing_ids) > 0:
             print("remaining items")
         self.session.commit()
 
     def get_ids(self, table_name: str) -> list:
         existing_ids = []
-        items = self.session.query(self.registry[table_name]).all()
+        __session__ = sessionmaker(self.engine)
-        for item in items:
+        with __session__() as session:
-            existing_ids.append(getattr(item, 'id'))
+            items = session.query(self.registry[table_name]).all()
+            for item in items:
+                existing_ids.append(getattr(item, 'id'))
         return existing_ids
 
+    def add_entry(self, update_item: dict):
+        media_file = MediaFile()
+        __session__ = sessionmaker(self.engine)
+        with __session__() as session:
+            for key in update_item.keys():
+                update_value = update_item[key]
+                setattr(media_file, key, update_value)
+            session.add(media_file)
+            session.commit()
+
     def update_entry(self, existing_item, update_item: dict, dry_run: bool) -> bool:
         changed = False
         for key in update_item.keys():
@@ -217,3 +246,20 @@ class KontorDB:
 
     def add_link(self, link: str, dry_run: bool):
         self.log.info(f"add link {link} to media_file")
+        __session__ = sessionmaker(self.engine)
+        with __session__() as session:
+            media_file = MediaFile()
+            media_file.id = str(uuid.uuid4())
+            media_file.created_date = datetime.now()
+            media_file.last_modified_date = datetime.now()
+            media_file.version = 0
+            media_file.url = link
+            media_file.review = 1
+            media_file.should_download = 1
+            try:
+                session.add(media_file)
+                session.commit()
+                self.log.info(f"entry {media_file} successfully added")
+            except IntegrityError as error:
+                self.session.rollback()
+                self.log.info(error.orig)

python/kontor/gui/main_window.py

@@ -12,7 +12,7 @@ from .table_model import KontorTableModel
 
 class MainWindow(QMainWindow):
 
-    def __init__(self, session, log):
+    def __init__(self, session, engine, log):
         super().__init__()
 
         self.tick = QIcon('kontor/res/tick.png')
@@ -30,7 +30,7 @@ class MainWindow(QMainWindow):
 
         self.data = []
         self.filter = {}
-        self.kontor_db = KontorDB(session, log)
+        self.kontor_db = KontorDB(session, engine, log)
         self.log = log
         self.central_widget = QWidget()
         parent_layout = QVBoxLayout()
@@ -114,7 +114,7 @@ class MainWindow(QMainWindow):
             self.log.info(export_dlg.get_tables_to_export())
             self.log.info(f"export DB to {export_dlg.file_name}")
             self.statusBar.showMessage(f"export DB to {export_dlg.file_name}", 3000)
-            self.kontor_db.export_db(export_dlg.current_export_type, export_dlg.file_name, export_dlg.get_tables_to_export())
+            self.kontor_db.export_db(export_dlg.current_export_type, export_dlg.file_name)
         else:
             self.statusBar.showMessage("Export cancelled", 3000)
 

python/kontor/main.py

@@ -35,6 +35,7 @@ def extend_sqlalchemy(app):
     Base.metadata.create_all(bind=engine)
     __session__ = sessionmaker(bind=engine)
     app.extend('session', __session__())
+    app.extend('engine', engine)
 
 
 def close_session(app):

python/setup.py

@@ -19,7 +19,7 @@ setup(
     url='https://gitlab.com/tpeetz/kontor',
     license='MIT',
     packages=find_packages(exclude=['ez_setup', 'tests*']),
-    package_data={'kontor': ['templates/*']},
+    package_data={'kontor': ['templates/*', 'res/*',]},
     include_package_data=True,
     entry_points="""
         [console_scripts]