tpeetz/kontor
1
0
Fork 0
Code Issues 6 Pull Requests 2 Actions Packages Projects Releases 2 Wiki Activity

use sqlalchemy session only as short as possible

Browse Source
This commit is contained in:
Thomas Peetz
2025-01-16 23:03:30 +01:00
parent 33dcbc4413
commit 58263cb854
8 changed files with 115 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/Dockerfile
+5 -1
View File
@@ -1,6 +1,10 @@
FROM python:3.9-alpine
#FROM python:3.9-alpine
FROM python:3.11-bookworm
LABEL MAINTAINER="Thomas Peetz <thomas.peetz@thpeetz.de>"
ENV PS1="\[\e[0;33m\]|> kontor <| \[\e[1;35m\]\W\[\e[0m\] \[\e[0m\]# "
ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update && apt-get install -y apt-utils libmariadb3 libmariadb-dev
RUN rm -rf /var/lib/apt/lists/*
WORKDIR /src
COPY . /src
python/kontor/controllers/clibase.py
+1 -1
View File
@@ -40,6 +40,6 @@ class CliBase(Controller):
)
def gui(self):
application = QApplication([])
window = MainWindow(self.app.session, self.app.log)
window = MainWindow(self.app.session, self.app.engine, self.app.log)
window.show()
application.exec()
python/kontor/controllers/database.py
+3 -4
View File
@@ -26,9 +26,8 @@ class Database(Controller):
}
if self.app.pargs.db_file is not None:
data['db_file'] = self.app.pargs.db_file
kontor_db = KontorDB(self.app.session, self.app.log)
table_list = kontor_db.get_table_names()
kontor_db.export_db(data['export_type'], data['db_file'], table_list)
kontor_db = KontorDB(self.app.session, self.app.engine, self.app.log)
kontor_db.export_db(data['export_type'], data['db_file'])
self.app.render(data, 'command1.jinja2')
@ex(
@@ -48,6 +47,6 @@ class Database(Controller):
}
if self.app.pargs.db_file is not None:
data['db_file'] = self.app.pargs.db_file
kontor_db = KontorDB(self.app.session, self.app.log)
kontor_db = KontorDB(self.app.session, self.app.engine, self.app.log)
self.app.render(data, 'import.jinja2')
kontor_db.import_db(data['db_file'], self.app.pargs.dry_run)
python/kontor/controllers/media.py
+1 -1
View File
@@ -58,7 +58,7 @@ class Media(Controller):
data['link_url'] = self.app.pargs.link
if self.app.pargs.dry_run:
print(f"add url {data['link_url']} to database")
kontor_db = KontorDB(self.app.session, self.app.log)
kontor_db = KontorDB(self.app.session, self.app.engine, self.app.log)
kontor_db.add_link(self.app.pargs.link, self.app.pargs.dry_run)
else:
python/kontor/database/__init__.py
+100 -54
View File
@@ -1,7 +1,11 @@
import json
import uuid
from datetime import datetime
from pathlib import Path
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import sessionmaker
from .base import Base
from .bookshelf import Article, Book, Author, BookshelfPublisher, ArticleAuthor, BookAuthor
from .comic import Comic, Artist, Publisher, Issue, StoryArc, TradePaperback, Volume, ComicWork, WorkType
@@ -12,8 +16,9 @@ from .media import MediaFile, MediaArticle, MediaVideo
class KontorDB:
def __init__(self, db_session, log):
def __init__(self, db_session, db_engine, log):
self.session = db_session
self.engine = db_engine
self.log = log
self.registry = {}
self.init_registry()
@@ -47,8 +52,11 @@ class KontorDB:
self.registry['media_video'] = MediaVideo
def get_table_names(self) -> list:
tables = self.session.query(MetaDataTable).all()
result = [table.table_name for table in tables]
result = []
__session__ = sessionmaker(self.engine)
with __session__() as session:
tables = self.session.query(MetaDataTable).all()
result = [table.table_name for table in tables]
return result
def get_column_meta_data(self, table_name: str, view_only=True) -> dict:
@@ -76,41 +84,46 @@ class KontorDB:
def get_filters(self, table_name):
_filter_map = {}
for (_, column) in (self.session.query(MetaDataTable, MetaDataColumn).
filter(MetaDataTable.id == MetaDataColumn.table_id).
filter(MetaDataTable.table_name == table_name).
filter(MetaDataColumn.show_filter == 1).all()):
_filter_map[column.column_name] = {'label': column.filter_label, 'widget': None}
self.log.info(f"retrieved {len(_filter_map)} filters: {_filter_map}")
__session__ = sessionmaker(self.engine)
with __session__() as session:
for (_, column) in (session.query(MetaDataTable, MetaDataColumn).
filter(MetaDataTable.id == MetaDataColumn.table_id).
filter(MetaDataTable.table_name == table_name).
filter(MetaDataColumn.show_filter == 1).all()):
_filter_map[column.column_name] = {'label': column.filter_label, 'widget': None}
self.log.debug(f"retrieved {len(_filter_map)} filters: {_filter_map}")
return _filter_map
def data(self, table, columns: dict, filters) -> list:
data = []
entries = []
if len(filters) == 0:
entries = self.session.query(table).all()
else:
entries = self.session.query(table).filter_by(**filters)
for entry in entries:
row = []
for order in columns.keys():
column_name = columns[order]['column']
if str(column_name).endswith("_id"):
ref_table = column_name[:-3]
# print(f"{ref_table=}")
ref = getattr(entry, ref_table)
value = getattr(ref, "name")
# print(f"{value=}")
row.append(value)
else:
row.append(getattr(entry, column_name))
# print(repr(row))
data.append(row)
__session__ = sessionmaker(self.engine)
with __session__() as session:
entries = []
if len(filters) == 0:
entries = session.query(table).all()
else:
entries = session.query(table).filter_by(**filters)
for entry in entries:
row = []
for order in columns.keys():
column_name = columns[order]['column']
if str(column_name).endswith("_id"):
ref_table = column_name[:-3]
# print(f"{ref_table=}")
ref = getattr(entry, ref_table)
value = getattr(ref, "name")
# print(f"{value=}")
row.append(value)
else:
row.append(getattr(entry, column_name))
# print(repr(row))
data.append(row)
return data
def export_db(self, export_type: str, export_file_name: str, export_table_list: list):
def export_db(self, export_type: str, export_file_name: str):
self.log.info(f"export DB to {export_file_name} as {export_type}")
db = {}
export_table_list = self.get_table_names()
for table in export_table_list:
columns = self.get_column_meta_data(table, view_only=False)
if table in self.registry:
@@ -118,27 +131,29 @@ class KontorDB:
else:
print(f"table {table} is not registered")
continue
rows = self.session.query(model).all()
entries = []
self.log.debug(f"found {len(rows)} entries")
self.log.debug(f"found {len(columns)} columns")
for row in rows:
# print(row)
entry = {}
for order in columns:
# print(columns[order])
column_name = columns[order]['column']
# print(f"get value {column_name} from {row} of table {table}")
try:
value = getattr(row, column_name)
if isinstance(value, datetime):
entry[column_name] = str(value)
else:
entry[column_name] = value
except AttributeError as error:
self.log.debug("could not get value")
entries.append(entry)
db[table] = entries
__session__ = sessionmaker(self.engine)
with __session__() as session:
rows = session.query(model).all()
entries = []
self.log.debug(f"found {len(rows)} entries")
self.log.debug(f"found {len(columns)} columns")
for row in rows:
# print(row)
entry = {}
for order in columns:
# print(columns[order])
column_name = columns[order]['column']
# print(f"get value {column_name} from {row} of table {table}")
try:
value = getattr(row, column_name)
if isinstance(value, datetime):
entry[column_name] = str(value)
else:
entry[column_name] = value
except AttributeError as error:
self.log.debug("could not get value")
entries.append(entry)
db[table] = entries
export_file = Path(export_file_name)
match export_type:
case "JSON":
@@ -187,17 +202,31 @@ class KontorDB:
if changed:
print(f"{current_id} has changed")
existing_ids.remove(current_id)
else:
self.log.info("item to import not found in database, add new one...")
if len(existing_ids) > 0:
print("remaining items")
self.session.commit()
def get_ids(self, table_name: str) -> list:
existing_ids = []
items = self.session.query(self.registry[table_name]).all()
for item in items:
existing_ids.append(getattr(item, 'id'))
__session__ = sessionmaker(self.engine)
with __session__() as session:
items = session.query(self.registry[table_name]).all()
for item in items:
existing_ids.append(getattr(item, 'id'))
return existing_ids
def add_entry(self, update_item: dict):
media_file = MediaFile()
__session__ = sessionmaker(self.engine)
with __session__() as session:
for key in update_item.keys():
update_value = update_item[key]
setattr(media_file, key, update_value)
session.add(media_file)
session.commit()
def update_entry(self, existing_item, update_item: dict, dry_run: bool) -> bool:
changed = False
for key in update_item.keys():
@@ -217,3 +246,20 @@ class KontorDB:
def add_link(self, link: str, dry_run: bool):
self.log.info(f"add link {link} to media_file")
__session__ = sessionmaker(self.engine)
with __session__() as session:
media_file = MediaFile()
media_file.id = str(uuid.uuid4())
media_file.created_date = datetime.now()
media_file.last_modified_date = datetime.now()
media_file.version = 0
media_file.url = link
media_file.review = 1
media_file.should_download = 1
try:
session.add(media_file)
session.commit()
self.log.info(f"entry {media_file} successfully added")
except IntegrityError as error:
self.session.rollback()
self.log.info(error.orig)
python/kontor/gui/main_window.py
+3 -3
View File
@@ -12,7 +12,7 @@ from .table_model import KontorTableModel
class MainWindow(QMainWindow):
def __init__(self, session, log):
def __init__(self, session, engine, log):
super().__init__()
self.tick = QIcon('kontor/res/tick.png')
@@ -30,7 +30,7 @@ class MainWindow(QMainWindow):
self.data = []
self.filter = {}
self.kontor_db = KontorDB(session, log)
self.kontor_db = KontorDB(session, engine, log)
self.log = log
self.central_widget = QWidget()
parent_layout = QVBoxLayout()
@@ -114,7 +114,7 @@ class MainWindow(QMainWindow):
self.log.info(export_dlg.get_tables_to_export())
self.log.info(f"export DB to {export_dlg.file_name}")
self.statusBar.showMessage(f"export DB to {export_dlg.file_name}", 3000)
self.kontor_db.export_db(export_dlg.current_export_type, export_dlg.file_name, export_dlg.get_tables_to_export())
self.kontor_db.export_db(export_dlg.current_export_type, export_dlg.file_name)
else:
self.statusBar.showMessage("Export cancelled", 3000)
python/kontor/main.py
+1
View File
@@ -35,6 +35,7 @@ def extend_sqlalchemy(app):
Base.metadata.create_all(bind=engine)
__session__ = sessionmaker(bind=engine)
app.extend('session', __session__())
app.extend('engine', engine)
def close_session(app):
python/setup.py
+1 -1
View File
@@ -19,7 +19,7 @@ setup(
url='https://gitlab.com/tpeetz/kontor',
license='MIT',
packages=find_packages(exclude=['ez_setup', 'tests*']),
package_data={'kontor': ['templates/*']},
package_data={'kontor': ['templates/*', 'res/*',]},
include_package_data=True,
entry_points="""
[console_scripts]