diff --git a/python/kontor-cli/kontor/controllers/__init__.py b/python/kontor-cli/kontor/controllers/__init__.py index e69de29..3f2fa1c 100644 --- a/python/kontor-cli/kontor/controllers/__init__.py +++ b/python/kontor-cli/kontor/controllers/__init__.py @@ -0,0 +1,7 @@ +from enum import Enum + +class ArgumentData(Enum): + EXPORT_TYPE = 'export_type' + DB_FILE = 'db_file' + DATA_TYPE = 'data_type' + DELETE_FIRST = 'delete_first' diff --git a/python/kontor-cli/kontor/controllers/database.py b/python/kontor-cli/kontor/controllers/database.py index c748f75..b5973d2 100644 --- a/python/kontor-cli/kontor/controllers/database.py +++ b/python/kontor-cli/kontor/controllers/database.py @@ -1,6 +1,8 @@ import mariadb from cement import Controller, ex +from kontor.controllers import ArgumentData + class Database(Controller): class Meta: @@ -24,16 +26,16 @@ class Database(Controller): ) def export(self): data = { - 'db_file': 'data.json', - 'export_type': 'JSON', + ArgumentData.DB_FILE: 'data.json', + ArgumentData.EXPORT_TYPE: 'JSON', } if self.app.pargs.db_file is not None: - data['db_file'] = self.app.pargs.db_file + data[ArgumentData.DB_FILE] = self.app.pargs.db_file db = self.app.kontor_db - self.app.log.info(f"export DB to {data['db_file']} as {data['export_type']}") - results = db.export_db(data['export_type'], data['db_file']) - data['results'] = results - self.app.render(data, 'export_db.jinja2') + self.app.log.info(f"export DB to {data[ArgumentData.DB_FILE]} as {data[ArgumentData.EXPORT_TYPE]}") + results = db.export_db(data[ArgumentData.EXPORT_TYPE], data[ArgumentData.DB_FILE]) + for key, value in results.items(): + self.app.log.info(f"{key}: {value}") @ex( label='import', @@ -51,16 +53,18 @@ class Database(Controller): ) def import_cmd(self): data = { - 'db_file': 'data.json', - 'data_type': 'JSON', - 'delete_first': False, + ArgumentData.DB_FILE: 'data.json', + ArgumentData.DATA_TYPE: 'JSON', + ArgumentData.DELETE_FIRST: False, } if self.app.pargs.db_file is not None: - data['db_file'] = self.app.pargs.db_file + data[ArgumentData.DB_FILE] = self.app.pargs.db_file if self.app.pargs.delete_first is not None: - data['delete_first'] = self.app.pargs.delete_first + data[ArgumentData.DELETE_FIRST] = self.app.pargs.delete_first db = self.app.kontor_db - db.import_db(data['db_file'], data['delete_first']) + if data[ArgumentData.DELETE_FIRST]: + db.delete_entries() + db.import_db(data[ArgumentData.DB_FILE]) @ex( help='check the db schema against MetaDataTable and MetaDataColumn' diff --git a/python/kontor-cli/kontor/templates/export_db.jinja2 b/python/kontor-cli/kontor/templates/export_db.jinja2 deleted file mode 100644 index 678aaa3..0000000 --- a/python/kontor-cli/kontor/templates/export_db.jinja2 +++ /dev/null @@ -1,5 +0,0 @@ - Following tables were exported: - -{% for key, value in results.items() %} -Table {{key}}: {{value}} entries -{% endfor %} diff --git a/python/kontor-gui/gui/main_window.py b/python/kontor-gui/gui/main_window.py index a1921d7..e30727a 100644 --- a/python/kontor-gui/gui/main_window.py +++ b/python/kontor-gui/gui/main_window.py @@ -181,7 +181,7 @@ class MainWindow(QMainWindow): import_dlg = ImportKontorDialog(self) if import_dlg.exec(): print(f"import DB from file {import_dlg.file_name}") - self.kontor_db.import_db(import_dlg.file_name, False) + self.kontor_db.import_db(import_dlg.file_name) else: print("do nothing for import") diff --git a/python/kontor-gui/gui/model_config.py b/python/kontor-gui/gui/model_config.py index 7aa05cc..990d3e4 100644 --- a/python/kontor-gui/gui/model_config.py +++ b/python/kontor-gui/gui/model_config.py @@ -1,5 +1,6 @@ from PySide6.QtWidgets import QHBoxLayout, QCheckBox, QMdiSubWindow from kontor_schema import KontorDB +from kontor_schema.database import ColumnEntry class KontorModelConfig: @@ -46,7 +47,7 @@ class KontorModelConfig: filter_layout = QHBoxLayout() for column, filter_info in self.filter.items(): filter_checkbox = QCheckBox() - filter_checkbox.setText(filter_info['label']) + filter_checkbox.setText(filter_info[ColumnEntry.COLUMN_LABEL]) filter_checkbox.checkStateChanged.connect(self.main_window.refresh) self.filter[column]['widget'] = filter_checkbox filter_layout.addWidget(filter_checkbox) diff --git a/python/kontor-gui/gui/table_model.py b/python/kontor-gui/gui/table_model.py index 6c7574b..59340a9 100644 --- a/python/kontor-gui/gui/table_model.py +++ b/python/kontor-gui/gui/table_model.py @@ -2,6 +2,7 @@ from datetime import datetime from PySide6.QtCore import QAbstractTableModel, QModelIndex from PySide6.QtGui import Qt +from kontor_schema.database import ColumnEntry from .model_config import KontorModelConfig @@ -43,7 +44,7 @@ class KontorTableModel(QAbstractTableModel): def headerData(self, col, orientation, role=Qt.ItemDataRole.DisplayRole): if orientation == Qt.Orientation.Horizontal and role == Qt.ItemDataRole.DisplayRole: - return self._config.header[col]['label'] + return self._config.header[col][ColumnEntry.COLUMN_LABEL] if orientation == Qt.Orientation.Vertical and role == Qt.ItemDataRole.DisplayRole: return str(col+1) diff --git a/python/kontor-schema/kontor_schema/__init__.py b/python/kontor-schema/kontor_schema/__init__.py index 5cb8bb6..827b402 100644 --- a/python/kontor-schema/kontor_schema/__init__.py +++ b/python/kontor-schema/kontor_schema/__init__.py @@ -1,357 +1,10 @@ -import json -import uuid -from datetime import datetime -from logging import Logger -from pathlib import Path +from enum import Enum, auto -from sqlalchemy import Engine, select -from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import sessionmaker - -from .base import Base, BaseMixin from .admin import User, Token, Role, AuthorizationMatrix, ModuleData, MailAccount, Mail from .bookshelf import Article, Book, Author, BookshelfPublisher, ArticleAuthor, BookAuthor from .comic import Comic, Artist, Publisher, Issue, StoryArc, TradePaperback, Volume, ComicWork, WorkType from .metadata import MetaDataTable, MetaDataColumn from .tysc import Card, CardSet, Sport, Team, FieldPosition, Rooster, Player, Vendor from .media import MediaFile, MediaArticle, MediaVideo - - -class KontorDB: - - def __init__(self, db_engine: Engine, log: Logger): - self.engine = db_engine - self.registry = {} - self.init_registry() - self.log = log - - def init_registry(self): - self.registry[Card.__tablename__] = Card - self.registry[CardSet.__tablename__] = CardSet - self.registry[Rooster.__tablename__] = Rooster - self.registry[Team.__tablename__] = Team - self.registry[FieldPosition.__tablename__] = FieldPosition - self.registry[Player.__tablename__] = Player - self.registry[Vendor.__tablename__] = Vendor - self.registry[Sport.__tablename__] = Sport - self.registry[Issue.__tablename__] = Issue - self.registry[TradePaperback.__tablename__] = TradePaperback - self.registry[StoryArc.__tablename__] = StoryArc - self.registry[Volume.__tablename__] = Volume - self.registry[ComicWork.__tablename__] = ComicWork - self.registry[Artist.__tablename__] = Artist - self.registry[Comic.__tablename__] = Comic - self.registry[Publisher.__tablename__] = Publisher - self.registry[WorkType.__tablename__] = WorkType - self.registry[ArticleAuthor.__tablename__] = ArticleAuthor - self.registry[BookAuthor.__tablename__] = BookAuthor - self.registry[BookshelfPublisher.__tablename__] = BookshelfPublisher - self.registry[Article.__tablename__] = Article - self.registry[Book.__tablename__] = Book - self.registry[Author.__tablename__] = Author - self.registry[MediaFile.__tablename__] = MediaFile - self.registry[MediaArticle.__tablename__] = MediaArticle - self.registry[MediaVideo.__tablename__] = MediaVideo - self.registry[MetaDataColumn.__tablename__] = MetaDataColumn - self.registry[MetaDataTable.__tablename__] = MetaDataTable - self.registry[AuthorizationMatrix.__tablename__] = AuthorizationMatrix - self.registry[Token.__tablename__] = Token - self.registry[User.__tablename__] = User - self.registry[Role.__tablename__] = Role - self.registry[ModuleData.__tablename__] = ModuleData - self.registry[MailAccount.__tablename__] = MailAccount - self.registry[Mail.__tablename__] = Mail - - def get_table_names(self) -> list: - result = [] - __session__ = sessionmaker(self.engine) - with __session__() as session: - tables = session.scalars(select(MetaDataTable)).all() - result = [table.table_name for table in tables] - return result - - def get_column_meta_data(self, table_name: str, view_only=True) -> dict: - meta_data = {} - order = 0 - __session__ = sessionmaker(self.engine) - with __session__() as session: - if view_only: - for (_, column) in (session.query(MetaDataTable, MetaDataColumn). - filter(MetaDataTable.id == MetaDataColumn.table_id). - filter(MetaDataTable.table_name == table_name). - filter(MetaDataColumn.is_shown == 1).all()): - # self.log.info("get_column_meta_data: %s %s %d", column.column_name, column.column_label, column.column_order) - meta_data[order] = {'column': column.column_name, 'label': column.column_label, - 'order': column.column_order, 'ref_column': column.ref_column} - order += 1 - else: - for (_, column) in (session.query(MetaDataTable, MetaDataColumn). - filter(MetaDataTable.id == MetaDataColumn.table_id). - filter(MetaDataTable.table_name == table_name).all()): - meta_data[order] = { - 'column': column.column_name, - 'order': column.column_order, - 'ref_column': column.ref_column - } - order += 1 - # self.log.info("get_column_meta_data: %s", meta_data) - return meta_data - - def get_columns(self, table_name: str) -> dict: - columns = {} - order = 0 - __session__ = sessionmaker(self.engine) - with __session__() as session: - for (_, column) in (session.query(MetaDataTable, MetaDataColumn). - filter(MetaDataTable.id == MetaDataColumn.table_id). - filter(MetaDataTable.table_name == table_name).all()): - columns[column.column_name] = {"order": column.column_order, "type": column.column_type} - return columns - - def get_filters(self, table_name: str) -> dict: - _filter_map = {} - __session__ = sessionmaker(self.engine) - with __session__() as session: - for (_, column) in (session.query(MetaDataTable, MetaDataColumn). - filter(MetaDataTable.id == MetaDataColumn.table_id). - filter(MetaDataTable.table_name == table_name). - filter(MetaDataColumn.show_filter == 1).all()): - _filter_map[column.column_name] = {'label': column.filter_label, 'widget': None} - return _filter_map - - def data(self, table_name: str, columns: dict, filters: dict) -> list: - data = [] - __session__ = sessionmaker(self.engine) - table = self.registry[table_name] - with __session__() as session: - entries = [] - if len(filters) == 0: - entries = session.scalars(select(table)).all() - else: - entries = session.scalars(select(table).filter_by(**filters)).all() - for entry in entries: - # self.log.info("data: %s", entry) - row = [] - for order in columns.keys(): - column_name = columns[order]['column'] - ref_column = columns[order]['ref_column'] - if str(column_name).endswith("_id"): - ref_table = column_name[:-3] - ref = getattr(entry, ref_table) - value = getattr(ref, ref_column) - row.append(value) - else: - row.append(getattr(entry, column_name)) - data.append(row) - # self.log.info("data: %s", data) - return data - - def export_db(self, export_type: str, export_file_name: str) -> dict: - results = {} - db = {} - export_table_list = self.get_table_names() - for table in export_table_list: - columns = self.get_column_meta_data(table, view_only=False) - if table in self.registry: - model = self.registry[table] - else: - self.log.info(f"table {table} is not registered") - continue - __session__ = sessionmaker(self.engine) - with __session__() as session: - rows = session.query(model).all() - entries = [] - for row in rows: - # print(row) - entry = {} - for order in columns: - # print(columns[order]) - column_name = columns[order]['column'] - # print(f"get value {column_name} from {row} of table {table}") - try: - value = getattr(row, column_name) - if isinstance(value, datetime): - entry[column_name] = str(value) - else: - entry[column_name] = value - except AttributeError: - pass - entries.append(entry) - db[table] = entries - results[table] = len(entries) - match export_type: - case "JSON": - json_dump = json.dumps(db, indent=4) - with open(export_file_name, "w") as dump_file: - dump_file.write(json_dump) - case "YAML": - export_file = Path(export_file_name) - case "SQLite": - export_file = Path(export_file_name) - self.log.info("%d tables exported", len(results)) - return results - - def import_db(self, import_file_name: str, delete_first: bool) -> dict: - result = {} - if delete_first: - self.delete_entries() - import_file = Path(import_file_name) - if not import_file.exists(): - self.log.info(f"File {import_file_name} does not exist. Do nothing.") - return result - match import_file.suffix: - case '.json': - print("read json file") - with open(import_file_name, 'r') as json_file: - json_load = json.load(json_file) - for table in json_load: - self.log.info(f"{table}: {len(json_load[table])}") - result[table] = self.import_table(table, json_load[table]) - case '.yml': - print("read yaml file") - case '.yaml': - print("read yaml file") - case '.db': - print("read sqlite file") - return result - - def import_table(self, table_name: str, items:list) -> dict: - result = {} - updated = [] - added = [] - remaining = [] - existing_ids = self.get_ids(table_name) - self.log.info(f"found {len(existing_ids)} existing ids for table {table_name}") - for item in items: - current_id = item['id'] - # print(f"import item: {item}") - found_item = None - __session__ = sessionmaker(self.engine) - with __session__() as session: - found_item = session.get(self.registry[table_name], current_id) - # print(f"found item: {found_item}") - if found_item is not None: - changed = self.update_entry(table_name, current_id, item) - updated.append(item) - if changed: - self.log.info(f"{current_id} has changed") - updated.append(item) - existing_ids.remove(current_id) - else: - try: - self.add_entry(table_name, item) - added.append(item) - except IntegrityError as error: - self.log.info(f"Could not add item, due to: {error.detail}") - if len(existing_ids) > 0: - print(f"remaining items: {existing_ids}") - remaining.extend(existing_ids) - result['updated'] = updated - result['added'] = added - result['remaining'] = remaining - return result - - def get_ids(self, table_name: str) -> list: - existing_ids = [] - __session__ = sessionmaker(self.engine) - with __session__() as session: - items = session.query(self.registry[table_name]).all() - for item in items: - existing_ids.append(getattr(item, 'id')) - return existing_ids - - def add_entry(self, table_name: str, update_item: dict): - self.log.debug(f"add entry to table {table_name} with {update_item}") - __session__ = sessionmaker(self.engine) - with __session__() as session: - add_item = self.registry[table_name]() - for key in update_item.keys(): - update_value = update_item[key] - setattr(add_item, key, update_value) - session.add(add_item) - session.commit() - - def update_entry(self, table_name, current_id, update_item: dict) -> bool: - # self.log.info("update entry to table %s", table_name) - __session__ = sessionmaker(self.engine) - with __session__() as session: - existing_item = session.query(self.registry[table_name]).get(current_id) - changed = False - for key in update_item.keys(): - update_value = update_item[key] - existing_value = getattr(existing_item, key) - if type(existing_value) is not type(update_value): - existing_value = str(existing_value) - if existing_value != update_value: - self.log.info(f"{key} has changed: {existing_value} != {update_value}") - setattr(existing_item, key, update_value) - session.commit() - changed = True - self.log.info(f"update {key} with {update_value}") - return changed - - def add_link(self, link: str) -> dict: - result = {} - __session__ = sessionmaker(self.engine) - with __session__() as session: - media_file = MediaFile() - media_file.id = str(uuid.uuid4()) - media_file.created_date = datetime.now() - media_file.last_modified_date = datetime.now() - media_file.version = 0 - media_file.url = link - media_file.review = 1 - media_file.should_download = 1 - try: - session.add(media_file) - session.commit() - result['added'] = {'url': media_file.url, 'title': media_file.title, 'review': media_file.review, 'download': media_file.should_download} - except IntegrityError as error: - session.rollback() - result['error'] = error.orig - return result - - def update_titles(self) -> dict: - update_list = {} - __session__ = sessionmaker(self.engine) - with __session__() as session: - links = session.query(MediaFile).filter(MediaFile.review == 1).all() - for link in links: - url = link.url - if url is None: - continue - link.update_title() - session.commit() - update_list[link.id] = link.title - return update_list - - def get_download_list(self) -> list: - download_list = [] - __session__ = sessionmaker(self.engine) - with __session__() as session: - links = session.query(MediaFile).filter(MediaFile.should_download == 1).all() - for link in links: - url = link.url - if url is None: - continue - download_list.append(link.id) - return download_list - - def download_file(self, entry_id: str, download_dir = "/data/media", dl_tool = "yt-dlp") -> str: - __session__ = sessionmaker(self.engine) - with __session__() as session: - link = session.query(MediaFile).get(entry_id) - link.download_file(download_dir, dl_tool) - session.commit() - file_name = link.file_name - return file_name - - def delete_entries(self): - for (table_name, table) in self.registry.items(): - # self.log.info("delete entries from table %s", table_name) - __session__ = sessionmaker(self.engine) - with __session__() as session: - items = session.query(table).all() - for item in items: - session.delete(item) - session.commit() +from .base import Base +from .database import KontorDB diff --git a/python/kontor-schema/kontor_schema/database.py b/python/kontor-schema/kontor_schema/database.py new file mode 100644 index 0000000..f722ef3 --- /dev/null +++ b/python/kontor-schema/kontor_schema/database.py @@ -0,0 +1,388 @@ +import json +import uuid +from datetime import datetime +from enum import Enum, auto +from logging import Logger +from pathlib import Path + +from sqlalchemy import Engine, select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import sessionmaker + +from .tysc import Card, CardSet, Rooster, Team, FieldPosition, Player, Vendor, Sport +from .comic import Issue, TradePaperback, StoryArc, Volume, ComicWork, Artist, Comic, Publisher, WorkType +from .bookshelf import ArticleAuthor, BookAuthor, BookshelfPublisher, Article, Book, Author +from .admin import Mail, MailAccount, ModuleData, Role, User, Token, AuthorizationMatrix +from .metadata import MetaDataTable, MetaDataColumn +from .media import MediaVideo, MediaArticle, MediaFile + + +class ColumnEntry(Enum): + COLUMN_NAME = 'column' + COLUMN_LABEL = 'label' + COLUMN_ORDER = 'order' + COLUMN_REF_COLUMN = 'ref_column' + COLUMN_TYPE = 'type' + COLUMN_WIDGET = 'widget' + + +class StatusType(Enum): + UNKNOWN = auto() + FILE_NAME = auto() + FILE_ID = auto() + DUPLICATE = auto() + CLOUD_LINK = auto() + CLOUD_LINK_ID = auto() + + +class KontorDB: + + def __init__(self, db_engine: Engine, log: Logger): + self.engine = db_engine + self.registry = {} + self.init_registry() + self.log = log + + def init_registry(self): + self.registry[Card.__tablename__] = Card + self.registry[CardSet.__tablename__] = CardSet + self.registry[Rooster.__tablename__] = Rooster + self.registry[Team.__tablename__] = Team + self.registry[FieldPosition.__tablename__] = FieldPosition + self.registry[Player.__tablename__] = Player + self.registry[Vendor.__tablename__] = Vendor + self.registry[Sport.__tablename__] = Sport + self.registry[Issue.__tablename__] = Issue + self.registry[TradePaperback.__tablename__] = TradePaperback + self.registry[StoryArc.__tablename__] = StoryArc + self.registry[Volume.__tablename__] = Volume + self.registry[ComicWork.__tablename__] = ComicWork + self.registry[Artist.__tablename__] = Artist + self.registry[Comic.__tablename__] = Comic + self.registry[Publisher.__tablename__] = Publisher + self.registry[WorkType.__tablename__] = WorkType + self.registry[ArticleAuthor.__tablename__] = ArticleAuthor + self.registry[BookAuthor.__tablename__] = BookAuthor + self.registry[BookshelfPublisher.__tablename__] = BookshelfPublisher + self.registry[Article.__tablename__] = Article + self.registry[Book.__tablename__] = Book + self.registry[Author.__tablename__] = Author + self.registry[MediaFile.__tablename__] = MediaFile + self.registry[MediaArticle.__tablename__] = MediaArticle + self.registry[MediaVideo.__tablename__] = MediaVideo + self.registry[MetaDataColumn.__tablename__] = MetaDataColumn + self.registry[MetaDataTable.__tablename__] = MetaDataTable + self.registry[AuthorizationMatrix.__tablename__] = AuthorizationMatrix + self.registry[Token.__tablename__] = Token + self.registry[User.__tablename__] = User + self.registry[Role.__tablename__] = Role + self.registry[ModuleData.__tablename__] = ModuleData + self.registry[MailAccount.__tablename__] = MailAccount + self.registry[Mail.__tablename__] = Mail + + def get_table_names(self) -> list: + result = [] + __session__ = sessionmaker(self.engine) + with __session__() as session: + tables = session.scalars(select(MetaDataTable)).all() + result = [table.table_name for table in tables] + return result + + def get_column_meta_data(self, table_name: str, view_only=True) -> dict: + meta_data = {} + order = 0 + __session__ = sessionmaker(self.engine) + with __session__() as session: + if view_only: + for (_, column) in (session.query(MetaDataTable, MetaDataColumn). + filter(MetaDataTable.id == MetaDataColumn.table_id). + filter(MetaDataTable.table_name == table_name). + filter(MetaDataColumn.is_shown == 1).all()): + # self.log.info("get_column_meta_data: %s %s %d", column.column_name, column.column_label, column.column_order) + meta_data[order] = { + ColumnEntry.COLUMN_NAME: column.column_name, + ColumnEntry.COLUMN_LABEL: column.column_label, + ColumnEntry.COLUMN_ORDER: column.column_order, + ColumnEntry.COLUMN_REF_COLUMN: column.ref_column, + ColumnEntry.COLUMN_TYPE: column.column_type + } + order += 1 + else: + for (_, column) in (session.query(MetaDataTable, MetaDataColumn). + filter(MetaDataTable.id == MetaDataColumn.table_id). + filter(MetaDataTable.table_name == table_name).all()): + meta_data[order] = { + ColumnEntry.COLUMN_NAME: column.column_name, + ColumnEntry.COLUMN_ORDER: column.column_order, + ColumnEntry.COLUMN_REF_COLUMN: column.ref_column, + ColumnEntry.COLUMN_TYPE: column.column_type + } + order += 1 + # self.log.info("get_column_meta_data: %s", meta_data) + return meta_data + + def get_columns(self, table_name: str) -> dict: + columns = {} + order = 0 + __session__ = sessionmaker(self.engine) + with __session__() as session: + for (_, column) in (session.query(MetaDataTable, MetaDataColumn). + filter(MetaDataTable.id == MetaDataColumn.table_id). + filter(MetaDataTable.table_name == table_name).all()): + columns[column.column_name] = { + ColumnEntry.COLUMN_ORDER: column.column_order, + ColumnEntry.COLUMN_TYPE: column.column_type + } + return columns + + def get_filters(self, table_name: str) -> dict: + _filter_map = {} + __session__ = sessionmaker(self.engine) + with __session__() as session: + for (_, column) in (session.query(MetaDataTable, MetaDataColumn). + filter(MetaDataTable.id == MetaDataColumn.table_id). + filter(MetaDataTable.table_name == table_name). + filter(MetaDataColumn.show_filter == 1).all()): + _filter_map[column.column_name] = { + ColumnEntry.COLUMN_LABEL: column.filter_label, + ColumnEntry.COLUMN_WIDGET: None + } + return _filter_map + + def data(self, table_name: str, columns: dict, filters: dict) -> list: + data = [] + __session__ = sessionmaker(self.engine) + table = self.registry[table_name] + with __session__() as session: + entries = [] + if len(filters) == 0: + entries = session.scalars(select(table)).all() + else: + entries = session.scalars(select(table).filter_by(**filters)).all() + for entry in entries: + # self.log.info("data: %s", entry) + row = [] + for order in columns.keys(): + column_name = columns[order][ColumnEntry.COLUMN_NAME] + ref_column = columns[order][ColumnEntry.COLUMN_REF_COLUMN] + if str(column_name).endswith("_id"): + ref_table = column_name[:-3] + ref = getattr(entry, ref_table) + value = getattr(ref, ref_column) + row.append(value) + else: + row.append(getattr(entry, column_name)) + data.append(row) + # self.log.info("data: %s", data) + return data + + def export_db(self, export_type: str, export_file_name: str) -> dict: + results = {} + db = {} + export_table_list = self.get_table_names() + for table in export_table_list: + columns = self.get_column_meta_data(table, view_only=False) + if table in self.registry: + model = self.registry[table] + else: + self.log.info(f"table {table} is not registered") + continue + __session__ = sessionmaker(self.engine) + with __session__() as session: + rows = session.query(model).all() + entries = [] + for row in rows: + # print(row) + entry = {} + for order in columns: + # print(columns[order]) + column_name = columns[order][ColumnEntry.COLUMN_NAME] + # print(f"get value {column_name} from {row} of table {table}") + try: + value = getattr(row, column_name) + if isinstance(value, datetime): + entry[column_name] = str(value) + else: + entry[column_name] = value + except AttributeError: + pass + entries.append(entry) + db[table] = entries + results[table] = len(entries) + match export_type: + case "JSON": + json_dump = json.dumps(db, indent=4) + with open(export_file_name, "w") as dump_file: + dump_file.write(json_dump) + case "YAML": + export_file = Path(export_file_name) + case "SQLite": + export_file = Path(export_file_name) + self.log.info("%d tables exported", len(results)) + return results + + def import_db(self, import_file_name: str) -> dict: + result = {} + import_file = Path(import_file_name) + if not import_file.exists(): + self.log.info(f"File {import_file_name} does not exist. Do nothing.") + return result + match import_file.suffix: + case '.json': + print("read json file") + with open(import_file_name, 'r') as json_file: + json_load = json.load(json_file) + for table in json_load: + self.log.info(f"{table}: {len(json_load[table])}") + result[table] = self.import_table(table, json_load[table]) + case '.yml': + print("read yaml file") + case '.yaml': + print("read yaml file") + case '.db': + print("read sqlite file") + return result + + def import_table(self, table_name: str, items:list) -> dict: + result = {} + updated = [] + added = [] + remaining = [] + existing_ids = self.get_ids(table_name) + self.log.info(f"found {len(existing_ids)} existing ids for table {table_name}") + for item in items: + current_id = item['id'] + # print(f"import item: {item}") + found_item = None + __session__ = sessionmaker(self.engine) + with __session__() as session: + found_item = session.get(self.registry[table_name], current_id) + # print(f"found item: {found_item}") + if found_item is not None: + changed = self.update_entry(table_name, current_id, item) + updated.append(item) + if changed: + self.log.info(f"{current_id} has changed") + updated.append(item) + existing_ids.remove(current_id) + else: + try: + self.add_entry(table_name, item) + added.append(item) + except IntegrityError as error: + self.log.info(f"Could not add item, due to: {error.detail}") + if len(existing_ids) > 0: + print(f"remaining items: {existing_ids}") + remaining.extend(existing_ids) + result['updated'] = updated + result['added'] = added + result['remaining'] = remaining + return result + + def get_ids(self, table_name: str) -> list: + existing_ids = [] + __session__ = sessionmaker(self.engine) + with __session__() as session: + items = session.query(self.registry[table_name]).all() + for item in items: + existing_ids.append(getattr(item, 'id')) + return existing_ids + + def add_entry(self, table_name: str, update_item: dict): + self.log.debug(f"add entry to table {table_name} with {update_item}") + __session__ = sessionmaker(self.engine) + with __session__() as session: + add_item = self.registry[table_name]() + for key in update_item.keys(): + update_value = update_item[key] + setattr(add_item, key, update_value) + session.add(add_item) + session.commit() + + def update_entry(self, table_name, current_id, update_item: dict) -> bool: + # self.log.info("update entry to table %s", table_name) + __session__ = sessionmaker(self.engine) + with __session__() as session: + existing_item = session.query(self.registry[table_name]).get(current_id) + changed = False + for key in update_item.keys(): + update_value = update_item[key] + existing_value = getattr(existing_item, key) + if type(existing_value) is not type(update_value): + existing_value = str(existing_value) + if existing_value != update_value: + self.log.info(f"{key} has changed: {existing_value} != {update_value}") + setattr(existing_item, key, update_value) + session.commit() + changed = True + self.log.info(f"update {key} with {update_value}") + return changed + + def add_link(self, link: str) -> dict: + result = {} + __session__ = sessionmaker(self.engine) + with __session__() as session: + media_file = MediaFile() + media_file.id = str(uuid.uuid4()) + media_file.created_date = datetime.now() + media_file.last_modified_date = datetime.now() + media_file.version = 0 + media_file.url = link + media_file.review = 1 + media_file.should_download = 1 + try: + session.add(media_file) + session.commit() + result['added'] = {'url': media_file.url, 'title': media_file.title, 'review': media_file.review, 'download': media_file.should_download} + except IntegrityError as error: + session.rollback() + result['error'] = error.orig + return result + + def update_titles(self) -> dict: + update_list = {} + __session__ = sessionmaker(self.engine) + with __session__() as session: + links = session.query(MediaFile).filter(MediaFile.review == 1).all() + for link in links: + url = link.url + if url is None: + continue + link.update_title() + session.commit() + update_list[link.id] = link.title + return update_list + + def get_download_list(self) -> list: + download_list = [] + __session__ = sessionmaker(self.engine) + with __session__() as session: + links = session.query(MediaFile).filter(MediaFile.should_download == 1).all() + for link in links: + url = link.url + if url is None: + continue + download_list.append(link.id) + return download_list + + def download_file(self, entry_id: str, download_dir = "/data/media", dl_tool = "yt-dlp") -> str: + __session__ = sessionmaker(self.engine) + with __session__() as session: + link = session.query(MediaFile).get(entry_id) + link.download_file(download_dir, dl_tool) + session.commit() + file_name = link.file_name + return file_name + + def delete_entries(self): + for (table_name, table) in self.registry.items(): + # self.log.info("delete entries from table %s", table_name) + __session__ = sessionmaker(self.engine) + with __session__() as session: + items = session.query(table).all() + for item in items: + session.delete(item) + session.commit() + + def check_files(self): + pass