From 727e95e732d3b945f113a88201480498f4f0b5b9 Mon Sep 17 00:00:00 2001 From: Thomas Peetz Date: Mon, 21 Apr 2025 21:45:06 +0200 Subject: [PATCH] setup kontor-schema --- kontor-api/src/models/tysc.py | 8 - kontor-api/src/schema/database.py | 29 +- kontor-api/src/schema/media.py | 5 +- .../build/lib/kontor_schema/__init__.py | 366 ------------------ kontor-schema/build/lib/kontor_schema/base.py | 20 - .../build/lib/kontor_schema/media.py | 39 -- kontor-scripts/download.py | 16 +- kontor-scripts/export.py | 6 +- kontor-scripts/json_to_mariadb.py | 1 - kontor-scripts/requirements-dev.txt | 8 - kontor-scripts/schema/__init__.py | 9 - kontor-scripts/schema/database.py | 25 +- kontor-scripts/schema/media.py | 19 +- kontor-spring/docker-compose.yml | 41 -- 14 files changed, 50 insertions(+), 542 deletions(-) delete mode 100644 kontor-api/src/models/tysc.py delete mode 100644 kontor-schema/build/lib/kontor_schema/__init__.py delete mode 100644 kontor-schema/build/lib/kontor_schema/base.py delete mode 100644 kontor-schema/build/lib/kontor_schema/media.py delete mode 100644 kontor-scripts/requirements-dev.txt delete mode 100644 kontor-spring/docker-compose.yml diff --git a/kontor-api/src/models/tysc.py b/kontor-api/src/models/tysc.py deleted file mode 100644 index 8ddbfb3..0000000 --- a/kontor-api/src/models/tysc.py +++ /dev/null @@ -1,8 +0,0 @@ -from uuid import UUID - -from pydantic import BaseModel - - -class SportResponse(BaseModel): - id: UUID - name: str diff --git a/kontor-api/src/schema/database.py b/kontor-api/src/schema/database.py index c242eb0..e4e35eb 100644 --- a/kontor-api/src/schema/database.py +++ b/kontor-api/src/schema/database.py @@ -1,4 +1,5 @@ import json +import logging import uuid from datetime import datetime from enum import Enum, auto @@ -43,11 +44,10 @@ class ExportType(Enum): class KontorDB: - def __init__(self, db_engine: Any, log: Logger): + def __init__(self, db_engine: Any): self.engine = db_engine self.registry = {} self.init_registry() - self.log = log def init_registry(self): self.registry[Card.__tablename__] = Card @@ -131,7 +131,6 @@ class KontorDB: def get_columns(self, table_name: str) -> dict: columns = {} - order = 0 __session__ = sessionmaker(self.engine) table_info = self.get_table_by_name(table_name) _filters = {'table_id': table_info['id']} @@ -192,7 +191,7 @@ class KontorDB: if table in self.registry: model = self.registry[table] else: - self.log.info(f"table {table} is not registered") + logging.info(f"table {table} is not registered") continue __session__ = sessionmaker(self.engine) with __session__() as session: @@ -222,17 +221,17 @@ class KontorDB: with open(export_file_name, "w") as dump_file: dump_file.write(json_dump) case "YAML": - export_file = Path(export_file_name) + pass case "SQLite": - export_file = Path(export_file_name) - self.log.info(f"{len(results)} tables exported") + pass + logging.info(f"{len(results)} tables exported") return results def import_db(self, import_file_name: str) -> dict: result = {} import_file = Path(import_file_name) if not import_file.exists(): - self.log.info(f"File {import_file_name} does not exist. Do nothing.") + logging.info(f"File {import_file_name} does not exist. Do nothing.") return result match import_file.suffix: case '.json': @@ -240,7 +239,7 @@ class KontorDB: with open(import_file_name, 'r') as json_file: json_load = json.load(json_file) for table in json_load: - self.log.info(f"{table}: {len(json_load[table])}") + logging.info(f"{table}: {len(json_load[table])}") result[table] = self.import_table(table, json_load[table]) case '.yml': print("read yaml file") @@ -256,7 +255,7 @@ class KontorDB: added = [] remaining = [] existing_ids = self.get_ids(table_name) - self.log.info(f"found {len(existing_ids)} existing ids for table {table_name}") + logging.info(f"found {len(existing_ids)} existing ids for table {table_name}") for item in items: current_id = item['id'] # print(f"import item: {item}") @@ -269,7 +268,7 @@ class KontorDB: changed = self.update_entry(table_name, current_id, item) updated.append(item) if changed: - self.log.info(f"{current_id} has changed") + logging.info(f"{current_id} has changed") updated.append(item) existing_ids.remove(current_id) else: @@ -277,7 +276,7 @@ class KontorDB: self.add_entry(table_name, item) added.append(item) except IntegrityError as error: - self.log.info(f"Could not add item, due to: {error.detail}") + logging.info(f"Could not add item, due to: {error.detail}") if len(existing_ids) > 0: print(f"remaining items for {table_name}: {existing_ids}") remaining.extend(existing_ids) @@ -296,7 +295,7 @@ class KontorDB: return existing_ids def add_entry(self, table_name: str, update_item: dict): - self.log.debug(f"add entry to table {table_name} with {update_item}") + logging.debug(f"add entry to table {table_name} with {update_item}") __session__ = sessionmaker(self.engine) with __session__() as session: add_item = self.registry[table_name]() @@ -318,11 +317,11 @@ class KontorDB: if type(existing_value) is not type(update_value): existing_value = str(existing_value) if existing_value != update_value: - self.log.info(f"{key} has changed: {existing_value} != {update_value}") + logging.info(f"{key} has changed: {existing_value} != {update_value}") setattr(existing_item, key, update_value) session.commit() changed = True - self.log.info(f"update {key} with {update_value}") + logging.info(f"update {key} with {update_value}") return changed def add_link(self, link: str) -> dict: diff --git a/kontor-api/src/schema/media.py b/kontor-api/src/schema/media.py index d1c568f..c1616f9 100644 --- a/kontor-api/src/schema/media.py +++ b/kontor-api/src/schema/media.py @@ -1,3 +1,4 @@ +import logging import re import subprocess from datetime import datetime @@ -23,7 +24,7 @@ class MediaFile(Base, BaseMixin, BaseVideoMixin): return f'{self.title}({self.id})' def update_title(self) -> None: - print(f"update title for {self.url}") + logging.info(f"update title for {self.url}") try: r = requests.get(self.url) soup = BeautifulSoup(r.content, "html.parser") @@ -36,7 +37,7 @@ class MediaFile(Base, BaseMixin, BaseVideoMixin): self.last_modified_date = datetime.now() def download_file(self, download_dir: str, dl_tool: str): - print(f"download file for {self.url} to {download_dir}") + logging.info(f"download file for {self.url} to {download_dir}") result = subprocess.run([dl_tool, self.url], cwd=download_dir, capture_output=True, text=True) if result.returncode == 0: output = result.stdout diff --git a/kontor-schema/build/lib/kontor_schema/__init__.py b/kontor-schema/build/lib/kontor_schema/__init__.py deleted file mode 100644 index 3c7ff98..0000000 --- a/kontor-schema/build/lib/kontor_schema/__init__.py +++ /dev/null @@ -1,366 +0,0 @@ -import json -import re -import subprocess -import uuid -from datetime import datetime -from pathlib import Path - -import requests -from bs4 import BeautifulSoup -from cement.core.config import ConfigHandler -from sqlalchemy import Engine -from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import sessionmaker - -from .bookshelf import Article, Book, Author, BookshelfPublisher, ArticleAuthor, BookAuthor -from .comic import Comic, Artist, Publisher, Issue, StoryArc, TradePaperback, Volume, ComicWork, WorkType -from .metadata import MetaDataTable, MetaDataColumn -from .tysc import Card, CardSet, Sport, Team, FieldPosition, Rooster, Player, Vendor -from .media import MediaFile, MediaArticle, MediaVideo - - -class KontorDB: - - def __init__(self, db_engine: Engine, config: ConfigHandler, log): - self.engine = db_engine - self.config = config - self.log = log - self.registry = {} - self.init_registry() - - def init_registry(self): - self.registry['card'] = Card - self.registry['card_set'] = CardSet - self.registry['sport'] = Sport - self.registry['team'] = Team - self.registry['field_position'] = FieldPosition - self.registry['rooster'] = Rooster - self.registry['player'] = Player - self.registry['vendor'] = Vendor - self.registry['artist'] = Artist - self.registry['publisher'] = Publisher - self.registry['comic'] = Comic - self.registry['issue'] = Issue - self.registry['story_arc'] = StoryArc - self.registry['trade_paperback'] = TradePaperback - self.registry['volume'] = Volume - self.registry['comic_work'] = ComicWork - self.registry['worktype'] = WorkType - self.registry['article'] = Article - self.registry['book'] = Book - self.registry['author'] = Author - self.registry['bookshelf_publisher'] = BookshelfPublisher - self.registry['article_author'] = ArticleAuthor - self.registry['book_author'] = BookAuthor - self.registry['media_file'] = MediaFile - self.registry['media_article'] = MediaArticle - self.registry['media_video'] = MediaVideo - self.registry['meta_data_table'] = MetaDataTable - self.registry[MetaDataColumn.__tablename__] = MetaDataColumn - - def get_table_names(self) -> list: - result = [] - __session__ = sessionmaker(self.engine) - with __session__() as session: - tables = session.query(MetaDataTable).all() - result = [table.table_name for table in tables] - return result - - def get_column_meta_data(self, table_name: str, view_only=True) -> dict: - meta_data = {} - order = 0 - __session__ = sessionmaker(self.engine) - with __session__() as session: - if view_only: - for (_, column) in (session.query(MetaDataTable, MetaDataColumn). - filter(MetaDataTable.id == MetaDataColumn.table_id). - filter(MetaDataTable.table_name == table_name). - filter(MetaDataColumn.is_shown == 1).all()): - meta_data[order] = {'column': column.column_name, 'label': column.column_label, - 'order': column.column_order, 'ref_column': column.ref_column} - order += 1 - else: - for (_, column) in (session.query(MetaDataTable, MetaDataColumn). - filter(MetaDataTable.id == MetaDataColumn.table_id). - filter(MetaDataTable.table_name == table_name).all()): - meta_data[order] = { - 'column': column.column_name, - 'order': column.column_order, - 'ref_column': column.ref_column - } - order += 1 - return meta_data - - def get_filters(self, table_name): - _filter_map = {} - __session__ = sessionmaker(self.engine) - with __session__() as session: - for (_, column) in (session.query(MetaDataTable, MetaDataColumn). - filter(MetaDataTable.id == MetaDataColumn.table_id). - filter(MetaDataTable.table_name == table_name). - filter(MetaDataColumn.show_filter == 1).all()): - _filter_map[column.column_name] = {'label': column.filter_label, 'widget': None} - self.log.debug(f"retrieved {len(_filter_map)} filters: {_filter_map}") - return _filter_map - - def data(self, table, columns: dict, filters) -> list: - data = [] - __session__ = sessionmaker(self.engine) - with __session__() as session: - entries = [] - if len(filters) == 0: - entries = session.query(table).all() - else: - entries = session.query(table).filter_by(**filters) - for entry in entries: - row = [] - for order in columns.keys(): - column_name = columns[order]['column'] - if str(column_name).endswith("_id"): - ref_table = column_name[:-3] - # print(f"{ref_table=}") - ref = getattr(entry, ref_table) - value = getattr(ref, "name") - # print(f"{value=}") - row.append(value) - else: - row.append(getattr(entry, column_name)) - # print(repr(row)) - data.append(row) - return data - - def export_db(self, export_type: str, export_file_name: str): - self.log.info(f"export DB to {export_file_name} as {export_type}") - db = {} - export_table_list = self.get_table_names() - for table in export_table_list: - columns = self.get_column_meta_data(table, view_only=False) - if table in self.registry: - model = self.registry[table] - else: - print(f"table {table} is not registered") - continue - __session__ = sessionmaker(self.engine) - with __session__() as session: - rows = session.query(model).all() - entries = [] - self.log.debug(f"found {len(rows)} entries") - self.log.debug(f"found {len(columns)} columns") - for row in rows: - # print(row) - entry = {} - for order in columns: - # print(columns[order]) - column_name = columns[order]['column'] - # print(f"get value {column_name} from {row} of table {table}") - try: - value = getattr(row, column_name) - if isinstance(value, datetime): - entry[column_name] = str(value) - else: - entry[column_name] = value - except AttributeError: - self.log.debug("could not get value") - entries.append(entry) - db[table] = entries - export_file = Path(export_file_name) - match export_type: - case "JSON": - json_dump = json.dumps(db, indent=4) - with open(export_file_name, "w") as dump_file: - dump_file.write(json_dump) - case "YAML": - export_file = Path(export_file_name) - case "SQLite": - export_file = Path(export_file_name) - case _: - self.log.debug("unknown export type") - if export_file.exists(): - self.log.debug(f"{export_file} exists") - - def import_db(self, import_file_name: str, dry_run: bool): - import_file = Path(import_file_name) - if not import_file.exists(): - print(f"File {import_file_name} does not exist. Do nothing.") - return - self.log.debug(f"evaluate type from file extension: {import_file.suffix}") - match import_file.suffix: - case '.json': - print("read json file") - with open(import_file_name, 'r') as json_file: - json_load = json.load(json_file) - for table in json_load: - print(f"{table}: {len(json_load[table])}") - self.import_table(table, json_load[table], dry_run) - case '.yml': - print("read yaml file") - case '.yaml': - print("read yaml file") - case '.db': - print("read sqlite file") - - def import_table(self, table_name, items, dry_run: bool): - existing_ids = self.get_ids(table_name) - for item in items: - # self.log.debug(f"{item}") - current_id = item['id'] - found_item = None - __session__ = sessionmaker(self.engine) - with __session__() as session: - found_item = session.query(self.registry[table_name]).get(current_id) - self.log.debug(f"found: {found_item}") - if found_item is not None: - changed = self.update_entry(found_item, item, dry_run) - if changed: - print(f"{current_id} has changed") - existing_ids.remove(current_id) - else: - self.log.info("item to import not found in database, add new one...") - self.add_entry(table_name, item, session, dry_run) - if len(existing_ids) > 0: - print("remaining items") - - def get_ids(self, table_name: str) -> list: - existing_ids = [] - __session__ = sessionmaker(self.engine) - with __session__() as session: - items = session.query(self.registry[table_name]).all() - for item in items: - existing_ids.append(getattr(item, 'id')) - return existing_ids - - def add_entry(self, table_name: str, update_item: dict, session, dry_run: bool): - add_item = self.registry[table_name]() - for key in update_item.keys(): - update_value = update_item[key] - setattr(add_item, key, update_value) - if dry_run: - self.log.info(f"add item {type(add_item)} with id {update_item['id']}") - else: - session.add(add_item) - session.commit() - - def update_entry(self, existing_item, update_item: dict, dry_run: bool) -> bool: - changed = False - for key in update_item.keys(): - update_value = update_item[key] - existing_value = getattr(existing_item, key) - if type(existing_value) is not type(update_value): - # self.log.debug(f"compare {type(existing_value)} with {type(update_value)}") - existing_value = str(existing_value) - if existing_value != update_value: - print(f"{key} has changed: {existing_value} != {update_value}") - if not dry_run: - setattr(existing_item, key, update_value) - # existing_item[key] = update_value - changed = True - self.log.info(f"update {key} with {update_value}") - return changed - - def add_link(self, link: str, dry_run: bool): - self.log.info(f"add link {link} to media_file") - __session__ = sessionmaker(self.engine) - with __session__() as session: - media_file = MediaFile() - media_file.id = str(uuid.uuid4()) - media_file.created_date = datetime.now() - media_file.last_modified_date = datetime.now() - media_file.version = 0 - media_file.url = link - media_file.review = 1 - media_file.should_download = 1 - try: - session.add(media_file) - session.commit() - self.log.info(f"entry {media_file} successfully added") - except IntegrityError as error: - session.rollback() - self.log.info(error.orig) - - def update_title(self, dry_run=False): - self.log.info("get links to review of media_file") - __session__ = sessionmaker(self.engine) - with __session__() as session: - links = session.query(MediaFile).filter(MediaFile.review == 1).all() - self.log.info(f"try to update {len(links)} items") - for link in links: - url = link.url - if url is None: - self.log.info(f"url has not been set for {link.id}") - continue - self.log.info('get title for url {}'.format(url)) - if dry_run: - continue - try: - r = requests.get(url) - soup = BeautifulSoup(r.content, "html.parser") - title = soup.title.string - except: - self.log.info("Sorry, could not retrieve title") - continue - self.log.info('ID {} has title {}'.format(link.id, title)) - link.title = title - link.review = 0 - session.commit() - - def download_file(self, dry_run=False): - self.log.info("download marked files of media_file") - __session__ = sessionmaker(self.engine) - with __session__() as session: - links = session.query(MediaFile).filter(MediaFile.should_download == 1).all() - self.log.info(f"try to download {len(links)} items") - for link in links: - url = link.url - if url is None: - self.log.info(f"url has not been set for {link.id}") - continue - if dry_run: - self.log.info(f"download {link.url} to {self.config.get('media', 'dir')}") - continue - filename = self.download_url(link) - if filename is None: - link.file_name = filename - link.should_download = 1 - else: - download_file = Path(filename) - download_file.with_name(f"{link.id}{download_file.suffix}") - link.file_name = download_file.name - link.should_download = 0 - link.cloud_link = download_file.absolute() - session.commit() - - def parse_output(self, lines_list): - file_name = "" - for line in lines_list: - if 'has already been downloaded' in line: - end_len = len(' has already been downloaded') - file_name = line[11:-end_len] - self.log.info('found file: "%s"', file_name) - if 'Destination' in line: - line_len = len(line) - start_len = len('[download] Destination: ') - file_len = line_len - start_len - file_name = line[-file_len:] - self.log.info('new file: "%s"', file_name) - return file_name - - def download_url(self, video_url): - media_dir = Path(self.config.get('media', 'dir')) - if not media_dir.exists(): - media_dir = Path().absolute() - self.log.info(f"download video to {media_dir}") - result = subprocess.run([self.config.get('media', 'yt-dlp'), video_url], cwd=media_dir, capture_output=True, - text=True) - if result.returncode == 0: - output = result.stdout - output = re.sub(' +', ' ', output) - lines_list = output.splitlines() - return self.parse_output(lines_list) - else: - return None - - def check_files(self): - media_dir = Path(self.config.get('media', 'dir')) - if not media_dir.exists(): - return - self.log.info(f"check files in {media_dir}") diff --git a/kontor-schema/build/lib/kontor_schema/base.py b/kontor-schema/build/lib/kontor_schema/base.py deleted file mode 100644 index 21186d4..0000000 --- a/kontor-schema/build/lib/kontor_schema/base.py +++ /dev/null @@ -1,20 +0,0 @@ -import uuid -from datetime import datetime - -from sqlalchemy import func -from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column - - -class Base(DeclarativeBase): - pass - - -class BaseMixin: - # id = Column(String, primary_key=True) - id: Mapped[str] = mapped_column(primary_key=True, default=uuid.uuid4()) - # created_date = Column(DateTime) - created_date: Mapped[datetime] = mapped_column(default=func.now()) - # last_modified_date = Column(DateTime) - last_modified_date: Mapped[datetime] = mapped_column(default=func.now()) - # version = Column(Integer) - version: Mapped[int] = mapped_column(default=0) diff --git a/kontor-schema/build/lib/kontor_schema/media.py b/kontor-schema/build/lib/kontor_schema/media.py deleted file mode 100644 index 2f8e865..0000000 --- a/kontor-schema/build/lib/kontor_schema/media.py +++ /dev/null @@ -1,39 +0,0 @@ -from sqlalchemy import Column, String -from sqlalchemy.dialects.mysql import BIT - -from .base import Base, BaseMixin - - -class MediaFile(Base, BaseMixin): - __tablename__ = 'media_file' - cloud_link = Column(String(255)) - file_name = Column(String(255)) - path = Column(String(255)) - review = Column(BIT(1)) - title = Column(String(255)) - url = Column(String(255), unique=True) - should_download = Column(BIT(1)) - - def __repr__(self): - return f'MediaFile({self.id} {self.title} {self.title})' - - def __str__(self): - return f'{self.title}({self.id})' - - -class MediaArticle(Base, BaseMixin): - __tablename__ = 'media_article' - review = Column(BIT(1)) - title = Column(String(255)) - url = Column(String(255), unique=True) - - -class MediaVideo(Base, BaseMixin): - __tablename__ = 'media_video' - cloud_link = Column(String(255)) - file_name = Column(String(255)) - path = Column(String(255)) - review = Column(BIT(1)) - title = Column(String(255)) - url = Column(String(255), unique=True) - should_download = Column(BIT(1)) diff --git a/kontor-scripts/download.py b/kontor-scripts/download.py index d74d9f9..2faa422 100644 --- a/kontor-scripts/download.py +++ b/kontor-scripts/download.py @@ -61,19 +61,19 @@ def __parse_output__(lines_list: list[str]) -> str | None: return file_name -def is_file_downloaded(item: dict, dir: Path) -> FileStatus: - file_name_as_title = f"{item['file_name']}" - file_title = Path(dir, file_name_as_title, ".mp4") +def is_file_downloaded(media_file: dict, dir: Path) -> FileStatus: + file_name_as_title = f"{media_file['file_name']}" + file_title = Path(dir, f"{file_name_as_title}.mp4") if file_title.exists(): log.info(f"{file_name_as_title} has been downloaded") - item['should_download'] = 0 + media_file['should_download'] = False return FileStatus.DOWNLOADED - file_name_as_id = f"{item['id']}" - file_with_id_as_name = Path(dir, file_name_as_id, ".mp4") + file_name_as_id = f"{media_file['id']}" + file_with_id_as_name = Path(dir, f"{file_name_as_id}.mp4") if file_with_id_as_name.exists(): log.info(f"{file_with_id_as_name} has been downloaded and renamed") - item['cloud_link'] = file_with_id_as_name - item['should_download'] = 0 + media_file['cloud_link'] = file_with_id_as_name + media_file['should_download'] = False return FileStatus.RENAMED log.info("could not find file - start download") return FileStatus.UNKNOWN diff --git a/kontor-scripts/export.py b/kontor-scripts/export.py index 13ccc93..913ce71 100644 --- a/kontor-scripts/export.py +++ b/kontor-scripts/export.py @@ -11,12 +11,12 @@ from pathlib import Path from schema import Base, KontorDB from config import get_logger +from schema.database import ExportType parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument('--verbose', '-v', action='count', default=0) parser.add_argument('--config', '-c', default='kontor-docker') -parser.add_argument('--recreate-db', action='store_true') -parser.add_argument('--file', '-f', default='~/data.json') +parser.add_argument('--file', '-f', default='data.json') args = parser.parse_args() @@ -38,5 +38,5 @@ if __name__ == '__main__': Base.metadata.create_all(bind=engine, checkfirst=True) __session__ = sessionmaker(bind=engine) kontor_db = KontorDB(engine, logger) - kontor_db.export_db("JSON", args.file) + kontor_db.export_db(ExportType.JSON, args.file) logger.info('kontor.export finished') diff --git a/kontor-scripts/json_to_mariadb.py b/kontor-scripts/json_to_mariadb.py index 22fa8be..e1c855f 100644 --- a/kontor-scripts/json_to_mariadb.py +++ b/kontor-scripts/json_to_mariadb.py @@ -15,7 +15,6 @@ args = parser.parse_args() def copy_data(mariadb_conn, data_file: Path, log): mariadb_cursor = mariadb_conn.cursor() - result = {} import_file = Path(data_file) if not import_file.exists(): log.info(f"File {data_file} does not exist. Do nothing.") diff --git a/kontor-scripts/requirements-dev.txt b/kontor-scripts/requirements-dev.txt deleted file mode 100644 index f20606e..0000000 --- a/kontor-scripts/requirements-dev.txt +++ /dev/null @@ -1,8 +0,0 @@ --r requirements.txt - -pytest -pytest-cov -coverage -twine>=1.11.0 -setuptools>=38.6.0 -wheel>=0.31.0 diff --git a/kontor-scripts/schema/__init__.py b/kontor-scripts/schema/__init__.py index e8a1685..e69de29 100644 --- a/kontor-scripts/schema/__init__.py +++ b/kontor-scripts/schema/__init__.py @@ -1,9 +0,0 @@ -from .admin import User, Token, Role, AuthorizationMatrix, ModuleData, MailAccount, Mail -from .bookshelf import Article, Book, Author, BookshelfPublisher, ArticleAuthor, BookAuthor -from .comic import Comic, Artist, Publisher, Issue, StoryArc, TradePaperback, Volume, ComicWork, WorkType -from .metadata import MetaDataTable, MetaDataColumn -from .tysc import Card, CardSet, Sport, Team, FieldPosition, Rooster, Player, Vendor -from .media import MediaFile, MediaArticle, MediaVideo -from .base import Base -from .database import KontorDB, ColumnEntry - diff --git a/kontor-scripts/schema/database.py b/kontor-scripts/schema/database.py index fd166d6..87f9669 100644 --- a/kontor-scripts/schema/database.py +++ b/kontor-scripts/schema/database.py @@ -35,6 +35,7 @@ class StatusType(Enum): CLOUD_LINK = auto() CLOUD_LINK_ID = auto() + class ExportType(Enum): JSON = "JSON" YAML = "YAML" @@ -131,7 +132,6 @@ class KontorDB: def get_columns(self, table_name: str) -> dict: columns = {} - order = 0 __session__ = sessionmaker(self.engine) table_info = self.get_table_by_name(table_name) _filters = {'table_id': table_info['id']} @@ -183,7 +183,7 @@ class KontorDB: # self.log.info("data: %s", data) return data - def export_db(self, export_type: str, export_file_name: str) -> dict: + def export_db(self, export_type: ExportType, export_file_name: str) -> dict: results = {} db = {} export_table_list = self.get_table_names() @@ -217,14 +217,14 @@ class KontorDB: db[table] = entries results[table] = len(entries) match export_type: - case "JSON": + case ExportType.JSON: json_dump = json.dumps(db, indent=4) with open(export_file_name, "w") as dump_file: dump_file.write(json_dump) - case "YAML": - export_file = Path(export_file_name) - case "SQLite": - export_file = Path(export_file_name) + case ExportType.YAML: + pass + case ExportType.SQLITE: + pass self.log.info(f"{len(results)} tables exported") return results @@ -250,7 +250,7 @@ class KontorDB: print("read sqlite file") return result - def import_table(self, table_name: str, items:list) -> dict: + def import_table(self, table_name: str, items: list) -> dict: result = {} updated = [] added = [] @@ -340,7 +340,8 @@ class KontorDB: try: session.add(media_file) session.commit() - result['added'] = {'url': media_file.url, 'title': media_file.title, 'review': media_file.review, 'download': media_file.should_download} + result['added'] = {'url': media_file.url, 'title': media_file.title, 'review': media_file.review, + 'download': media_file.should_download} except IntegrityError as error: session.rollback() result['error'] = error.orig @@ -349,7 +350,7 @@ class KontorDB: def update_titles(self) -> dict: update_list = {} __session__ = sessionmaker(self.engine) - _filter = { 'review': True} + _filter = {'review': True} with __session__() as session: links = session.query(MediaFile).filter_by(**_filter).all() self.log.info("%d entries found for updating titles", len(links)) @@ -365,7 +366,7 @@ class KontorDB: def get_download_list(self) -> list[UUID]: download_list = [] __session__ = sessionmaker(self.engine) - _filter = { 'should_download': True} + _filter = {'should_download': True} with __session__() as session: links = session.query(MediaFile).filter_by(**_filter).all() for link in links: @@ -375,7 +376,7 @@ class KontorDB: download_list.append(link.id) return download_list - def download_file(self, entry_id: str, download_dir = "/data/media", dl_tool = "yt-dlp") -> str: + def download_file(self, entry_id: str, download_dir="/data/media", dl_tool="yt-dlp") -> str: __session__ = sessionmaker(self.engine) with __session__() as session: link = session.query(MediaFile).get(entry_id) diff --git a/kontor-scripts/schema/media.py b/kontor-scripts/schema/media.py index d1c568f..5dcc5c6 100644 --- a/kontor-scripts/schema/media.py +++ b/kontor-scripts/schema/media.py @@ -5,8 +5,7 @@ from pathlib import Path import requests from bs4 import BeautifulSoup -from sqlalchemy import Column, String, ForeignKey -from sqlalchemy.dialects.mysql import BIT +from sqlalchemy import Boolean, Column, False_, String, ForeignKey from sqlalchemy.orm import relationship from .base import Base, BaseMixin, BaseVideoMixin @@ -29,10 +28,10 @@ class MediaFile(Base, BaseMixin, BaseVideoMixin): soup = BeautifulSoup(r.content, "html.parser") title = soup.title.string self.title = title - self.review = 0 + self.review = False_ except: self.title = None - self.review = 1 + self.review = True self.last_modified_date = datetime.now() def download_file(self, download_dir: str, dl_tool: str): @@ -44,12 +43,12 @@ class MediaFile(Base, BaseMixin, BaseVideoMixin): lines_list = output.splitlines() file_name = self.__parse_output__(lines_list) if file_name is None: - self.review = 1 - self.should_download = 1 + self.review = True + self.should_download = True self.file_name = None else: download_file = Path(file_name) - self.should_download = 0 + self.should_download = False_ self.file_name = download_file.name self.cloud_link = str(download_file.absolute()) self.last_modified_date = datetime.now() @@ -84,7 +83,7 @@ class MediaActorFile(Base, BaseMixin): class MediaArticle(Base, BaseMixin): __tablename__ = 'media_article' - review = Column(BIT(1)) + review = Column(Boolean) title = Column(String(255)) url = Column(String(255), unique=True) @@ -94,7 +93,7 @@ class MediaVideo(Base, BaseMixin): cloud_link = Column(String(255)) file_name = Column(String(255)) path = Column(String(255)) - review = Column(BIT(1)) + review = Column(Boolean) title = Column(String(255)) url = Column(String(255), unique=True) - should_download = Column(BIT(1)) + should_download = Column(Boolean) diff --git a/kontor-spring/docker-compose.yml b/kontor-spring/docker-compose.yml deleted file mode 100644 index 2b36ef1..0000000 --- a/kontor-spring/docker-compose.yml +++ /dev/null @@ -1,41 +0,0 @@ - -services: - mariadb: - image: mariadb - restart: unless-stopped - environment: - MYSQL_ROOT_PASSWORD: kontor - MYSQL_USER: kontor - MYSQL_PASSWORD: kontor - MYSQL_DATABASE: kontor - ports: - - 3316:3306 - networks: - - database - volumes: - - mariadb-storage:/var/lib/mysql:rw - kontor: - image: kontor - restart: unless-stopped - networks: - - database - - frontend - ports: - - 8000:8000 - kontor-api: - image: kontor-api - restart: unless-stopped - networks: - - database - - frontend - ports: - - 8800:8800 - - -networks: - database: - frontend: - -volumes: - mariadb-storage: -