diff --git a/fastapi/app/routers/media.py b/fastapi/app/routers/media.py index 37b8e9e..286774a 100644 --- a/fastapi/app/routers/media.py +++ b/fastapi/app/routers/media.py @@ -13,6 +13,25 @@ router = APIRouter( tags=["media"] ) +@router.get("/update-titles") +def update_titles() -> list[MediaFileResponse]: + results: list[MediaFileResponse] = [] + with __session__() as session: + files = session.query(MediaFile).filter(MediaFile.review == 1).all() + for mediafile in files: + mediafile.update_title() + session.add(mediafile) + response = MediaFileResponse(id=mediafile.id, + title=mediafile.title, + file_name=mediafile.file_name, + cloud_link= mediafile.cloud_link, + url=str(mediafile.url), + review=(mediafile.review == 1), + should_download=(mediafile.should_download == 1)) + results.append(response) + session.commit() + return results + @router.get("/files") def get_files() -> list[MediaFileResponse]: results: list[MediaFileResponse] = [] @@ -25,7 +44,7 @@ def get_files() -> list[MediaFileResponse]: cloud_link= mediafile.cloud_link, url=str(mediafile.url), review=(mediafile.review == 1), - shoud_download=(mediafile.should_download==1)) + should_download=(mediafile.should_download == 1)) results.append(response) return results diff --git a/scripts/sqlalchemy/Makefile b/scripts/Makefile similarity index 100% rename from scripts/sqlalchemy/Makefile rename to scripts/Makefile diff --git a/scripts/sqlalchemy/setup.py b/scripts/__init__.py similarity index 100% rename from scripts/sqlalchemy/setup.py rename to scripts/__init__.py diff --git a/scripts/check_kontor.py b/scripts/check_kontor.py index 8e350cb..81ea1cb 100644 --- a/scripts/check_kontor.py +++ b/scripts/check_kontor.py @@ -6,10 +6,12 @@ from enum import Enum, auto import mariadb from pathlib import Path from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -from setup import get_database_cursors, get_logger + +from config import get_logger, get_database_cursors parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument('--verbose', '-v', action='count', default=0) +parser.add_argument('--config', '-c', default='kontor-docker') parser.add_argument('--dir', '-d', default='/media/tpeetz/Media') parser.add_argument('--dry-run', '-m', action='store_true') parser.add_argument('--reset-cloud-link', '-r', action='store_true') @@ -114,9 +116,9 @@ def reset_cloud_link(conn, dry_run): if __name__ == '__main__': - logger = get_logger(args.verbose) + logger = get_logger(args.verbose, args.config) logger.info("kontor.check_kontor started") - _, mariadb_conn = get_database_cursors(logger) + _, mariadb_conn = get_database_cursors(logger, args.config) mariadb_cursor = mariadb_conn.cursor() if args.reset_cloud_link: reset_cloud_link(mariadb_conn, args.dry_run) diff --git a/scripts/sqlalchemy/config.py b/scripts/config.py similarity index 97% rename from scripts/sqlalchemy/config.py rename to scripts/config.py index 03342a6..3e738e9 100644 --- a/scripts/sqlalchemy/config.py +++ b/scripts/config.py @@ -9,8 +9,8 @@ from pathlib import Path import yaml -def get_database_cursors(log): - dirs = PlatformDirs("kontor") +def get_database_cursors(log, config: str): + dirs = PlatformDirs(config) database_config = Path(dirs.user_config_dir, 'database-config.yaml') with open(database_config, 'rt') as f: db_config = yaml.safe_load(f.read()) @@ -40,8 +40,8 @@ def create_tables(sqlite_conn, logger, recreate_db, scripts): cursor.execute(create_statement) -def get_logger(level): - dirs = PlatformDirs("kontor") +def get_logger(level, config: str): + dirs = PlatformDirs(config) logging_config = Path(dirs.user_config_dir, 'logging-config.yaml') with open(logging_config, 'rt') as f: config = yaml.safe_load(f.read()) diff --git a/scripts/copy_to_mariadb.py b/scripts/copy_to_mariadb.py index bb029be..38d0f24 100644 --- a/scripts/copy_to_mariadb.py +++ b/scripts/copy_to_mariadb.py @@ -4,7 +4,8 @@ copy data from SQLite to MariaDB import sqlite3 import mariadb from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -from setup import get_database_cursors, create_tables, get_logger, get_scripts, get_meta_data + +from config import get_logger, get_database_cursors, get_meta_data, get_scripts parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument('--recreate-db', action='store_true') diff --git a/scripts/copy_to_sqlite.py b/scripts/copy_to_sqlite.py index 3544a96..3455cb4 100644 --- a/scripts/copy_to_sqlite.py +++ b/scripts/copy_to_sqlite.py @@ -4,7 +4,7 @@ copy data from MariaDB to SQLite import sqlite3 import mariadb from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -from setup import get_database_cursors, create_tables, get_logger, get_meta_data, get_scripts +from config import get_database_cursors, create_tables, get_logger, get_meta_data, get_scripts parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument('--recreate-db', action='store_true') diff --git a/scripts/db_structure.py b/scripts/db_structure.py index d363a52..947ab4a 100644 --- a/scripts/db_structure.py +++ b/scripts/db_structure.py @@ -3,7 +3,7 @@ Prints the database kontor structure """ import mariadb from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -from setup import get_database_cursors, get_logger +from config import get_database_cursors, get_logger parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument('--verbose', '-v', action='count', default=0) diff --git a/scripts/download.py b/scripts/download.py index 787ff9d..1d774f6 100644 --- a/scripts/download.py +++ b/scripts/download.py @@ -1,73 +1,58 @@ """ download files with URLs from DB """ -import re -import subprocess from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -import mariadb -from setup import get_database_cursors, create_tables, get_logger +from platformdirs import PlatformDirs +from pathlib import Path +import yaml +from sqlalchemy import create_engine, select +from sqlalchemy.orm import sessionmaker +from schema import Base, KontorDB, MediaFile +from config import get_logger + parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument('--verbose', '-v', action='count', default=0) +parser.add_argument('--config', '-c', default='kontor-docker') parser.add_argument('--dir', '-d', default='/data/media') +parser.add_argument('--tool', '-t', default='yt-dlp') parser.add_argument('--dry-run', '-m', action='store_true') parser.add_argument('--rename', '-r', action='store_true') args = parser.parse_args() -def parse_output(lines_list, log): - file_name = "" - for line in lines_list: - if 'has already been downloaded' in line: - end_len = len(' has already been downloaded') - file_name = line[11:-end_len] - log.info('found file: "%s"', file_name) - if 'Destination' in line: - line_len = len(line) - start_len = len('[download] Destination: ') - file_len = line_len-start_len - file_name = line[-file_len:] - log.info('new file: "%s"', file_name) - return file_name - - -def download_url(video_url, log): - result = subprocess.run(["/home/tpeetz/bin/yt-dlp", video_url], cwd=args.dir, capture_output=True, text=True) - if result.returncode == 0: - output = result.stdout - output = re.sub(' +', ' ', output) - lines_list = output.splitlines() - return parse_output(lines_list, log) - else: - return None - - -def download_and_update(link, entry_id, conn): - m_cursor = conn.cursor() - filename = download_url(link, logger) - if filename is None: - update_statement = 'UPDATE media_file set review = true WHERE id = ?' - logger.debug(f'entry {entry_id} could not downloaded, set to Review') - m_cursor.execute(update_statement, (entry_id,)) - else: - update_statement = 'UPDATE media_file set file_name = ?, should_download = false, review = false WHERE id = ?' - logger.debug(f'entry {entry_id} successfully downloaded, set review and should_download to false') - m_cursor.execute(update_statement, (filename, entry_id)) - conn.commit() - - if __name__ == '__main__': - logger = get_logger(args.verbose) - logger.info('kontor.download started') - s_conn, m_conn = get_database_cursors(logger) - cursor = m_conn.cursor() - cursor.execute('SELECT id, url FROM media_file where should_download is true') - for (link_id, url) in cursor.fetchall(): - if url is None: - logger.info('There is no url for id {}'.format(link_id)) - else: - if args.dry_run: - logger.info(f'download {url} for {link_id}') - else: - download_and_update(url, link_id, m_conn) - logger.info('kontor.download finished') + log = get_logger(args.verbose, args.config) + log.info('kontor.download started') + dirs = PlatformDirs(args.config) + database_config = Path(dirs.user_config_dir, 'database-config.yaml') + with open(database_config, 'rt') as f: + db_config = yaml.safe_load(f.read()) + print(db_config) + connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format( + db_config['mariadb']['user'], + db_config['mariadb']['password'], + db_config['mariadb']['host'], + db_config['mariadb']['port'], + db_config['mariadb']['database'] + )) + engine = create_engine(connect_string) + Base.metadata.create_all(bind=engine, checkfirst=True) + __session__ = sessionmaker(bind=engine) + _filter = {'should_download': 1} + with __session__() as session: + files = session.query(MediaFile).filter_by(**_filter).all() + log.info("found %d entries", len(files)) + files2 = session.query(MediaFile).filter(MediaFile.should_download == 1).all() + log.info("found %d entries", len(files2)) + for mediafile in files2: + mediafile.download_file(download_dir=args.dir, dl_tool="yt-dlp") + log.info("Datei {} erfolgreich heruntergeladen".format(mediafile.file_name)) + if args.rename: + current_file = Path(mediafile.file_name) + new_file_path = current_file.with_name(f"{mediafile.id}{current_file.suffix}") + current_file.rename(Path(new_file_path)) + mediafile.cloud_link = new_file_path + session.add(mediafile) + session.commit() + log.info('kontor.download finished') diff --git a/scripts/sqlalchemy/import.py b/scripts/import.py similarity index 93% rename from scripts/sqlalchemy/import.py rename to scripts/import.py index 6d632bc..ac30a03 100644 --- a/scripts/sqlalchemy/import.py +++ b/scripts/import.py @@ -10,7 +10,7 @@ from platformdirs import PlatformDirs from pathlib import Path from schema import Base, KontorDB -from config import get_database_cursors, create_tables, get_logger, get_scripts, get_meta_data +from config import get_logger parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument('--recreate-db', action='store_true') diff --git a/scripts/sqlalchemy/kontor.py b/scripts/kontor.py similarity index 100% rename from scripts/sqlalchemy/kontor.py rename to scripts/kontor.py diff --git a/scripts/sqlalchemy/pyproject.toml b/scripts/pyproject.toml similarity index 100% rename from scripts/sqlalchemy/pyproject.toml rename to scripts/pyproject.toml diff --git a/scripts/sqlalchemy/requirements-dev.txt b/scripts/requirements-dev.txt similarity index 100% rename from scripts/sqlalchemy/requirements-dev.txt rename to scripts/requirements-dev.txt diff --git a/scripts/sqlalchemy/requirements.txt b/scripts/requirements.txt similarity index 100% rename from scripts/sqlalchemy/requirements.txt rename to scripts/requirements.txt diff --git a/scripts/sqlalchemy/schema/__init__.py b/scripts/schema/__init__.py similarity index 99% rename from scripts/sqlalchemy/schema/__init__.py rename to scripts/schema/__init__.py index 66fabbc..e8a1685 100644 --- a/scripts/sqlalchemy/schema/__init__.py +++ b/scripts/schema/__init__.py @@ -6,3 +6,4 @@ from .tysc import Card, CardSet, Sport, Team, FieldPosition, Rooster, Player, Ve from .media import MediaFile, MediaArticle, MediaVideo from .base import Base from .database import KontorDB, ColumnEntry + diff --git a/scripts/sqlalchemy/schema/admin.py b/scripts/schema/admin.py similarity index 100% rename from scripts/sqlalchemy/schema/admin.py rename to scripts/schema/admin.py diff --git a/scripts/sqlalchemy/schema/base.py b/scripts/schema/base.py similarity index 100% rename from scripts/sqlalchemy/schema/base.py rename to scripts/schema/base.py diff --git a/scripts/sqlalchemy/schema/bookshelf.py b/scripts/schema/bookshelf.py similarity index 100% rename from scripts/sqlalchemy/schema/bookshelf.py rename to scripts/schema/bookshelf.py diff --git a/scripts/sqlalchemy/schema/comic.py b/scripts/schema/comic.py similarity index 100% rename from scripts/sqlalchemy/schema/comic.py rename to scripts/schema/comic.py diff --git a/scripts/sqlalchemy/schema/database.py b/scripts/schema/database.py similarity index 99% rename from scripts/sqlalchemy/schema/database.py rename to scripts/schema/database.py index 013e2f1..1018cd5 100644 --- a/scripts/sqlalchemy/schema/database.py +++ b/scripts/schema/database.py @@ -6,7 +6,7 @@ from logging import Logger from pathlib import Path from typing import Any -from sqlalchemy import select +from sqlalchemy import UUID, select from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import sessionmaker @@ -347,6 +347,7 @@ class KontorDB: _filter = { 'review': True} with __session__() as session: links = session.query(MediaFile).filter_by(**_filter).all() + self.log.info("%d entries found for updating titles", len(links)) for link in links: url = link.url if url is None: @@ -356,7 +357,7 @@ class KontorDB: update_list[link.id] = link.title return update_list - def get_download_list(self) -> list: + def get_download_list(self) -> list[UUID]: download_list = [] __session__ = sessionmaker(self.engine) _filter = { 'should_download': True} diff --git a/scripts/sqlalchemy/schema/media.py b/scripts/schema/media.py similarity index 100% rename from scripts/sqlalchemy/schema/media.py rename to scripts/schema/media.py diff --git a/scripts/sqlalchemy/schema/metadata.py b/scripts/schema/metadata.py similarity index 100% rename from scripts/sqlalchemy/schema/metadata.py rename to scripts/schema/metadata.py diff --git a/scripts/sqlalchemy/schema/tysc.py b/scripts/schema/tysc.py similarity index 100% rename from scripts/sqlalchemy/schema/tysc.py rename to scripts/schema/tysc.py diff --git a/scripts/setup.py b/scripts/setup.py index 03342a6..e69de29 100644 --- a/scripts/setup.py +++ b/scripts/setup.py @@ -1,116 +0,0 @@ -""" -Setup database connections -""" -import sqlite3 -import mariadb -import logging.config -from platformdirs import PlatformDirs -from pathlib import Path -import yaml - - -def get_database_cursors(log): - dirs = PlatformDirs("kontor") - database_config = Path(dirs.user_config_dir, 'database-config.yaml') - with open(database_config, 'rt') as f: - db_config = yaml.safe_load(f.read()) - sqlite_db = db_config["sqlite"]["file"] - log.info('using SQLite3 database {}'.format(sqlite_db)) - sqlite_conn = sqlite3.connect(sqlite_db, detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES) - mariadb_conn = mariadb.connect( - host=db_config['mariadb']['host'], - port=db_config['mariadb']['port'], - user=db_config['mariadb']['user'], - password=db_config['mariadb']['password'], - database=db_config['mariadb']['database'] - ) - return sqlite_conn, mariadb_conn - - -def create_tables(sqlite_conn, logger, recreate_db, scripts): - logger.info('create_tables') - for table_id in scripts: - create_statement = scripts[table_id]['create'] - drop_statement = scripts[table_id]['drop'] - logger.debug(create_statement) - cursor = sqlite_conn.cursor() - if recreate_db: - logger.debug(drop_statement) - cursor.execute(drop_statement) - cursor.execute(create_statement) - - -def get_logger(level): - dirs = PlatformDirs("kontor") - logging_config = Path(dirs.user_config_dir, 'logging-config.yaml') - with open(logging_config, 'rt') as f: - config = yaml.safe_load(f.read()) - logging.config.dictConfig(config) - logger = logging.getLogger('development') - if level is not None: - match level: - case 0: - logger.setLevel(logging.INFO) - case 1: - logger.setLevel(logging.DEBUG) - case _: - logger.setLevel(logging.CRITICAL) - return logger - - -def get_meta_data(mariadb_conn): - mariadb_cursor = mariadb_conn.cursor() - select_statement = "SELECT id, table_name FROM meta_data_table" - mariadb_cursor.execute(select_statement) - rows = mariadb_cursor.fetchall() - meta_data = {} - for (identifier, table_name) in rows: - table_data = {"name": table_name} - mariadb_cursor.execute("SELECT column_name, column_sync_name, column_type, column_modifier, column_order FROM meta_data_column WHERE table_id=?", (identifier, )) - column_rows = mariadb_cursor.fetchall() - column_list = [] - for (column_name, column_sync_name, column_type, column_modifier, column_order) in column_rows: - column_data = {"column_name": column_name, "column_sync_name": column_sync_name, "column_type": column_type, - "column_modifier": column_modifier, "column_order": column_order} - column_list.append(column_data) - # logger.info(column_list) - table_data["columns"] = column_list - meta_data[identifier] = table_data - return meta_data - - -def get_scripts(meta_data, logger): - scripts_map = {} - for table_id in meta_data: - table_scripts = {} - m_columns = [] - s_columns = [] - columns = [] - for column_data in meta_data[table_id]["columns"]: - column_line = "{} {}".format(column_data["column_sync_name"], column_data["column_type"]) - if column_data["column_modifier"]: - column_line += " " + column_data["column_modifier"] - columns.append(column_line) - m_columns.append(column_data['column_name']) - s_columns.append(column_data['column_sync_name']) - table_name = meta_data[table_id]["name"] - create_statement = "CREATE TABLE IF NOT EXISTS {} ({});".format(table_name, ", ".join(columns)) - drop_statement = 'DROP TABLE IF EXISTS {}'.format(table_name) - select_mariadb_statement = 'SELECT {} FROM {}'.format(', '.join(m_columns), table_name) - select_sqlite_statement = 'SELECT {} FROM {}'.format(', '.join(s_columns), table_name) - insert_sqlite_statement = 'INSERT INTO {}({}) VALUES({})'.format(table_name, ', '.join(s_columns), ', '.join(['?']*len(s_columns))) - insert_mariadb_statement = 'INSERT INTO {}({}) VALUES({})'.format(table_name, ', '.join(m_columns), ', '.join(['?']*len(m_columns))) - truncate_mariadb_statement = 'TRUNCATE {}'.format(table_name) - #logger.debug(create_statement) - #logger.debug(select_mariadb_statement) - table_scripts["create"] = create_statement - table_scripts["drop"] = drop_statement - table_scripts["select_mariadb"] = select_mariadb_statement - table_scripts["select_sqlite"] = select_sqlite_statement - table_scripts["insert_sqlite"] = insert_sqlite_statement - table_scripts["insert_mariadb"] = insert_mariadb_statement - table_scripts["truncate_mariadb"] = truncate_mariadb_statement - table_scripts["count"] = "SELECT COUNT(*) FROM {}".format(table_name) - table_scripts["name"] = table_name - scripts_map[table_id] = table_scripts - return scripts_map diff --git a/scripts/sqlmodel/Makefile b/scripts/sqlmodel/Makefile deleted file mode 100644 index 407957a..0000000 --- a/scripts/sqlmodel/Makefile +++ /dev/null @@ -1,31 +0,0 @@ -.PHONY: clean virtualenv test docker dist dist-upload - -clean: - find . -name '*.py[co]' -delete - -virtualenv: - virtualenv --prompt '|> kontor <| ' env - env/bin/pip install -r requirements-dev.txt - env/bin/python setup.py develop - @echo - @echo "VirtualENV Setup Complete. Now run: source env/bin/activate" - @echo - -test: - python -m pytest \ - -v \ - --cov=kontor \ - --cov-report=term \ - --cov-report=html:coverage-report \ - tests/ - -docker: clean - docker build -t kontor-api:latest . - -dist: clean - rm -rf dist/* - python setup.py sdist - python setup.py bdist_wheel - -dist-upload: - twine upload dist/* diff --git a/scripts/sqlmodel/kontor.py b/scripts/sqlmodel/kontor.py deleted file mode 100644 index 8e62dd3..0000000 --- a/scripts/sqlmodel/kontor.py +++ /dev/null @@ -1,59 +0,0 @@ -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -from pathlib import Path - -import logging.config -import yaml -from platformdirs import PlatformDirs -from sqlmodel import SQLModel, create_engine, Session, select - -from model import Comic, Publisher -from model.media import MediaFile - -parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) -parser.add_argument('--recreate-db', action='store_true') -parser.add_argument('--verbose', '-v', action='count', default=0) -parser.add_argument('--file', '-f', default='~/data.json') -parser.add_argument('--config', '-c', default='kontor') -args = parser.parse_args() - - -def get_logger(level: int, config: str): - dirs = PlatformDirs(config) - logging_config = Path(dirs.user_config_dir, 'logging-config.yaml') - with open(logging_config, 'rt') as f: - configDict = yaml.safe_load(f.read()) - logging.config.dictConfig(configDict) - logger = logging.getLogger('development') - if level is not None: - match level: - case 0: - logger.setLevel(logging.INFO) - case 1: - logger.setLevel(logging.DEBUG) - case _: - logger.setLevel(logging.CRITICAL) - return logger - - -if __name__ == '__main__': - log = get_logger(args.verbose, args.config) - log.info('kontor started') - dirs = PlatformDirs(args.config) - database_config = Path(dirs.user_config_dir, 'database-config.yaml') - with open(database_config, 'rt') as f: - db_config = yaml.safe_load(f.read()) - print(db_config) - connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format( - db_config['mariadb']['user'], - db_config['mariadb']['password'], - db_config['mariadb']['host'], - db_config['mariadb']['port'], - db_config['mariadb']['database'] - )) - engine = create_engine(connect_string, echo=True) - SQLModel.metadata.create_all(engine) - with Session(engine) as session: - files = session.exec(select(MediaFile)).all() - for file in files: - print("{} {}".format(file, file.actors)) - log.info('kontor finished') diff --git a/scripts/sqlmodel/model/__init__.py b/scripts/sqlmodel/model/__init__.py deleted file mode 100644 index 7af2d43..0000000 --- a/scripts/sqlmodel/model/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .comic import Comic, Publisher, ComicWork, Artist, Worktype diff --git a/scripts/sqlmodel/model/base.py b/scripts/sqlmodel/model/base.py deleted file mode 100644 index 80adc8f..0000000 --- a/scripts/sqlmodel/model/base.py +++ /dev/null @@ -1,11 +0,0 @@ -import uuid -from datetime import datetime - -from sqlmodel import SQLModel, Field - - -class AbstractEntity(SQLModel, table=False): - id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True) - created_date: datetime = Field(default_factory=datetime.now, nullable=False) - last_modified_date: datetime = Field(default_factory=datetime.now, nullable=False) - version: int = Field(default=0) diff --git a/scripts/sqlmodel/model/comic.py b/scripts/sqlmodel/model/comic.py deleted file mode 100644 index 53b7c7d..0000000 --- a/scripts/sqlmodel/model/comic.py +++ /dev/null @@ -1,58 +0,0 @@ -import uuid -from datetime import datetime - -from sqlmodel import Field, Relationship, SQLModel - -from model.base import AbstractEntity - - -class Publisher(AbstractEntity, table=True): - name: str = Field(index=True, unique=True) - comics: list["Comic"] = Relationship(back_populates="publisher") - - def __repr__(self): - return f'Publisher({self.id} {self.name})' - - def __str__(self): - return self.__repr__() - - -class Comic(AbstractEntity, table=True): - title: str = Field(index=True, unique=True) - publisher_id: uuid.UUID | None = Field(default=None, foreign_key="publisher.id") - publisher: Publisher | None = Relationship(back_populates="comics") - current_order: int = Field(default=False) - completed: int = Field(nullable=False) - #issues: list["Issue"] = Relationship(back_populates="comic") - #story_arcs: list["StoryArc"] = Relationship(back_populates="comic") - #trade_paperbacks: list["TradePaperback"] = Relationship(back_populates="comic") - #volumes: list["Volume"] = Relationship(back_populates="comic") - #comic_works: list["ComicWork"] = Relationship(back_populates="comic") - - -class Artist(AbstractEntity, table=True): - name: str = Field(nullable=False) - comic_works: list["ComicWork"] = Relationship(back_populates="artist") - - -class Worktype(AbstractEntity, table=True): - name: str = Field(nullable=False, unique=True) - - #comic_works: list["ComicWork"] = Relationship(back_populates="worktype") - - def __repr__(self): - return f'Worktype({self.id} {self.version} {self.name} {len(self.comic_works)})' - - def __str__(self): - return f'{self.name}({self.id})' - - -class ComicWork(AbstractEntity, table=True): - __tablename__ = "comic_work" - - comic_id: uuid.UUID | None = Field(nullable=False, foreign_key="comic.id") - #comic: Comic = Relationship(back_populates="comic_works") - artist_id: uuid.UUID | None = Field(nullable=False, foreign_key="artist.id") - artist: Artist = Relationship(back_populates="comic_works") - work_type_id: uuid.UUID | None = Field(nullable=False, foreign_key="worktype.id") - #worktype = Relationship(back_populates="comic_works") diff --git a/scripts/sqlmodel/model/media.py b/scripts/sqlmodel/model/media.py deleted file mode 100644 index fea5642..0000000 --- a/scripts/sqlmodel/model/media.py +++ /dev/null @@ -1,28 +0,0 @@ -from sqlmodel import Field, Relationship, table -from uuid import UUID -from .base import AbstractEntity - - - -class MediaActorFile(AbstractEntity, table=True): - __tablename__ = "media_actor_file" - - media_actor_id: UUID = Field(nullable=False, foreign_key="media_actor.id") - media_file_id: UUID = Field(nullable=False, foreign_key="media_file.id") - - -class MediaFile(AbstractEntity, table=True): - __tablename__ = "media_file" - cloud_link: str = Field(nullable=True, max_length=255) - file_name: str = Field(nullable=True, max_length=255) - path : str = Field(nullable=True, max_length=255) - title: str = Field(nullable=True, max_length=255) - url: str = Field(nullable=True, max_length=255) - actors : list["MediaActor"] = Relationship(back_populates="videos", link_model=MediaActorFile) - - -class MediaActor(AbstractEntity, table=True): - __tablename__ = "media_actor" - name: str = Field(nullable=True, max_length=255) - videos : list["MediaFile"] = Relationship(back_populates="actors", link_model=MediaActorFile) - diff --git a/scripts/sqlmodel/pyproject.toml b/scripts/sqlmodel/pyproject.toml deleted file mode 100644 index 13dc13c..0000000 --- a/scripts/sqlmodel/pyproject.toml +++ /dev/null @@ -1,29 +0,0 @@ -[build-system] -requires = ["setuptools"] - -[project] -name = "kontor-cli" -version = "0.1.0" -dependencies = [ - "mariadb", - "sqlmodel", - "pathlib", - "platformdirs", - "pyyaml", - "beautifulsoup4", -] -requires-python = ">=3.10" -authors = [ - {name = "Thomas Peetz", email = "thomas.peetz@thpeetz.de"} -] -maintainers = [ - {name = "Thomas Peetz", email = "thomas.peetz@thpeetz.de"} -] -description = "CLI for Kontor application" -readme = "README.md" -classifiers = [ - "Development Status :: 4 - Beta", - "Programming Language :: Python" -] -[project-scripts] -kontor = "kontor::main" diff --git a/scripts/sqlmodel/requirements-dev.txt b/scripts/sqlmodel/requirements-dev.txt deleted file mode 100644 index f20606e..0000000 --- a/scripts/sqlmodel/requirements-dev.txt +++ /dev/null @@ -1,8 +0,0 @@ --r requirements.txt - -pytest -pytest-cov -coverage -twine>=1.11.0 -setuptools>=38.6.0 -wheel>=0.31.0 diff --git a/scripts/sqlmodel/requirements.txt b/scripts/sqlmodel/requirements.txt deleted file mode 100644 index 1e00fdc..0000000 --- a/scripts/sqlmodel/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -mariadb -sqlalchemy -pathlib -platformdirs -pyyaml -beautifulsoup4 -sqlmodel diff --git a/scripts/sqlmodel/setup.py b/scripts/sqlmodel/setup.py deleted file mode 100644 index e69de29..0000000 diff --git a/scripts/update_title.py b/scripts/update_title.py index 3107630..a91df45 100644 --- a/scripts/update_title.py +++ b/scripts/update_title.py @@ -1,49 +1,66 @@ """ download files with URLs from DB """ -import re -import subprocess -import datetime -import logging -import mariadb -import requests -from bs4 import BeautifulSoup +import logging.config +import yaml from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -import mariadb -from setup import get_database_cursors, get_logger +from pathlib import Path +from platformdirs import PlatformDirs +from sqlalchemy import create_engine, select +from sqlalchemy.orm import sessionmaker + +from schema import MediaFile, Base parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument('--verbose', '-v', action='count', default=0) parser.add_argument('--config', '-c', default='kontor-docker') args = parser.parse_args() +def get_logger(level: int, config: str): + dirs = PlatformDirs(config) + logging_config = Path(dirs.user_config_dir, 'logging-config.yaml') + with open(logging_config, 'rt') as f: + configDict = yaml.safe_load(f.read()) + logging.config.dictConfig(configDict) + logger = logging.getLogger('development') + if level is not None: + match level: + case 0: + logger.setLevel(logging.INFO) + case 1: + logger.setLevel(logging.DEBUG) + case _: + logger.setLevel(logging.CRITICAL) + return logger + if __name__ == '__main__': - logger = get_logger(args.verbose) - logger.info('kontor.download started') - s_conn, m_conn = get_database_cursors(logger) - cursor = m_conn.cursor() - cursor.execute('SELECT id, url FROM media_file where review is true') - for (link_id, url) in cursor.fetchall(): - if url is None: - logger.info('There is no url for id {}'.format(link_id)) - else: - logger.info('get title for url {}'.format(url)) - try: - r = requests.get(url) - soup = BeautifulSoup(r.content, "html.parser") - title = soup.title.string - except: - logger.info("Sorry, could not retrieve title") - update_statement = 'UPDATE media_file set review = true WHERE id = ?' - cursor.execute(update_statement, (link_id, )) - logger.info('ID {} has title {}'.format(link_id, title)) - update = 'UPDATE media_file SET title = ?, review= False where id= ?' - try: - cursor.execute(update, (title, link_id)) - logger.info('entry {} updated'.format(link_id)) - except mariadb.Error as error: - logger.info(error) - m_conn.commit() - logger.info('kontor.download finished') - + log = get_logger(args.verbose, args.config) + log.info('kontor.update_titles started') + dirs = PlatformDirs(args.config) + database_config = Path(dirs.user_config_dir, 'database-config.yaml') + with open(database_config, 'rt') as f: + db_config = yaml.safe_load(f.read()) + print(db_config) + connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format( + db_config['mariadb']['user'], + db_config['mariadb']['password'], + db_config['mariadb']['host'], + db_config['mariadb']['port'], + db_config['mariadb']['database'] + )) + engine = create_engine(connect_string) + Base.metadata.create_all(bind=engine, checkfirst=True) + __session__ = sessionmaker(engine) + _filter = {'review': 1} + with __session__() as session: + files = session.query(MediaFile).filter_by(**_filter).all() + log.info("found %d entries", len(files)) + files2 = session.query(MediaFile).filter(MediaFile.review ==1).all + log.info("found %d entries", len(files2)) + for mediafile in files: + mediafile.update_title() + session.add(mediafile) + session.commit() + log.info("found %d entries", len(files)) + log.info('kontor.update_titles finished')