reorganize python projects

fastapi/app/routers/media.py

@@ -13,6 +13,25 @@ router = APIRouter(
     tags=["media"]
 )
 
+@router.get("/update-titles")
+def update_titles() -> list[MediaFileResponse]:
+    results: list[MediaFileResponse] = []
+    with __session__() as session:
+        files = session.query(MediaFile).filter(MediaFile.review == 1).all()
+        for mediafile in files:
+            mediafile.update_title()
+            session.add(mediafile)
+            response = MediaFileResponse(id=mediafile.id,
+                                         title=mediafile.title,
+                                         file_name=mediafile.file_name,
+                                         cloud_link= mediafile.cloud_link,
+                                         url=str(mediafile.url),
+                                         review=(mediafile.review == 1),
+                                         should_download=(mediafile.should_download == 1))
+            results.append(response)
+        session.commit()
+    return results
+
 @router.get("/files")
 def get_files() -> list[MediaFileResponse]:
     results: list[MediaFileResponse] = []
@@ -25,7 +44,7 @@ def get_files() -> list[MediaFileResponse]:
                                          cloud_link= mediafile.cloud_link,
                                          url=str(mediafile.url),
                                          review=(mediafile.review == 1),
-                                         shoud_download=(mediafile.should_download==1))
+                                         should_download=(mediafile.should_download == 1))
             results.append(response)
     return results
 

scripts/check_kontor.py

@@ -6,10 +6,12 @@ from enum import Enum, auto
 import mariadb
 from pathlib import Path
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
-from setup import get_database_cursors, get_logger
+
+from config import get_logger, get_database_cursors
 
 parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
 parser.add_argument('--verbose', '-v', action='count', default=0)
+parser.add_argument('--config', '-c', default='kontor-docker')
 parser.add_argument('--dir', '-d', default='/media/tpeetz/Media')
 parser.add_argument('--dry-run', '-m', action='store_true')
 parser.add_argument('--reset-cloud-link', '-r', action='store_true')
@@ -114,9 +116,9 @@ def reset_cloud_link(conn, dry_run):
 
 
 if __name__ == '__main__':
-    logger = get_logger(args.verbose)
+    logger = get_logger(args.verbose, args.config)
     logger.info("kontor.check_kontor started")
-    _, mariadb_conn = get_database_cursors(logger)
+    _, mariadb_conn = get_database_cursors(logger, args.config)
     mariadb_cursor = mariadb_conn.cursor()
     if args.reset_cloud_link:
         reset_cloud_link(mariadb_conn, args.dry_run)

scripts/config.py

@@ -9,8 +9,8 @@ from pathlib import Path
 import yaml
 
 
-def get_database_cursors(log):
+def get_database_cursors(log, config: str):
-    dirs = PlatformDirs("kontor")
+    dirs = PlatformDirs(config)
     database_config = Path(dirs.user_config_dir, 'database-config.yaml')
     with open(database_config, 'rt') as f:
         db_config = yaml.safe_load(f.read())
@@ -40,8 +40,8 @@ def create_tables(sqlite_conn, logger, recreate_db, scripts):
         cursor.execute(create_statement)
 
 
-def get_logger(level):
+def get_logger(level, config: str):
-    dirs = PlatformDirs("kontor")
+    dirs = PlatformDirs(config)
     logging_config = Path(dirs.user_config_dir, 'logging-config.yaml')
     with open(logging_config, 'rt') as f:
         config = yaml.safe_load(f.read())

scripts/copy_to_mariadb.py

@@ -4,7 +4,8 @@ copy data from SQLite to MariaDB
 import sqlite3
 import mariadb
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
-from setup import get_database_cursors, create_tables, get_logger, get_scripts, get_meta_data
+
+from config import get_logger, get_database_cursors, get_meta_data, get_scripts
 
 parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
 parser.add_argument('--recreate-db', action='store_true')

scripts/copy_to_sqlite.py

@@ -4,7 +4,7 @@ copy data from MariaDB to SQLite
 import sqlite3
 import mariadb
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
-from setup import get_database_cursors, create_tables, get_logger, get_meta_data, get_scripts
+from config import get_database_cursors, create_tables, get_logger, get_meta_data, get_scripts
 
 parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
 parser.add_argument('--recreate-db', action='store_true')

scripts/db_structure.py

@@ -3,7 +3,7 @@ Prints the database kontor structure
 """
 import mariadb
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
-from setup import get_database_cursors, get_logger
+from config import get_database_cursors, get_logger
 
 parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
 parser.add_argument('--verbose', '-v', action='count', default=0)

scripts/download.py

@@ -1,73 +1,58 @@
 """
 download files with URLs from DB
 """
-import re
-import subprocess
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
-import mariadb
+from platformdirs import PlatformDirs
-from setup import get_database_cursors, create_tables, get_logger
+from pathlib import Path
+import yaml
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import sessionmaker
+from schema import Base, KontorDB, MediaFile
+from config import get_logger
+
 
 parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
 parser.add_argument('--verbose', '-v', action='count', default=0)
+parser.add_argument('--config', '-c', default='kontor-docker')
 parser.add_argument('--dir', '-d', default='/data/media')
+parser.add_argument('--tool', '-t', default='yt-dlp')
 parser.add_argument('--dry-run', '-m', action='store_true')
 parser.add_argument('--rename', '-r', action='store_true')
 args = parser.parse_args()
 
 
-def parse_output(lines_list, log):
-    file_name = ""
-    for line in lines_list:
-        if 'has already been downloaded' in line:
-            end_len = len(' has already been downloaded')
-            file_name = line[11:-end_len]
-            log.info('found file: "%s"', file_name)
-        if 'Destination' in line:
-            line_len = len(line)
-            start_len = len('[download] Destination: ')
-            file_len = line_len-start_len
-            file_name = line[-file_len:]
-            log.info('new file: "%s"', file_name)
-    return file_name
-
-
-def download_url(video_url, log):
-    result = subprocess.run(["/home/tpeetz/bin/yt-dlp", video_url], cwd=args.dir, capture_output=True, text=True)
-    if result.returncode == 0:
-        output = result.stdout
-        output = re.sub(' +', ' ', output)
-        lines_list = output.splitlines()
-        return parse_output(lines_list, log)
-    else:
-        return None
-
-
-def download_and_update(link, entry_id, conn):
-    m_cursor = conn.cursor()
-    filename = download_url(link, logger)
-    if filename is None:
-        update_statement = 'UPDATE media_file set review = true WHERE id = ?'
-        logger.debug(f'entry {entry_id} could not downloaded, set to Review')
-        m_cursor.execute(update_statement, (entry_id,))
-    else:
-        update_statement = 'UPDATE media_file set file_name = ?, should_download = false, review = false WHERE id = ?'
-        logger.debug(f'entry {entry_id} successfully downloaded, set review and should_download to false')
-        m_cursor.execute(update_statement, (filename, entry_id))
-    conn.commit()
-
-
 if __name__ == '__main__':
-    logger = get_logger(args.verbose)
+    log = get_logger(args.verbose, args.config)
-    logger.info('kontor.download started')
+    log.info('kontor.download started')
-    s_conn, m_conn = get_database_cursors(logger)
+    dirs = PlatformDirs(args.config)
-    cursor = m_conn.cursor()
+    database_config = Path(dirs.user_config_dir, 'database-config.yaml')
-    cursor.execute('SELECT id, url FROM media_file where should_download is true')
+    with open(database_config, 'rt') as f:
-    for (link_id, url) in cursor.fetchall():
+        db_config = yaml.safe_load(f.read())
-        if url is None:
+    print(db_config)
-            logger.info('There is no url for id {}'.format(link_id))
+    connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format(
-        else:
+        db_config['mariadb']['user'],
-            if args.dry_run:
+        db_config['mariadb']['password'],
-                logger.info(f'download {url} for {link_id}')
+        db_config['mariadb']['host'],
-            else:
+        db_config['mariadb']['port'],
-                download_and_update(url, link_id, m_conn)
+        db_config['mariadb']['database']
-    logger.info('kontor.download finished')
+    ))
+    engine = create_engine(connect_string)
+    Base.metadata.create_all(bind=engine, checkfirst=True)
+    __session__ = sessionmaker(bind=engine)
+    _filter = {'should_download': 1}
+    with __session__() as session:
+        files = session.query(MediaFile).filter_by(**_filter).all()
+        log.info("found %d entries", len(files))
+        files2 = session.query(MediaFile).filter(MediaFile.should_download == 1).all()
+        log.info("found %d entries", len(files2))
+        for mediafile in files2:
+            mediafile.download_file(download_dir=args.dir, dl_tool="yt-dlp")
+            log.info("Datei {} erfolgreich heruntergeladen".format(mediafile.file_name))
+            if args.rename:
+                current_file = Path(mediafile.file_name)
+                new_file_path = current_file.with_name(f"{mediafile.id}{current_file.suffix}")
+                current_file.rename(Path(new_file_path))
+                mediafile.cloud_link = new_file_path
+            session.add(mediafile)
+        session.commit()
+    log.info('kontor.download finished')

scripts/import.py

@@ -10,7 +10,7 @@ from platformdirs import PlatformDirs
 from pathlib import Path
 
 from schema import Base, KontorDB
-from config import get_database_cursors, create_tables, get_logger, get_scripts, get_meta_data
+from config import get_logger
 
 parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
 parser.add_argument('--recreate-db', action='store_true')

scripts/schema/__init__.py

@@ -6,3 +6,4 @@ from .tysc import Card, CardSet, Sport, Team, FieldPosition, Rooster, Player, Ve
 from .media import MediaFile, MediaArticle, MediaVideo
 from .base import Base
 from .database import KontorDB, ColumnEntry
+

scripts/schema/database.py

@@ -6,7 +6,7 @@ from logging import Logger
 from pathlib import Path
 from typing import Any
 
-from sqlalchemy import select
+from sqlalchemy import UUID, select
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import sessionmaker
 
@@ -347,6 +347,7 @@ class KontorDB:
         _filter = { 'review': True}
         with __session__() as session:
             links = session.query(MediaFile).filter_by(**_filter).all()
+            self.log.info("%d entries found for updating titles", len(links))
             for link in links:
                 url = link.url
                 if url is None:
@@ -356,7 +357,7 @@ class KontorDB:
                 update_list[link.id] = link.title
         return update_list
 
-    def get_download_list(self) -> list:
+    def get_download_list(self) -> list[UUID]:
         download_list = []
         __session__ = sessionmaker(self.engine)
         _filter = { 'should_download': True}

scripts/setup.py

@@ -1,116 +0,0 @@
-"""
-Setup database connections
-"""
-import sqlite3
-import mariadb
-import logging.config
-from platformdirs import PlatformDirs
-from pathlib import Path
-import yaml
-
-
-def get_database_cursors(log):
-    dirs = PlatformDirs("kontor")
-    database_config = Path(dirs.user_config_dir, 'database-config.yaml')
-    with open(database_config, 'rt') as f:
-        db_config = yaml.safe_load(f.read())
-    sqlite_db = db_config["sqlite"]["file"]
-    log.info('using SQLite3 database {}'.format(sqlite_db))
-    sqlite_conn = sqlite3.connect(sqlite_db, detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES)
-    mariadb_conn = mariadb.connect(
-        host=db_config['mariadb']['host'],
-        port=db_config['mariadb']['port'],
-        user=db_config['mariadb']['user'],
-        password=db_config['mariadb']['password'],
-        database=db_config['mariadb']['database']
-    )
-    return sqlite_conn, mariadb_conn
-
-
-def create_tables(sqlite_conn, logger, recreate_db, scripts):
-    logger.info('create_tables')
-    for table_id in scripts:
-        create_statement = scripts[table_id]['create']
-        drop_statement = scripts[table_id]['drop']
-        logger.debug(create_statement)
-        cursor = sqlite_conn.cursor()
-        if recreate_db:
-            logger.debug(drop_statement)
-            cursor.execute(drop_statement)
-        cursor.execute(create_statement)
-
-
-def get_logger(level):
-    dirs = PlatformDirs("kontor")
-    logging_config = Path(dirs.user_config_dir, 'logging-config.yaml')
-    with open(logging_config, 'rt') as f:
-        config = yaml.safe_load(f.read())
-    logging.config.dictConfig(config)
-    logger = logging.getLogger('development')
-    if level is not None:
-        match level:
-            case 0:
-                logger.setLevel(logging.INFO)
-            case 1:
-                logger.setLevel(logging.DEBUG)
-            case _:
-                logger.setLevel(logging.CRITICAL)
-    return logger
-
-
-def get_meta_data(mariadb_conn):
-    mariadb_cursor = mariadb_conn.cursor()
-    select_statement = "SELECT id, table_name FROM meta_data_table"
-    mariadb_cursor.execute(select_statement)
-    rows = mariadb_cursor.fetchall()
-    meta_data = {}
-    for (identifier, table_name) in rows:
-        table_data = {"name": table_name}
-        mariadb_cursor.execute("SELECT column_name, column_sync_name, column_type, column_modifier, column_order FROM meta_data_column WHERE table_id=?", (identifier, ))
-        column_rows = mariadb_cursor.fetchall()
-        column_list = []
-        for (column_name, column_sync_name, column_type, column_modifier, column_order) in column_rows:
-            column_data = {"column_name": column_name, "column_sync_name": column_sync_name, "column_type": column_type,
-                           "column_modifier": column_modifier, "column_order": column_order}
-            column_list.append(column_data)
-        # logger.info(column_list)
-        table_data["columns"] = column_list
-        meta_data[identifier] = table_data
-    return meta_data
-
-
-def get_scripts(meta_data, logger):
-    scripts_map = {}
-    for table_id in meta_data:
-        table_scripts = {}
-        m_columns = []
-        s_columns = []
-        columns = []
-        for column_data in meta_data[table_id]["columns"]:
-            column_line = "{} {}".format(column_data["column_sync_name"], column_data["column_type"])
-            if column_data["column_modifier"]:
-                column_line += " " + column_data["column_modifier"]
-            columns.append(column_line)
-            m_columns.append(column_data['column_name'])
-            s_columns.append(column_data['column_sync_name'])
-        table_name = meta_data[table_id]["name"]
-        create_statement = "CREATE TABLE IF NOT EXISTS {} ({});".format(table_name, ", ".join(columns))
-        drop_statement = 'DROP TABLE IF EXISTS {}'.format(table_name)
-        select_mariadb_statement = 'SELECT {} FROM {}'.format(', '.join(m_columns), table_name)
-        select_sqlite_statement = 'SELECT {} FROM {}'.format(', '.join(s_columns), table_name)
-        insert_sqlite_statement = 'INSERT INTO {}({}) VALUES({})'.format(table_name, ', '.join(s_columns), ', '.join(['?']*len(s_columns)))
-        insert_mariadb_statement = 'INSERT INTO {}({}) VALUES({})'.format(table_name, ', '.join(m_columns), ', '.join(['?']*len(m_columns)))
-        truncate_mariadb_statement = 'TRUNCATE {}'.format(table_name)
-        #logger.debug(create_statement)
-        #logger.debug(select_mariadb_statement)
-        table_scripts["create"] = create_statement
-        table_scripts["drop"] = drop_statement
-        table_scripts["select_mariadb"] = select_mariadb_statement
-        table_scripts["select_sqlite"] = select_sqlite_statement
-        table_scripts["insert_sqlite"] = insert_sqlite_statement
-        table_scripts["insert_mariadb"] = insert_mariadb_statement
-        table_scripts["truncate_mariadb"] = truncate_mariadb_statement
-        table_scripts["count"] = "SELECT COUNT(*) FROM {}".format(table_name)
-        table_scripts["name"] = table_name
-        scripts_map[table_id] = table_scripts
-    return scripts_map

scripts/sqlmodel/Makefile

@@ -1,31 +0,0 @@
-.PHONY: clean virtualenv test docker dist dist-upload
-
-clean:
-	find . -name '*.py[co]' -delete
-
-virtualenv:
-	virtualenv --prompt '|> kontor <| ' env
-	env/bin/pip install -r requirements-dev.txt
-	env/bin/python setup.py develop
-	@echo
-	@echo "VirtualENV Setup Complete. Now run: source env/bin/activate"
-	@echo
-
-test:
-	python -m pytest \
-		-v \
-		--cov=kontor \
-		--cov-report=term \
-		--cov-report=html:coverage-report \
-		tests/
-
-docker: clean
-	docker build -t kontor-api:latest .
-
-dist: clean
-	rm -rf dist/*
-	python setup.py sdist
-	python setup.py bdist_wheel
-
-dist-upload:
-	twine upload dist/*

scripts/sqlmodel/kontor.py

@@ -1,59 +0,0 @@
-from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
-from pathlib import Path
-
-import logging.config
-import yaml
-from platformdirs import PlatformDirs
-from sqlmodel import SQLModel, create_engine, Session, select
-
-from model import Comic, Publisher
-from model.media import MediaFile
-
-parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
-parser.add_argument('--recreate-db', action='store_true')
-parser.add_argument('--verbose', '-v', action='count', default=0)
-parser.add_argument('--file', '-f', default='~/data.json')
-parser.add_argument('--config', '-c', default='kontor')
-args = parser.parse_args()
-
-
-def get_logger(level: int, config: str):
-    dirs = PlatformDirs(config)
-    logging_config = Path(dirs.user_config_dir, 'logging-config.yaml')
-    with open(logging_config, 'rt') as f:
-        configDict = yaml.safe_load(f.read())
-    logging.config.dictConfig(configDict)
-    logger = logging.getLogger('development')
-    if level is not None:
-        match level:
-            case 0:
-                logger.setLevel(logging.INFO)
-            case 1:
-                logger.setLevel(logging.DEBUG)
-            case _:
-                logger.setLevel(logging.CRITICAL)
-    return logger
-
-
-if __name__ == '__main__':
-    log = get_logger(args.verbose, args.config)
-    log.info('kontor started')
-    dirs = PlatformDirs(args.config)
-    database_config = Path(dirs.user_config_dir, 'database-config.yaml')
-    with open(database_config, 'rt') as f:
-        db_config = yaml.safe_load(f.read())
-    print(db_config)
-    connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format(
-        db_config['mariadb']['user'],
-        db_config['mariadb']['password'],
-        db_config['mariadb']['host'],
-        db_config['mariadb']['port'],
-        db_config['mariadb']['database']
-    ))
-    engine = create_engine(connect_string, echo=True)
-    SQLModel.metadata.create_all(engine)
-    with Session(engine) as session:
-        files = session.exec(select(MediaFile)).all()
-        for file in files:
-            print("{} {}".format(file, file.actors))
-    log.info('kontor finished')

scripts/sqlmodel/model/__init__.py

@@ -1 +0,0 @@
-from .comic import Comic, Publisher, ComicWork, Artist, Worktype

scripts/sqlmodel/model/base.py

@@ -1,11 +0,0 @@
-import uuid
-from datetime import datetime
-
-from sqlmodel import SQLModel, Field
-
-
-class AbstractEntity(SQLModel, table=False):
-    id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True)
-    created_date: datetime = Field(default_factory=datetime.now, nullable=False)
-    last_modified_date: datetime = Field(default_factory=datetime.now, nullable=False)
-    version: int = Field(default=0)

scripts/sqlmodel/model/comic.py

@@ -1,58 +0,0 @@
-import uuid
-from datetime import datetime
-
-from sqlmodel import Field, Relationship, SQLModel
-
-from model.base import AbstractEntity
-
-
-class Publisher(AbstractEntity, table=True):
-    name: str = Field(index=True, unique=True)
-    comics: list["Comic"] = Relationship(back_populates="publisher")
-
-    def __repr__(self):
-        return f'Publisher({self.id} {self.name})'
-
-    def __str__(self):
-        return self.__repr__()
-
-
-class Comic(AbstractEntity, table=True):
-    title: str = Field(index=True, unique=True)
-    publisher_id: uuid.UUID | None = Field(default=None, foreign_key="publisher.id")
-    publisher: Publisher | None = Relationship(back_populates="comics")
-    current_order: int = Field(default=False)
-    completed: int = Field(nullable=False)
-    #issues: list["Issue"] = Relationship(back_populates="comic")
-    #story_arcs: list["StoryArc"] = Relationship(back_populates="comic")
-    #trade_paperbacks: list["TradePaperback"] = Relationship(back_populates="comic")
-    #volumes: list["Volume"] = Relationship(back_populates="comic")
-    #comic_works: list["ComicWork"] = Relationship(back_populates="comic")
-
-
-class Artist(AbstractEntity, table=True):
-    name: str = Field(nullable=False)
-    comic_works: list["ComicWork"] = Relationship(back_populates="artist")
-
-
-class Worktype(AbstractEntity, table=True):
-    name: str = Field(nullable=False, unique=True)
-
-    #comic_works: list["ComicWork"] = Relationship(back_populates="worktype")
-
-    def __repr__(self):
-        return f'Worktype({self.id} {self.version} {self.name} {len(self.comic_works)})'
-
-    def __str__(self):
-        return f'{self.name}({self.id})'
-
-
-class ComicWork(AbstractEntity, table=True):
-    __tablename__ = "comic_work"
-
-    comic_id: uuid.UUID | None = Field(nullable=False, foreign_key="comic.id")
-    #comic: Comic = Relationship(back_populates="comic_works")
-    artist_id: uuid.UUID | None = Field(nullable=False, foreign_key="artist.id")
-    artist: Artist = Relationship(back_populates="comic_works")
-    work_type_id: uuid.UUID | None = Field(nullable=False, foreign_key="worktype.id")
-    #worktype = Relationship(back_populates="comic_works")

scripts/sqlmodel/model/media.py

@@ -1,28 +0,0 @@
-from sqlmodel import Field, Relationship, table
-from uuid import UUID
-from .base import AbstractEntity
-
-
-
-class MediaActorFile(AbstractEntity, table=True):
-    __tablename__ = "media_actor_file"
-
-    media_actor_id: UUID = Field(nullable=False, foreign_key="media_actor.id")
-    media_file_id: UUID = Field(nullable=False, foreign_key="media_file.id")
-
-
-class MediaFile(AbstractEntity, table=True):
-    __tablename__ = "media_file"
-    cloud_link: str = Field(nullable=True, max_length=255)
-    file_name: str = Field(nullable=True, max_length=255)
-    path : str =  Field(nullable=True, max_length=255)
-    title: str =  Field(nullable=True, max_length=255)
-    url: str =  Field(nullable=True, max_length=255)
-    actors : list["MediaActor"] = Relationship(back_populates="videos", link_model=MediaActorFile)
-
-
-class MediaActor(AbstractEntity, table=True):
-    __tablename__ = "media_actor"
-    name: str = Field(nullable=True, max_length=255)
-    videos : list["MediaFile"] = Relationship(back_populates="actors", link_model=MediaActorFile)
-

scripts/sqlmodel/pyproject.toml

@@ -1,29 +0,0 @@
-[build-system]
-requires  = ["setuptools"]
-
-[project]
-name = "kontor-cli"
-version = "0.1.0"
-dependencies = [
-    "mariadb",
-    "sqlmodel",
-    "pathlib",
-    "platformdirs",
-    "pyyaml",
-    "beautifulsoup4",
-]
-requires-python = ">=3.10"
-authors = [
-    {name = "Thomas Peetz", email = "thomas.peetz@thpeetz.de"}
-]
-maintainers = [
-    {name = "Thomas Peetz", email = "thomas.peetz@thpeetz.de"}
-]
-description = "CLI for Kontor application"
-readme = "README.md"
-classifiers = [
-    "Development Status :: 4 - Beta",
-    "Programming Language :: Python"
-]
-[project-scripts]
-kontor = "kontor::main"

scripts/sqlmodel/requirements-dev.txt

@@ -1,8 +0,0 @@
--r requirements.txt
-
-pytest
-pytest-cov
-coverage
-twine>=1.11.0
-setuptools>=38.6.0
-wheel>=0.31.0

scripts/sqlmodel/requirements.txt

@@ -1,7 +0,0 @@
-mariadb
-sqlalchemy
-pathlib
-platformdirs
-pyyaml
-beautifulsoup4
-sqlmodel

scripts/update_title.py

@@ -1,49 +1,66 @@
 """
 download files with URLs from DB
 """
-import re
+import logging.config
-import subprocess
+import yaml
-import datetime
-import logging
-import mariadb
-import requests
-from bs4 import BeautifulSoup
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
-import mariadb
+from pathlib import Path
-from setup import get_database_cursors, get_logger
+from platformdirs import PlatformDirs
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import sessionmaker
+
+from schema import MediaFile, Base
 
 parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
 parser.add_argument('--verbose', '-v', action='count', default=0)
 parser.add_argument('--config', '-c', default='kontor-docker')
 args = parser.parse_args()
 
+def get_logger(level: int, config: str):
+    dirs = PlatformDirs(config)
+    logging_config = Path(dirs.user_config_dir, 'logging-config.yaml')
+    with open(logging_config, 'rt') as f:
+        configDict = yaml.safe_load(f.read())
+    logging.config.dictConfig(configDict)
+    logger = logging.getLogger('development')
+    if level is not None:
+        match level:
+            case 0:
+                logger.setLevel(logging.INFO)
+            case 1:
+                logger.setLevel(logging.DEBUG)
+            case _:
+                logger.setLevel(logging.CRITICAL)
+    return logger
+
 
 if __name__ == '__main__':
-    logger = get_logger(args.verbose)
+    log = get_logger(args.verbose, args.config)
-    logger.info('kontor.download started')
+    log.info('kontor.update_titles started')
-    s_conn, m_conn = get_database_cursors(logger)
+    dirs = PlatformDirs(args.config)
-    cursor = m_conn.cursor()
+    database_config = Path(dirs.user_config_dir, 'database-config.yaml')
-    cursor.execute('SELECT id, url FROM media_file where review is true')
+    with open(database_config, 'rt') as f:
-    for (link_id, url) in cursor.fetchall():
+        db_config = yaml.safe_load(f.read())
-        if url is None:
+    print(db_config)
-            logger.info('There is no url for id {}'.format(link_id))
+    connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format(
-        else:
+        db_config['mariadb']['user'],
-            logger.info('get title for url {}'.format(url))
+        db_config['mariadb']['password'],
-            try:
+        db_config['mariadb']['host'],
-                r = requests.get(url)
+        db_config['mariadb']['port'],
-                soup = BeautifulSoup(r.content, "html.parser")
+        db_config['mariadb']['database']
-                title = soup.title.string
+    ))
-            except:
+    engine = create_engine(connect_string)
-                logger.info("Sorry, could not retrieve title")
+    Base.metadata.create_all(bind=engine, checkfirst=True)
-                update_statement = 'UPDATE media_file set review = true WHERE id = ?'
+    __session__ = sessionmaker(engine)
-                cursor.execute(update_statement, (link_id, ))
+    _filter = {'review': 1}
-            logger.info('ID {} has title {}'.format(link_id, title))
+    with __session__() as session:
-            update = 'UPDATE media_file SET title = ?, review= False where id= ?'
+        files = session.query(MediaFile).filter_by(**_filter).all()
-            try:
+        log.info("found %d entries", len(files))
-                cursor.execute(update, (title, link_id))
+        files2 = session.query(MediaFile).filter(MediaFile.review ==1).all
-                logger.info('entry {} updated'.format(link_id))
+        log.info("found %d entries", len(files2))
-            except mariadb.Error as error:
+        for mediafile in files:
-                logger.info(error)
+            mediafile.update_title()
-            m_conn.commit()
+            session.add(mediafile)
-    logger.info('kontor.download finished')
+        session.commit()
-
+        log.info("found %d entries", len(files))
+    log.info('kontor.update_titles finished')