Vorbereitung Release 0.2.0

2026-01-29 23:50:41 +01:00
parent 58f80b3e76
commit b26b5ecc9c
571 changed files with 35728 additions and 5022 deletions
@@ -1,42 +1,176 @@
 """
-import data from json file to MariaDB
+import data from json file to PostgreSQL
 """
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter

-import yaml
-from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker
-from platformdirs import PlatformDirs
-from pathlib import Path
+from datetime import datetime, date
+from logging import Logger
+from typing import Any, Dict, List
+import os
+import json

-from schema import Base, KontorDB
-from config import get_logger
+from pathlib import Path
+import requests
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker, Session
+from db.models.base import Base
+from db.models import registry
+from psycopg2.errors import NotNullViolation
+from config import get_api_config, get_logger

 parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
 parser.add_argument('--verbose', '-v', action='count', default=0)
-parser.add_argument('--config', '-c', default='kontor-docker')
-parser.add_argument('--recreate-db', action='store_true')
+parser.add_argument("--config", "-c", default="kontor-docker")
+parser.add_argument('--dry-run', '-m', action='store_true')
+parser.add_argument('--cleanup', '-d', action='store_true')
 parser.add_argument('--file', '-f', default='~/data.json')
 args = parser.parse_args()

+DB_USER: str = os.getenv("DB_USER", "kontor")
+DB_PASSWORD: str = os.getenv("DB_PASSWORD", "kontor")
+DB_SERVER: str = os.getenv("DB_SERVER", "127.0.0.1")
+DB_PORT: int = int(os.getenv("DB_PORT", 5432))
+DB_DBNAME: str = os.getenv("DB_DBNAME", "kontor")
+DATABASE_URL: str = f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_SERVER}:{DB_PORT}/{DB_DBNAME}"
+
+
+def cleanup_database(db: Session, log, dry_run: bool):
+    log.debug("cleanup_database")
+    # get tables from registry
+    for table in registry:
+        log.info(f"{table}")
+        model = registry[table]
+        entries = db.query(model).all()
+        for entry in entries:
+            if not dry_run:
+                db.delete(entry)
+                db.commit()
+
+def load_data(filename: str, log) -> Dict[str, List[Dict[str, Any]]]:
+    log.debug("load_data")
+    import_file = Path(filename)
+    if not import_file.exists():
+        log.info(f"File {filename} does not exist. Do nothing.")
+        raise FileNotFoundError()
+    log.info("read json file")
+    with open(filename, 'r') as json_file:
+        json_load = json.load(json_file)
+    return json_load
+
+def get_ids(items: List[Any]) -> List[str]:
+    result: List[str] = []
+    for item in items:
+        result.append(item.id)
+    return result
+
+def update_item(db: Session, import_data: Dict[str, Any], item: Any, dry_run: bool, log):
+    for (key, value) in import_data.items():
+        existing_value = getattr(item, str(key))
+        update: bool = has_changed(existing_value, value, log)
+        #if key == 'published_on':
+        #    log.info(f"{type(value)}:{value} != {type(existing_value)}:{existing_value} : {update}")
+        if update:
+            log.info(f"update {key}({existing_value}) with {value}")
+            if not dry_run:
+                if value == 'None':
+                    setattr(item, str(key), None)
+                else:
+                    setattr(item, str(key), value)
+                db.add(item)
+                db.commit()
+
+def has_changed(existing_data: Any, import_data: str, log) -> bool:
+    if existing_data is None and import_data == 'None':
+        return False
+    if isinstance(existing_data, str):
+        return existing_data != import_data
+    if isinstance(existing_data, date):
+        if len(import_data) > 19:
+            import_date = datetime.strptime(import_data, "%Y-%m-%d %H:%M:%S.%f")
+            log.debug(f"{type(existing_data)}:{existing_data} == {import_date} : {existing_data != import_date}")
+            return existing_data != import_date
+        if len(import_data) > 10:
+            import_date = datetime.strptime(import_data, "%Y-%m-%d %H:%M:%S")
+            log.debug(f"{type(existing_data)}:{existing_data} == {import_date} : {existing_data != import_date}")
+            return existing_data != import_date
+        return existing_data.strftime("%Y-%m-%d") != import_data
+    return existing_data != import_data
+
+
+def item_import(db: Session, import_data: Dict[str, Any], dry_run: bool, log):
+    log.info(f"import {import_data}")
+    if not dry_run:
+        log.debug(f"model: {repr(model)} {import_data}")
+        try:
+            new_item = model()
+            new_item.import_dict(import_data)
+            log.info(f"new item: {new_item}")
+            db.add(new_item)
+            db.commit()
+        except NotNullViolation as notnull:
+            log.info(f"import failed: {notnull} {import_data}")
+        except Exception as error:
+            log.info(f"import failed: {error}")
+
+def item_delete(table_name: str, item_id: str, api_data: Dict[str, Any], log: Logger):
+    log.info(f"delete item {item_id} from {table_name}")
+    host = api_data["host"]
+    port = api_data["port"]
+    token = api_data['token']
+    url = ""
+    match table_name:
+        case "media_file":
+            url = f"http://{host}:{port}/api/media/files/{item_id}"
+        case "media_actor_file":
+            url = f"http://{host}:{port}/api/media/actorfiles/{item_id}"
+        case "media_actor":
+            url = f"http://{host}:{port}/api/media/actors/{item_id}"
+    headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
+    response = requests.delete(url, headers=headers)
+    log.debug(f"Status: {response.status_code}")
+

 if __name__ == '__main__':
-    logger = get_logger(args.verbose, args.config)
+    logger = get_logger(args.verbose, "kontor")
    logger.info('kontor.import started')
-    dirs = PlatformDirs(args.config)
-    database_config = Path(dirs.user_config_dir, 'database-config.yaml')
-    with open(database_config, 'rt') as f:
-        db_config = yaml.safe_load(f.read())
-    connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format(
-        db_config['mariadb']['user'],
-        db_config['mariadb']['password'],
-        db_config['mariadb']['host'],
-        db_config['mariadb']['port'],
-        db_config['mariadb']['database']
-    ))
-    engine = create_engine(connect_string)
+    api_data = get_api_config(logger, args.config)
+    engine = create_engine(DATABASE_URL)
    Base.metadata.create_all(bind=engine, checkfirst=True)
-    __session__ = sessionmaker(bind=engine)
-    kontor_db = KontorDB(engine, logger)
-    kontor_db.import_db(args.file)
+    SessionLocal = sessionmaker(bind=engine)
+    with SessionLocal() as db:
+        if args.cleanup:
+            cleanup_database(db, logger, args.dry_run)
+        data: Dict[str, List[Dict[str, Any]]] = load_data(args.file, logger)
+        table_list: List[str] = list(data.keys())
+        logger.debug(f"Liste der Tabellen: {table_list}")
+        sorted_table_list: List[str] = table_list
+        for tablename in sorted_table_list:
+            model = registry[tablename]
+            existing_items = db.query(model).all()
+            existing_ids: List[str] = get_ids(existing_items)
+            logger.debug(f"found {len(existing_items)} for table {tablename}")
+            import_items: List[Dict[str, Any]] = data[tablename]
+            for import_item in import_items:
+                item_id: str = import_item['id']
+                if item_id in existing_ids:
+                    logger.debug(f"update {item_id}")
+                    existing_item = db.get(model, item_id)
+                    update_item(db, import_item, existing_item, args.dry_run, logger)
+                    existing_ids.remove(item_id)
+                else:
+                    logger.debug(f"import {item_id}")
+                    item_import(db, import_item, args.dry_run, logger)
+            logger.debug(f"remaining items for {tablename}: {len(existing_ids)}")
+            if len(existing_ids) > 0:
+                logger.info(f"remaining items for {tablename}: {existing_ids}")
+            for item_id in existing_ids:
+                match tablename:
+                    case "media_file":
+                        item_delete(table_name=tablename, item_id=item_id, api_data=api_data, log=logger)
+                    case "media_actor_file":
+                        item_delete(table_name=tablename, item_id=item_id, api_data=api_data, log=logger)
+                    case "media_actor":
+                        item_delete(table_name=tablename, item_id=item_id, api_data=api_data, log=logger)
+                    case _:
+                        logger.info("Method to remove remaining item not implemented")
    logger.info('kontor.import finished')