""" import data from json file to PostgreSQL """ import json import os from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser from datetime import date, datetime from logging import Logger from pathlib import Path from typing import Any, Dict, List import requests from psycopg2.errors import NotNullViolation from sqlalchemy import create_engine from sqlalchemy.orm import Session, sessionmaker from config import get_api_config, get_logger from db.models import registry from db.models.base import Base parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("--config", "-c", default="kontor-docker") parser.add_argument("--dry-run", "-m", action="store_true") parser.add_argument("--cleanup", "-d", action="store_true") parser.add_argument("--file", "-f", default="~/data.json") args = parser.parse_args() DB_USER: str = os.getenv("DB_USER", "kontor") DB_PASSWORD: str = os.getenv("DB_PASSWORD", "kontor") DB_SERVER: str = os.getenv("DB_SERVER", "127.0.0.1") DB_PORT: int = int(os.getenv("DB_PORT", 5432)) DB_DBNAME: str = os.getenv("DB_DBNAME", "kontor") DATABASE_URL: str = ( f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_SERVER}:{DB_PORT}/{DB_DBNAME}" ) def cleanup_database(db: Session, log, dry_run: bool): log.debug("cleanup_database") # get tables from registry for table in registry: log.info(f"{table}") model = registry[table] entries = db.query(model).all() for entry in entries: if not dry_run: db.delete(entry) db.commit() def load_data(filename: str, log) -> Dict[str, List[Dict[str, Any]]]: log.debug("load_data") import_file = Path(filename) if not import_file.exists(): log.info(f"File {filename} does not exist. Do nothing.") raise FileNotFoundError() log.info("read json file") with open(filename, "r") as json_file: json_load = json.load(json_file) return json_load def get_ids(items: List[Any]) -> List[str]: result: List[str] = [] for item in items: result.append(item.id) return result def update_item( db: Session, import_data: Dict[str, Any], item: Any, dry_run: bool, log ): for key, value in import_data.items(): existing_value = getattr(item, str(key)) update: bool = has_changed(existing_value, value, log) # if key == 'published_on': # log.info(f"{type(value)}:{value} != {type(existing_value)}:{existing_value} : {update}") if update: log.info(f"update {key}({existing_value}) with {value}") if not dry_run: if value == "None": setattr(item, str(key), None) else: setattr(item, str(key), value) db.add(item) db.commit() def has_changed(existing_data: Any, import_data: str, log) -> bool: if existing_data is None and import_data == "None": return False if isinstance(existing_data, str): return existing_data != import_data if isinstance(existing_data, date): if len(import_data) > 19: import_date = datetime.strptime(import_data, "%Y-%m-%d %H:%M:%S.%f") log.debug( f"{type(existing_data)}:{existing_data} == {import_date} : {existing_data != import_date}" ) return existing_data != import_date if len(import_data) > 10: import_date = datetime.strptime(import_data, "%Y-%m-%d %H:%M:%S") log.debug( f"{type(existing_data)}:{existing_data} == {import_date} : {existing_data != import_date}" ) return existing_data != import_date return existing_data.strftime("%Y-%m-%d") != import_data return existing_data != import_data def item_import(db: Session, import_data: Dict[str, Any], dry_run: bool, log): log.info(f"import {import_data}") if not dry_run: log.debug(f"model: {repr(model)} {import_data}") try: new_item = model() new_item.import_dict(import_data) log.info(f"new item: {new_item}") db.add(new_item) db.commit() except NotNullViolation as notnull: log.info(f"import failed: {notnull} {import_data}") except Exception as error: log.info(f"import failed: {error}") def item_delete(table_name: str, item_id: str, api_data: Dict[str, Any], log: Logger): log.info(f"delete item {item_id} from {table_name}") host = api_data["host"] port = api_data["port"] token = api_data["token"] url = "" match table_name: case "media_file": url = f"http://{host}:{port}/api/media/files/{item_id}" case "media_actor_file": url = f"http://{host}:{port}/api/media/actorfiles/{item_id}" case "media_actor": url = f"http://{host}:{port}/api/media/actors/{item_id}" case "profile": url = f"http://{host}:{port}/api/user/profile/{item_id}" headers: Dict[str, str] = {"Authorization": f"Bearer {token}"} response = requests.delete(url, headers=headers) log.debug(f"Status: {response.status_code}") if __name__ == "__main__": logger = get_logger(args.verbose, "kontor") logger.info("kontor.import started") api_data = get_api_config(logger, args.config) engine = create_engine(DATABASE_URL) Base.metadata.create_all(bind=engine, checkfirst=True) SessionLocal = sessionmaker(bind=engine) with SessionLocal() as db: if args.cleanup: cleanup_database(db, logger, args.dry_run) data: Dict[str, List[Dict[str, Any]]] = load_data(args.file, logger) table_list: List[str] = list(data.keys()) logger.debug(f"Liste der Tabellen: {table_list}") sorted_table_list: List[str] = table_list for tablename in sorted_table_list: model = registry[tablename] existing_items = db.query(model).all() existing_ids: List[str] = get_ids(existing_items) logger.debug(f"found {len(existing_items)} for table {tablename}") import_items: List[Dict[str, Any]] = data[tablename] for import_item in import_items: item_id: str = import_item["id"] if item_id in existing_ids: logger.debug(f"update {item_id}") existing_item = db.get(model, item_id) update_item(db, import_item, existing_item, args.dry_run, logger) existing_ids.remove(item_id) else: logger.debug(f"import {item_id}") item_import(db, import_item, args.dry_run, logger) logger.debug(f"remaining items for {tablename}: {len(existing_ids)}") if len(existing_ids) > 0: logger.info(f"remaining items for {tablename}: {existing_ids}") for item_id in existing_ids: match tablename: case "media_file": item_delete( table_name=tablename, item_id=item_id, api_data=api_data, log=logger, ) case "media_actor_file": item_delete( table_name=tablename, item_id=item_id, api_data=api_data, log=logger, ) case "media_actor": item_delete( table_name=tablename, item_id=item_id, api_data=api_data, log=logger, ) case "profile": item_delete( table_name=tablename, item_id=item_id, api_data=api_data, log=logger ) case _: logger.info("Method to remove remaining item not implemented") logger.info("kontor.import finished")