diff --git a/kontor-scripts/import.py b/kontor-scripts/import.py index 283d017..9449942 100644 --- a/kontor-scripts/import.py +++ b/kontor-scripts/import.py @@ -1,29 +1,30 @@ """ import data from json file to PostgreSQL """ -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -from datetime import datetime, date -from logging import Logger -from typing import Any, Dict, List -import os import json - +import os +from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser +from datetime import date, datetime +from logging import Logger from pathlib import Path +from typing import Any, Dict, List + import requests -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker, Session -from db.models.base import Base -from db.models import registry from psycopg2.errors import NotNullViolation +from sqlalchemy import create_engine +from sqlalchemy.orm import Session, sessionmaker + from config import get_api_config, get_logger +from db.models import registry +from db.models.base import Base parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) -parser.add_argument('--verbose', '-v', action='count', default=0) +parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("--config", "-c", default="kontor-docker") -parser.add_argument('--dry-run', '-m', action='store_true') -parser.add_argument('--cleanup', '-d', action='store_true') -parser.add_argument('--file', '-f', default='~/data.json') +parser.add_argument("--dry-run", "-m", action="store_true") +parser.add_argument("--cleanup", "-d", action="store_true") +parser.add_argument("--file", "-f", default="~/data.json") args = parser.parse_args() DB_USER: str = os.getenv("DB_USER", "kontor") @@ -31,7 +32,9 @@ DB_PASSWORD: str = os.getenv("DB_PASSWORD", "kontor") DB_SERVER: str = os.getenv("DB_SERVER", "127.0.0.1") DB_PORT: int = int(os.getenv("DB_PORT", 5432)) DB_DBNAME: str = os.getenv("DB_DBNAME", "kontor") -DATABASE_URL: str = f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_SERVER}:{DB_PORT}/{DB_DBNAME}" +DATABASE_URL: str = ( + f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_SERVER}:{DB_PORT}/{DB_DBNAME}" +) def cleanup_database(db: Session, log, dry_run: bool): @@ -46,6 +49,7 @@ def cleanup_database(db: Session, log, dry_run: bool): db.delete(entry) db.commit() + def load_data(filename: str, log) -> Dict[str, List[Dict[str, Any]]]: log.debug("load_data") import_file = Path(filename) @@ -53,45 +57,54 @@ def load_data(filename: str, log) -> Dict[str, List[Dict[str, Any]]]: log.info(f"File {filename} does not exist. Do nothing.") raise FileNotFoundError() log.info("read json file") - with open(filename, 'r') as json_file: + with open(filename, "r") as json_file: json_load = json.load(json_file) return json_load + def get_ids(items: List[Any]) -> List[str]: result: List[str] = [] for item in items: result.append(item.id) return result -def update_item(db: Session, import_data: Dict[str, Any], item: Any, dry_run: bool, log): - for (key, value) in import_data.items(): + +def update_item( + db: Session, import_data: Dict[str, Any], item: Any, dry_run: bool, log +): + for key, value in import_data.items(): existing_value = getattr(item, str(key)) update: bool = has_changed(existing_value, value, log) - #if key == 'published_on': + # if key == 'published_on': # log.info(f"{type(value)}:{value} != {type(existing_value)}:{existing_value} : {update}") if update: log.info(f"update {key}({existing_value}) with {value}") if not dry_run: - if value == 'None': + if value == "None": setattr(item, str(key), None) else: setattr(item, str(key), value) db.add(item) db.commit() + def has_changed(existing_data: Any, import_data: str, log) -> bool: - if existing_data is None and import_data == 'None': + if existing_data is None and import_data == "None": return False if isinstance(existing_data, str): return existing_data != import_data if isinstance(existing_data, date): if len(import_data) > 19: import_date = datetime.strptime(import_data, "%Y-%m-%d %H:%M:%S.%f") - log.debug(f"{type(existing_data)}:{existing_data} == {import_date} : {existing_data != import_date}") + log.debug( + f"{type(existing_data)}:{existing_data} == {import_date} : {existing_data != import_date}" + ) return existing_data != import_date if len(import_data) > 10: import_date = datetime.strptime(import_data, "%Y-%m-%d %H:%M:%S") - log.debug(f"{type(existing_data)}:{existing_data} == {import_date} : {existing_data != import_date}") + log.debug( + f"{type(existing_data)}:{existing_data} == {import_date} : {existing_data != import_date}" + ) return existing_data != import_date return existing_data.strftime("%Y-%m-%d") != import_data return existing_data != import_data @@ -112,11 +125,12 @@ def item_import(db: Session, import_data: Dict[str, Any], dry_run: bool, log): except Exception as error: log.info(f"import failed: {error}") + def item_delete(table_name: str, item_id: str, api_data: Dict[str, Any], log: Logger): log.info(f"delete item {item_id} from {table_name}") host = api_data["host"] port = api_data["port"] - token = api_data['token'] + token = api_data["token"] url = "" match table_name: case "media_file": @@ -125,14 +139,16 @@ def item_delete(table_name: str, item_id: str, api_data: Dict[str, Any], log: Lo url = f"http://{host}:{port}/api/media/actorfiles/{item_id}" case "media_actor": url = f"http://{host}:{port}/api/media/actors/{item_id}" + case "profile": + url = f"http://{host}:{port}/api/user/profile/{item_id}" headers: Dict[str, str] = {"Authorization": f"Bearer {token}"} response = requests.delete(url, headers=headers) log.debug(f"Status: {response.status_code}") -if __name__ == '__main__': +if __name__ == "__main__": logger = get_logger(args.verbose, "kontor") - logger.info('kontor.import started') + logger.info("kontor.import started") api_data = get_api_config(logger, args.config) engine = create_engine(DATABASE_URL) Base.metadata.create_all(bind=engine, checkfirst=True) @@ -151,7 +167,7 @@ if __name__ == '__main__': logger.debug(f"found {len(existing_items)} for table {tablename}") import_items: List[Dict[str, Any]] = data[tablename] for import_item in import_items: - item_id: str = import_item['id'] + item_id: str = import_item["id"] if item_id in existing_ids: logger.debug(f"update {item_id}") existing_item = db.get(model, item_id) @@ -166,11 +182,26 @@ if __name__ == '__main__': for item_id in existing_ids: match tablename: case "media_file": - item_delete(table_name=tablename, item_id=item_id, api_data=api_data, log=logger) + item_delete( + table_name=tablename, + item_id=item_id, + api_data=api_data, + log=logger, + ) case "media_actor_file": - item_delete(table_name=tablename, item_id=item_id, api_data=api_data, log=logger) + item_delete( + table_name=tablename, + item_id=item_id, + api_data=api_data, + log=logger, + ) case "media_actor": - item_delete(table_name=tablename, item_id=item_id, api_data=api_data, log=logger) + item_delete( + table_name=tablename, + item_id=item_id, + api_data=api_data, + log=logger, + ) case _: logger.info("Method to remove remaining item not implemented") - logger.info('kontor.import finished') + logger.info("kontor.import finished")