From f9d09d5357e6ca17201a3c6912628d02e3ce341a Mon Sep 17 00:00:00 2001 From: Thomas Peetz Date: Thu, 21 May 2026 16:51:38 +0200 Subject: [PATCH] update cli scripts to use REST API --- kontor-scripts/add_links.py | 194 +++++++++++++++++------------------- kontor-scripts/api.py | 3 + kontor-scripts/sync.py | 18 ++-- 3 files changed, 105 insertions(+), 110 deletions(-) diff --git a/kontor-scripts/add_links.py b/kontor-scripts/add_links.py index 43a906e..2b8bbbd 100644 --- a/kontor-scripts/add_links.py +++ b/kontor-scripts/add_links.py @@ -2,19 +2,17 @@ read file with links and store it in DB """ from datetime import datetime -import logging.config +import logging import re -from typing import Dict, List +from typing import Dict, List, Optional import uuid from bs4 import BeautifulSoup import requests -import yaml from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter from pathlib import Path -from platformdirs import PlatformDirs -from pathlib import Path from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker, Session +from api import Server, get_api_config, get_logger from db.models.base import Base import os @@ -23,7 +21,8 @@ from db.models.media import MediaActor, MediaActorFile, MediaFile parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument('--file', '-f', help='file with links', default='~/.sync/media/list.txt') parser.add_argument('--video', help='store Url as VideoFile', action="store_true") -parser.add_argument('--config', '-c', default='kontor-docker') +parser.add_argument('--config', '-c', default='kontor-api') +parser.add_argument("--server", "-s") parser.add_argument('--verbose', '-v', action='count', default=0) parser.add_argument('--limit', '-l', type=int, help='maximum number of links to check') parser.add_argument('--dry-run', '-m', help='excute script without storing', action="store_true") @@ -36,25 +35,6 @@ DB_PORT: int = int(os.getenv("DB_PORT", 5432)) DB_DBNAME: str = os.getenv("DB_DBNAME", "kontor") DATABASE_URL: str = f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_SERVER}:{DB_PORT}/{DB_DBNAME}" -def get_logger(level, config: str): - dirs = PlatformDirs(config) - logging_config = Path(dirs.user_config_dir, 'logging-config.yaml') - with open(logging_config, 'rt') as f: - log_config = yaml.safe_load(f.read()) - logging.config.dictConfig(log_config) - logger = logging.getLogger('development') - if level is not None: - match level: - case 0: - logger.setLevel(logging.CRITICAL) - case 1: - logger.setLevel(logging.INFO) - case 2: - logger.setLevel(logging.DEBUG) - case _: - logger.setLevel(logging.INFO) - return logger - def get_session() -> Session: engine = create_engine(DATABASE_URL) Base.metadata.create_all(bind=engine, checkfirst=True) @@ -78,13 +58,21 @@ def load_data(filename: str, log) -> List[str]: def get_actors_mapping(actor_list: List[MediaActor]) -> Dict[str, MediaActor]: mapping: Dict[str, MediaActor] = {} for actor in actor_list: - mapping[str(actor.url)] = actor + if isinstance(actor, dict): + url: str = actor["url"] + else: + url: str = str(actor.url) + mapping[url] = actor return mapping def get_actornames_mapping(actor_list: List[MediaActor]) -> Dict[str, MediaActor]: mapping: Dict[str, MediaActor] = {} for actor in actor_list: - mapping[str(actor.name)] = actor + if isinstance(actor, dict): + name: str = actor["name"] + else: + name: str = str(actor.name) + mapping[name] = actor return mapping def get_meta_info(media_file: MediaFile, log) -> List[str]: @@ -132,85 +120,87 @@ def get_actor_name(actor_url: str, log: logging.Logger) -> str | None: if __name__ == '__main__': - logger = get_logger(args.verbose, "kontor") + logger = get_logger(args.verbose, args.config) logger.info('kontor.add_links started') if args.limit: logger.warning(f"check the first {args.limit} links") - session = get_session() + apiConfig = get_api_config(logger, args.config) + server_list: List[Server] = [] + server: Optional[Server] = None + if args.server: + server = apiConfig.get_server(args.server) + if not server: + server = apiConfig.server[0] + else: + server = apiConfig.server[0] links_index = 1 - with session as db: - links = load_data(args.file, logger) - for link in links: - logger.debug(f"process {link}") - media_files = db.query(MediaFile).filter(MediaFile.url == link).all() - media_actors = db.query(MediaActor).all() - actor_mapping = get_actors_mapping(media_actors) - actorname_mapping = get_actornames_mapping(media_actors) - if len(media_files) == 0: - logger.info(f"MediaFile for link {link} not found") - media_file = MediaFile() - media_file.id = str(uuid.uuid4()) - media_file.created_date = datetime.now() - media_file.last_modified_date = datetime.now() - media_file.version = 0 - media_file.url = link - media_file.review = True - media_file.should_download = True - media_file.path = None - media_file.cloud_link = None - media_file.file_name = None - actor_urls: List[str] = get_meta_info(media_file, logger) - if not args.dry_run: - db.add(media_file) - db.commit() - db.refresh(media_file) - for actor_url in actor_urls: - if actor_url in actor_mapping: - media_actor: MediaActor = actor_mapping[actor_url] - # logger.info(f"create mapping for {repr(media_actor)}") - media_actor_file = MediaActorFile() - media_actor_file.id = str(uuid.uuid4()) - media_actor_file.created_date = datetime.now() - media_actor_file.last_modified_date = datetime.now() - media_actor_file.version = 0 - media_actor_file.media_file_id = media_file.id - media_actor_file.media_actor_id = media_actor.id - logger.info(f"create mapping with {media_actor_file}") - if not args.dry_run: - db.add(media_actor_file) - db.commit() + links = load_data(args.file, logger) + all_media_files = server.request(logger, table="media_file") + media_actors: List[MediaActor] = server.request(log=logger, table="media_actor") + for link in links: + logger.info(f"process {link}") + media_files = [media_file for media_file in all_media_files if media_file["url"] == link] + actor_mapping = get_actors_mapping(media_actors) + actorname_mapping = get_actornames_mapping(media_actors) + if len(media_files) == 0: + logger.info(f"MediaFile for link {link} not found") + media_file = MediaFile() + media_file.id = str(uuid.uuid4()) + media_file.created_date = datetime.now() + media_file.last_modified_date = datetime.now() + media_file.version = 0 + media_file.url = link + media_file.review = True + media_file.should_download = True + media_file.path = None + media_file.cloud_link = None + media_file.file_name = None + actor_urls: List[str] = get_meta_info(media_file, logger) + if not args.dry_run: + logger.info("add MediaFile %s", media_file) + for actor_url in actor_urls: + if actor_url in actor_mapping: + media_actor: Optional[MediaActor] = actor_mapping[actor_url] + # logger.info(f"create mapping for {repr(media_actor)}") + media_actor_file = MediaActorFile() + media_actor_file.id = str(uuid.uuid4()) + media_actor_file.created_date = datetime.now() + media_actor_file.last_modified_date = datetime.now() + media_actor_file.version = 0 + media_actor_file.media_file_id = media_file.id + media_actor_file.media_actor_id = media_actor.id + logger.info(f"create mapping with {media_actor_file}") + if not args.dry_run: + logger.info("add MediaFile Actor mapping %s", media_actor_file) + else: + actor_name = get_actor_name(actor_url, logger) + if actor_name in actorname_mapping: + media_actor = actorname_mapping[actor_name] else: - media_actor: MediaActor = None # type: ignore - actor_name = get_actor_name(actor_url, logger) - if actor_name in actorname_mapping: - media_actor = actorname_mapping[actor_name] - else: - media_actor = MediaActor() - media_actor.id = str(uuid.uuid4()) - media_actor.created_date = datetime.now() - media_actor.last_modified_date = datetime.now() - media_actor.version = 0 - media_actor.name = get_actor_name(actor_url, logger) - media_actor.url = actor_url - logger.info(f"update MediaActor with {repr(media_actor)}") - if not args.dry_run: - db.add(media_actor) - db.commit() - media_actor_file = MediaActorFile() - media_actor_file.id = str(uuid.uuid4()) - media_actor_file.created_date = datetime.now() - media_actor_file.last_modified_date = datetime.now() - media_actor_file.version = 0 - media_actor_file.media_file_id = media_file.id - media_actor_file.media_actor_id = media_actor.id - logger.info(f"create mapping with {media_actor_file}") + media_actor = MediaActor() + media_actor.id = str(uuid.uuid4()) + media_actor.created_date = datetime.now() + media_actor.last_modified_date = datetime.now() + media_actor.version = 0 + media_actor.name = get_actor_name(actor_url, logger) + media_actor.url = actor_url + logger.info(f"update MediaActor with {repr(media_actor)}") if not args.dry_run: - db.add(media_actor_file) - db.commit() - else: - for media_file in media_files: - logger.debug(f"MediaFile with {media_file.id} is found") - links_index += 1 - if args.limit and args.limit < links_index: - break + logger.info("Update MediaActor %s", media_actor) + media_actor_file = MediaActorFile() + media_actor_file.id = str(uuid.uuid4()) + media_actor_file.created_date = datetime.now() + media_actor_file.last_modified_date = datetime.now() + media_actor_file.version = 0 + media_actor_file.media_file_id = media_file.id + media_actor_file.media_actor_id = media_actor.id + logger.info(f"create mapping with {media_actor_file}") + if not args.dry_run: + logger.info("Add MediaFile Actor mapping") + else: + for media_file in media_files: + logger.info(f"MediaFile with {media_file["id"]} is found") + links_index += 1 + if args.limit and args.limit < links_index: + break logger.info('kontor.add_link finished') diff --git a/kontor-scripts/api.py b/kontor-scripts/api.py index 4d27742..b3d5305 100644 --- a/kontor-scripts/api.py +++ b/kontor-scripts/api.py @@ -51,6 +51,7 @@ MAPPING: Dict[str, str] = { class OptionType(Enum): PARAM = auto() ID = auto() + URL = auto() class Option: @@ -174,6 +175,8 @@ def get_logger(level, config: str): match level: case 0: logger.setLevel(logging.CRITICAL) + logging.getLogger("requests").setLevel(logging.WARNING) + logging.getLogger("urllib3").setLevel(logging.WARNING) case 1: logging.getLogger("requests").setLevel(logging.WARNING) logging.getLogger("urllib3").setLevel(logging.WARNING) diff --git a/kontor-scripts/sync.py b/kontor-scripts/sync.py index 8ca56f2..3f14264 100644 --- a/kontor-scripts/sync.py +++ b/kontor-scripts/sync.py @@ -16,6 +16,7 @@ parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("--config", "-c", default="kontor-api") parser.add_argument("--dry-run", "-m", action="store_true") +parser.add_argument("--to-file", "-f", action="store_true") parser.add_argument("--server", "-s") parser.add_argument("--cleanup", "-d", action="store_true") args = parser.parse_args() @@ -43,13 +44,14 @@ if __name__ == "__main__": logger.info("%s: %s exported", table, len(data)) except EndPointNotAvailableException: logger.info("Endpoint not implemented") - try: - json_dump = json.dumps(export_data[server.name], indent=4) - file_name = f"{server.name}-data.json" - with open(file_name, "w") as dump_file: - dump_file.write(json_dump) - except TypeError as error: - logger.info(f"{error}") + if args.to_file: + try: + json_dump = json.dumps(export_data[server.name], indent=4) + file_name = f"{server.name}-data.json" + with open(file_name, "w") as dump_file: + dump_file.write(json_dump) + except TypeError as error: + logger.info(f"{error}") for server in server_list: logger.info(f"{server.name}: {len(export_data[server.name])} tables exported") if len(server_list) > 1: @@ -61,5 +63,5 @@ if __name__ == "__main__": if item != item_data: logger.debug("diff: %s\n%s", item, item_data) else: - logger.info("no changes for: %s(%s)", table, item["id"]) + logger.debug("no changes for: %s(%s)", table, item["id"]) logger.info("kontor.sync finished")