update cli scripts to use REST API

kontor-scripts/add_links.py

@@ -2,19 +2,17 @@
 read file with links and store it in DB
 """
 from datetime import datetime
-import logging.config
+import logging
 import re
-from typing import Dict, List
+from typing import Dict, List, Optional
 import uuid
 from bs4 import BeautifulSoup
 import requests
-import yaml
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
 from pathlib import Path
-from platformdirs import PlatformDirs
-from pathlib import Path
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker, Session
+from api import Server, get_api_config, get_logger
 from db.models.base import Base
 import os
 
@@ -23,7 +21,8 @@ from db.models.media import MediaActor, MediaActorFile, MediaFile
 parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
 parser.add_argument('--file', '-f', help='file with links', default='~/.sync/media/list.txt')
 parser.add_argument('--video', help='store Url as VideoFile', action="store_true")
-parser.add_argument('--config', '-c', default='kontor-docker')
+parser.add_argument('--config', '-c', default='kontor-api')
+parser.add_argument("--server", "-s")
 parser.add_argument('--verbose', '-v', action='count', default=0)
 parser.add_argument('--limit', '-l', type=int, help='maximum number of links to check')
 parser.add_argument('--dry-run', '-m', help='excute script without storing', action="store_true")
@@ -36,25 +35,6 @@ DB_PORT: int = int(os.getenv("DB_PORT", 5432))
 DB_DBNAME: str = os.getenv("DB_DBNAME", "kontor")
 DATABASE_URL: str = f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_SERVER}:{DB_PORT}/{DB_DBNAME}"
 
-def get_logger(level, config: str):
-    dirs = PlatformDirs(config)
-    logging_config = Path(dirs.user_config_dir, 'logging-config.yaml')
-    with open(logging_config, 'rt') as f:
-        log_config = yaml.safe_load(f.read())
-    logging.config.dictConfig(log_config)
-    logger = logging.getLogger('development')
-    if level is not None:
-        match level:
-            case 0:
-                logger.setLevel(logging.CRITICAL)
-            case 1:
-                logger.setLevel(logging.INFO)
-            case 2:
-                logger.setLevel(logging.DEBUG)
-            case _:
-                logger.setLevel(logging.INFO)
-    return logger
-
 def get_session() -> Session:
     engine = create_engine(DATABASE_URL)
     Base.metadata.create_all(bind=engine, checkfirst=True)
@@ -78,13 +58,21 @@ def load_data(filename: str, log) -> List[str]:
 def get_actors_mapping(actor_list: List[MediaActor]) -> Dict[str, MediaActor]:
     mapping: Dict[str, MediaActor] = {}
     for actor in actor_list:
-        mapping[str(actor.url)] = actor
+        if isinstance(actor, dict):
+            url: str = actor["url"]
+        else:
+            url: str = str(actor.url)
+        mapping[url] = actor
     return mapping
 
 def get_actornames_mapping(actor_list: List[MediaActor]) -> Dict[str, MediaActor]:
     mapping: Dict[str, MediaActor] = {}
     for actor in actor_list:
-        mapping[str(actor.name)] = actor
+        if isinstance(actor, dict):
+            name: str = actor["name"]
+        else:
+            name: str = str(actor.name)
+        mapping[name] = actor
     return mapping
 
 def get_meta_info(media_file: MediaFile, log) -> List[str]:
@@ -132,18 +120,26 @@ def get_actor_name(actor_url: str, log: logging.Logger) -> str | None:
 
 
 if __name__ == '__main__':
-    logger = get_logger(args.verbose, "kontor")
+    logger = get_logger(args.verbose, args.config)
     logger.info('kontor.add_links started')
     if args.limit:
         logger.warning(f"check the first {args.limit} links")
-    session = get_session()
+    apiConfig = get_api_config(logger, args.config)
+    server_list: List[Server] = []
+    server: Optional[Server] = None
+    if args.server:
+        server = apiConfig.get_server(args.server)
+        if not server:
+            server = apiConfig.server[0]
+    else:
+        server = apiConfig.server[0]
     links_index = 1
-    with session as db:
     links = load_data(args.file, logger)
+    all_media_files = server.request(logger, table="media_file")
+    media_actors: List[MediaActor] = server.request(log=logger, table="media_actor")
     for link in links:
-            logger.debug(f"process {link}")
+        logger.info(f"process {link}")
-            media_files = db.query(MediaFile).filter(MediaFile.url == link).all()
+        media_files = [media_file for media_file in all_media_files if media_file["url"] == link]
-            media_actors = db.query(MediaActor).all()
         actor_mapping = get_actors_mapping(media_actors)
         actorname_mapping = get_actornames_mapping(media_actors)
         if len(media_files) == 0:
@@ -161,12 +157,10 @@ if __name__ == '__main__':
             media_file.file_name = None
             actor_urls: List[str] = get_meta_info(media_file, logger)
             if not args.dry_run:
-                    db.add(media_file)
+                logger.info("add MediaFile %s", media_file)
-                    db.commit()
-                    db.refresh(media_file)
             for actor_url in actor_urls:
                 if actor_url in actor_mapping:
-                        media_actor: MediaActor = actor_mapping[actor_url]
+                    media_actor: Optional[MediaActor] = actor_mapping[actor_url]
                     # logger.info(f"create mapping for {repr(media_actor)}")
                     media_actor_file = MediaActorFile()
                     media_actor_file.id = str(uuid.uuid4())
@@ -177,10 +171,8 @@ if __name__ == '__main__':
                     media_actor_file.media_actor_id = media_actor.id
                     logger.info(f"create mapping with {media_actor_file}")
                     if not args.dry_run:
-                            db.add(media_actor_file)
+                        logger.info("add MediaFile Actor mapping %s", media_actor_file)
-                            db.commit()
                 else:
-                        media_actor: MediaActor = None # type: ignore
                     actor_name = get_actor_name(actor_url, logger)
                     if actor_name in actorname_mapping:
                         media_actor = actorname_mapping[actor_name]
@@ -194,8 +186,7 @@ if __name__ == '__main__':
                         media_actor.url = actor_url
                         logger.info(f"update MediaActor with {repr(media_actor)}")
                         if not args.dry_run:
-                                db.add(media_actor)
+                            logger.info("Update MediaActor %s", media_actor)
-                                db.commit()
                     media_actor_file = MediaActorFile()
                     media_actor_file.id = str(uuid.uuid4())
                     media_actor_file.created_date = datetime.now()
@@ -205,11 +196,10 @@ if __name__ == '__main__':
                     media_actor_file.media_actor_id = media_actor.id
                     logger.info(f"create mapping with {media_actor_file}")
                     if not args.dry_run:
-                            db.add(media_actor_file)
+                        logger.info("Add MediaFile Actor mapping")
-                            db.commit()
         else:
             for media_file in media_files:
-                    logger.debug(f"MediaFile with {media_file.id} is found")
+                logger.info(f"MediaFile with {media_file["id"]} is found")
         links_index += 1
         if args.limit and args.limit < links_index:
             break

kontor-scripts/api.py

@@ -51,6 +51,7 @@ MAPPING: Dict[str, str] = {
 class OptionType(Enum):
     PARAM = auto()
     ID = auto()
+    URL = auto()
 
 
 class Option:
@@ -174,6 +175,8 @@ def get_logger(level, config: str):
         match level:
             case 0:
                 logger.setLevel(logging.CRITICAL)
+                logging.getLogger("requests").setLevel(logging.WARNING)
+                logging.getLogger("urllib3").setLevel(logging.WARNING)
             case 1:
                 logging.getLogger("requests").setLevel(logging.WARNING)
                 logging.getLogger("urllib3").setLevel(logging.WARNING)

kontor-scripts/sync.py

@@ -16,6 +16,7 @@ parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
 parser.add_argument("--verbose", "-v", action="count", default=0)
 parser.add_argument("--config", "-c", default="kontor-api")
 parser.add_argument("--dry-run", "-m", action="store_true")
+parser.add_argument("--to-file", "-f", action="store_true")
 parser.add_argument("--server", "-s")
 parser.add_argument("--cleanup", "-d", action="store_true")
 args = parser.parse_args()
@@ -43,6 +44,7 @@ if __name__ == "__main__":
                 logger.info("%s: %s exported", table, len(data))
             except EndPointNotAvailableException:
                 logger.info("Endpoint not implemented")
+        if args.to_file:
             try:
                 json_dump = json.dumps(export_data[server.name], indent=4)
                 file_name = f"{server.name}-data.json"
@@ -61,5 +63,5 @@ if __name__ == "__main__":
                 if item != item_data:
                     logger.debug("diff: %s\n%s", item, item_data)
                 else:
-                    logger.info("no changes for: %s(%s)", table, item["id"])
+                    logger.debug("no changes for: %s(%s)", table, item["id"])
     logger.info("kontor.sync finished")