update cli scripts to use REST API

kontor-scripts/add_links.py

@@ -2,19 +2,17 @@
 read file with links and store it in DB
 """
 from datetime import datetime
-import logging.config
+import logging
 import re
-from typing import Dict, List
+from typing import Dict, List, Optional
 import uuid
 from bs4 import BeautifulSoup
 import requests
-import yaml
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
 from pathlib import Path
-from platformdirs import PlatformDirs
-from pathlib import Path
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker, Session
+from api import Server, get_api_config, get_logger
 from db.models.base import Base
 import os
 
@@ -23,7 +21,8 @@ from db.models.media import MediaActor, MediaActorFile, MediaFile
 parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
 parser.add_argument('--file', '-f', help='file with links', default='~/.sync/media/list.txt')
 parser.add_argument('--video', help='store Url as VideoFile', action="store_true")
-parser.add_argument('--config', '-c', default='kontor-docker')
+parser.add_argument('--config', '-c', default='kontor-api')
+parser.add_argument("--server", "-s")
 parser.add_argument('--verbose', '-v', action='count', default=0)
 parser.add_argument('--limit', '-l', type=int, help='maximum number of links to check')
 parser.add_argument('--dry-run', '-m', help='excute script without storing', action="store_true")
@@ -36,25 +35,6 @@ DB_PORT: int = int(os.getenv("DB_PORT", 5432))
 DB_DBNAME: str = os.getenv("DB_DBNAME", "kontor")
 DATABASE_URL: str = f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_SERVER}:{DB_PORT}/{DB_DBNAME}"
 
-def get_logger(level, config: str):
-    dirs = PlatformDirs(config)
-    logging_config = Path(dirs.user_config_dir, 'logging-config.yaml')
-    with open(logging_config, 'rt') as f:
-        log_config = yaml.safe_load(f.read())
-    logging.config.dictConfig(log_config)
-    logger = logging.getLogger('development')
-    if level is not None:
-        match level:
-            case 0:
-                logger.setLevel(logging.CRITICAL)
-            case 1:
-                logger.setLevel(logging.INFO)
-            case 2:
-                logger.setLevel(logging.DEBUG)
-            case _:
-                logger.setLevel(logging.INFO)
-    return logger
-
 def get_session() -> Session:
     engine = create_engine(DATABASE_URL)
     Base.metadata.create_all(bind=engine, checkfirst=True)
@@ -78,13 +58,21 @@ def load_data(filename: str, log) -> List[str]:
 def get_actors_mapping(actor_list: List[MediaActor]) -> Dict[str, MediaActor]:
     mapping: Dict[str, MediaActor] = {}
     for actor in actor_list:
-        mapping[str(actor.url)] = actor
+        if isinstance(actor, dict):
+            url: str = actor["url"]
+        else:
+            url: str = str(actor.url)
+        mapping[url] = actor
     return mapping
 
 def get_actornames_mapping(actor_list: List[MediaActor]) -> Dict[str, MediaActor]:
     mapping: Dict[str, MediaActor] = {}
     for actor in actor_list:
-        mapping[str(actor.name)] = actor
+        if isinstance(actor, dict):
+            name: str = actor["name"]
+        else:
+            name: str = str(actor.name)
+        mapping[name] = actor
     return mapping
 
 def get_meta_info(media_file: MediaFile, log) -> List[str]:
@@ -132,85 +120,87 @@ def get_actor_name(actor_url: str, log: logging.Logger) -> str | None:
 
 
 if __name__ == '__main__':
-    logger = get_logger(args.verbose, "kontor")
+    logger = get_logger(args.verbose, args.config)
     logger.info('kontor.add_links started')
     if args.limit:
         logger.warning(f"check the first {args.limit} links")
-    session = get_session()
+    apiConfig = get_api_config(logger, args.config)
+    server_list: List[Server] = []
+    server: Optional[Server] = None
+    if args.server:
+        server = apiConfig.get_server(args.server)
+        if not server:
+            server = apiConfig.server[0]
+    else:
+        server = apiConfig.server[0]
     links_index = 1
-    with session as db:
+    links = load_data(args.file, logger)
-        links = load_data(args.file, logger)
+    all_media_files = server.request(logger, table="media_file")
-        for link in links:
+    media_actors: List[MediaActor] = server.request(log=logger, table="media_actor")
-            logger.debug(f"process {link}")
+    for link in links:
-            media_files = db.query(MediaFile).filter(MediaFile.url == link).all()
+        logger.info(f"process {link}")
-            media_actors = db.query(MediaActor).all()
+        media_files = [media_file for media_file in all_media_files if media_file["url"] == link]
-            actor_mapping = get_actors_mapping(media_actors)
+        actor_mapping = get_actors_mapping(media_actors)
-            actorname_mapping = get_actornames_mapping(media_actors)
+        actorname_mapping = get_actornames_mapping(media_actors)
-            if len(media_files) == 0:
+        if len(media_files) == 0:
-                logger.info(f"MediaFile for link {link} not found")
+            logger.info(f"MediaFile for link {link} not found")
-                media_file = MediaFile()
+            media_file = MediaFile()
-                media_file.id = str(uuid.uuid4())
+            media_file.id = str(uuid.uuid4())
-                media_file.created_date = datetime.now()
+            media_file.created_date = datetime.now()
-                media_file.last_modified_date = datetime.now()
+            media_file.last_modified_date = datetime.now()
-                media_file.version = 0
+            media_file.version = 0
-                media_file.url = link
+            media_file.url = link
-                media_file.review = True
+            media_file.review = True
-                media_file.should_download = True
+            media_file.should_download = True
-                media_file.path = None
+            media_file.path = None
-                media_file.cloud_link = None
+            media_file.cloud_link = None
-                media_file.file_name = None
+            media_file.file_name = None
-                actor_urls: List[str] = get_meta_info(media_file, logger)
+            actor_urls: List[str] = get_meta_info(media_file, logger)
-                if not args.dry_run:
+            if not args.dry_run:
-                    db.add(media_file)
+                logger.info("add MediaFile %s", media_file)
-                    db.commit()
+            for actor_url in actor_urls:
-                    db.refresh(media_file)
+                if actor_url in actor_mapping:
-                for actor_url in actor_urls:
+                    media_actor: Optional[MediaActor] = actor_mapping[actor_url]
-                    if actor_url in actor_mapping:
+                    # logger.info(f"create mapping for {repr(media_actor)}")
-                        media_actor: MediaActor = actor_mapping[actor_url]
+                    media_actor_file = MediaActorFile()
-                        # logger.info(f"create mapping for {repr(media_actor)}")
+                    media_actor_file.id = str(uuid.uuid4())
-                        media_actor_file = MediaActorFile()
+                    media_actor_file.created_date = datetime.now()
-                        media_actor_file.id = str(uuid.uuid4())
+                    media_actor_file.last_modified_date = datetime.now()
-                        media_actor_file.created_date = datetime.now()
+                    media_actor_file.version = 0
-                        media_actor_file.last_modified_date = datetime.now()
+                    media_actor_file.media_file_id = media_file.id
-                        media_actor_file.version = 0
+                    media_actor_file.media_actor_id = media_actor.id
-                        media_actor_file.media_file_id = media_file.id
+                    logger.info(f"create mapping with {media_actor_file}")
-                        media_actor_file.media_actor_id = media_actor.id
+                    if not args.dry_run:
-                        logger.info(f"create mapping with {media_actor_file}")
+                        logger.info("add MediaFile Actor mapping %s", media_actor_file)
-                        if not args.dry_run:
+                else:
-                            db.add(media_actor_file)
+                    actor_name = get_actor_name(actor_url, logger)
-                            db.commit()
+                    if actor_name in actorname_mapping:
+                        media_actor = actorname_mapping[actor_name]
                     else:
-                        media_actor: MediaActor = None # type: ignore
+                        media_actor = MediaActor()
-                        actor_name = get_actor_name(actor_url, logger)
+                        media_actor.id = str(uuid.uuid4())
-                        if actor_name in actorname_mapping:
+                        media_actor.created_date = datetime.now()
-                            media_actor = actorname_mapping[actor_name]
+                        media_actor.last_modified_date = datetime.now()
-                        else:
+                        media_actor.version = 0
-                            media_actor = MediaActor()
+                        media_actor.name = get_actor_name(actor_url, logger)
-                            media_actor.id = str(uuid.uuid4())
+                        media_actor.url = actor_url
-                            media_actor.created_date = datetime.now()
+                        logger.info(f"update MediaActor with {repr(media_actor)}")
-                            media_actor.last_modified_date = datetime.now()
-                            media_actor.version = 0
-                            media_actor.name = get_actor_name(actor_url, logger)
-                            media_actor.url = actor_url
-                            logger.info(f"update MediaActor with {repr(media_actor)}")
-                            if not args.dry_run:
-                                db.add(media_actor)
-                                db.commit()
-                        media_actor_file = MediaActorFile()
-                        media_actor_file.id = str(uuid.uuid4())
-                        media_actor_file.created_date = datetime.now()
-                        media_actor_file.last_modified_date = datetime.now()
-                        media_actor_file.version = 0
-                        media_actor_file.media_file_id = media_file.id
-                        media_actor_file.media_actor_id = media_actor.id
-                        logger.info(f"create mapping with {media_actor_file}")
                         if not args.dry_run:
-                            db.add(media_actor_file)
+                            logger.info("Update MediaActor %s", media_actor)
-                            db.commit()
+                    media_actor_file = MediaActorFile()
-            else:
+                    media_actor_file.id = str(uuid.uuid4())
-                for media_file in media_files:
+                    media_actor_file.created_date = datetime.now()
-                    logger.debug(f"MediaFile with {media_file.id} is found")
+                    media_actor_file.last_modified_date = datetime.now()
-            links_index += 1
+                    media_actor_file.version = 0
-            if args.limit and args.limit < links_index:
+                    media_actor_file.media_file_id = media_file.id
-                break
+                    media_actor_file.media_actor_id = media_actor.id
+                    logger.info(f"create mapping with {media_actor_file}")
+                    if not args.dry_run:
+                        logger.info("Add MediaFile Actor mapping")
+        else:
+            for media_file in media_files:
+                logger.info(f"MediaFile with {media_file["id"]} is found")
+        links_index += 1
+        if args.limit and args.limit < links_index:
+            break
     logger.info('kontor.add_link finished')

kontor-scripts/api.py

@@ -51,6 +51,7 @@ MAPPING: Dict[str, str] = {
 class OptionType(Enum):
     PARAM = auto()
     ID = auto()
+    URL = auto()
 
 
 class Option:
@@ -174,6 +175,8 @@ def get_logger(level, config: str):
         match level:
             case 0:
                 logger.setLevel(logging.CRITICAL)
+                logging.getLogger("requests").setLevel(logging.WARNING)
+                logging.getLogger("urllib3").setLevel(logging.WARNING)
             case 1:
                 logging.getLogger("requests").setLevel(logging.WARNING)
                 logging.getLogger("urllib3").setLevel(logging.WARNING)

kontor-scripts/download.py

@@ -128,11 +128,6 @@ if __name__ == "__main__":
     apiConfig = get_api_config(log, args.config)
     server: Server = apiConfig.server[0]
     data = server.request(log=log, table="media_file", param=Option(OptionType.PARAM, "download=true"))
-    # url: str = f"http://{host}:{port}/api/media/files?download=true"
-    # headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
-    # response = requests.get(url, headers=headers)
-    # log.info(f"Status: {response.status_code}")
-    # data = response.json()
     entries_count = len(data)
     log.info(f"data: {entries_count}")
     mediafile_index = 1

kontor-scripts/sync.py

@@ -16,6 +16,7 @@ parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
 parser.add_argument("--verbose", "-v", action="count", default=0)
 parser.add_argument("--config", "-c", default="kontor-api")
 parser.add_argument("--dry-run", "-m", action="store_true")
+parser.add_argument("--to-file", "-f", action="store_true")
 parser.add_argument("--server", "-s")
 parser.add_argument("--cleanup", "-d", action="store_true")
 args = parser.parse_args()
@@ -43,13 +44,14 @@ if __name__ == "__main__":
                 logger.info("%s: %s exported", table, len(data))
             except EndPointNotAvailableException:
                 logger.info("Endpoint not implemented")
-        try:
+        if args.to_file:
-            json_dump = json.dumps(export_data[server.name], indent=4)
+            try:
-            file_name = f"{server.name}-data.json"
+                json_dump = json.dumps(export_data[server.name], indent=4)
-            with open(file_name, "w") as dump_file:
+                file_name = f"{server.name}-data.json"
-                dump_file.write(json_dump)
+                with open(file_name, "w") as dump_file:
-        except TypeError as error:
+                    dump_file.write(json_dump)
-            logger.info(f"{error}")
+            except TypeError as error:
+                logger.info(f"{error}")
     for server in server_list:
         logger.info(f"{server.name}: {len(export_data[server.name])} tables exported")
     if len(server_list) > 1:
@@ -61,5 +63,5 @@ if __name__ == "__main__":
                 if item != item_data:
                     logger.debug("diff: %s\n%s", item, item_data)
                 else:
-                    logger.info("no changes for: %s(%s)", table, item["id"])
+                    logger.debug("no changes for: %s(%s)", table, item["id"])
     logger.info("kontor.sync finished")