kontor-scripts/config.py

@@ -1,11 +1,14 @@
 """
 Setup database connections
 """
+from logging import Logger
 import sqlite3
+from typing import Any, Dict
 import psycopg2
 import logging.config
 from platformdirs import PlatformDirs
 from pathlib import Path
+import requests
 import yaml
 
 
@@ -54,3 +57,37 @@ def get_logger(level, config: str):
                 logger.setLevel(logging.INFO)
     return logger
 
+def get_api_config(log: Logger, config: str) -> Dict[str, Any]:
+    api_data: Dict[str, Any] = {}
+    token: str | None = None
+    host: str | None = None
+    port: int = 0
+    dirs = PlatformDirs(config)
+    api_config = Path(dirs.user_config_dir, 'api.yaml')
+    with open(api_config, 'rt') as f:
+        api_data = yaml.safe_load(f.read())
+        if not api_data:
+            log.fatal("API configuration is missing")
+            return api_data
+    host = api_data["host"]
+    port = api_data["login_port"]
+    if not token:
+        log.info("Call login first")
+        login_url = f"http://{host}:{port}/login"
+        login_data = {}
+        login_data['user'] = api_data["user"]
+        login_data['pass'] = api_data["pass"]
+        response = requests.post(login_url, data=login_data)
+        status = response.status_code
+        log.info(f"Status: {status}")
+        if status != 200:
+            log.fatal("authentication failed")
+            return api_data
+        data = response.json()
+        token = data['token']
+        api_data['token'] = token
+        with open(api_config, 'w') as f:
+            yaml.dump(api_data, f)
+    else:
+        token = api_data['token']
+    return api_data

kontor-scripts/download.py

@@ -17,7 +17,7 @@ from platformdirs import PlatformDirs
 import requests
 import yaml
 
-from config import get_logger
+from config import get_api_config, get_logger
 
 parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
 parser.add_argument("--verbose", "-v", action="count", default=0)
@@ -106,9 +106,11 @@ def is_file_downloaded(media_file: dict, dir: Path) -> FileStatus:
     return FileStatus.UNKNOWN
 
 
-def update_status(item_id: UUID, file_info: dict):
+def update_status(item_id: UUID, file_info: dict, api_data: Dict[str, Any]):
-    url: str = f"http://127.0.0.1:8800/api/media/files/{item_id}"
+    host = api_data["host"]
-    headers: Dict[str, str] = {"Authorization": f"Bearer {bearer_token}"}
+    token = api_data['token']
+    url: str = f"http://{host}:{port}/api/media/files/{item_id}"
+    headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
     update = requests.put(url, headers=headers, json=file_info)
     log.info(f"update status: {update.status_code}")
     log.info(f"update result: {update.json()}")
@@ -132,39 +134,11 @@ def rename_file(file_info: dict):
 if __name__ == "__main__":
     log = get_logger(args.verbose, args.config)
     log.info("kontor.download started")
-    api_data: Dict[str, Any] = {}
+    api_data = get_api_config(log, args.config)
-    token: str | None = None
-    host: str | None = None
-    port: int = 0
-    dirs = PlatformDirs(args.config)
-    api_config = Path(dirs.user_config_dir, 'api.yaml')
-    with open(api_config, 'rt') as f:
-        api_data = yaml.safe_load(f.read())
-        if not api_data:
-            log.fatal("API configuration is missing")
-            sys.exit(1)
     host = api_data["host"]
     port = api_data["port"]
-    if not token:
+    token = api_data['token']
-        log.info("Call login first")
+    url: str = f"http://{host}:{port}/api/media/files?download=true"
-        login_url = f"http://{host}:{port}/login"
-        login_data = {}
-        login_data['user'] = api_data["user"]
-        login_data['pass'] = api_data["pass"]
-        response = requests.post(login_url, data=login_data)
-        status = response.status_code
-        log.info(f"Status: {status}")
-        if status != 200:
-            log.fatal("authentication failed")
-            sys.exit(0)
-        data = response.json()
-        token = data['token']
-        api_data['token'] = token
-        with open(api_config, 'w') as f:
-            yaml.dump(api_data, f)
-    else:
-        token = api_data['token']
-    url: str = f"http://{host}:8500/api/media/files?download=true"
     headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
     response = requests.get(url, headers=headers)
     log.info(f"Status: {response.status_code}")
@@ -186,15 +160,15 @@ if __name__ == "__main__":
         match download_status:
             case FileStatus.DOWNLOADED:
                 rename_file(item)
-                update_status(file_id, item)
+                update_status(file_id, item, api_data)
             case FileStatus.RENAMED:
                 log.info("update status")
-                update_status(file_id, item)
+                update_status(file_id, item, api_data)
             case FileStatus.UNKNOWN:
                 download_file(link, item, args.dir)
                 rename_file(item)
                 log.info(f"{item}")
-                update_status(file_id, item)
+                update_status(file_id, item, api_data)
         log.warning(f"processed {mediafile_index}/{entries_count}")
         if args.limit and args.limit <= mediafile_index:
             break

kontor-scripts/find_links.py

@@ -3,15 +3,18 @@ download files with URLs from DB
 """
 import logging.config
 import sys
-from typing import Any
+from typing import Any, Dict
 import requests
 import re
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
 
 from bs4 import BeautifulSoup
 
+from config import get_api_config
+
 parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
 parser.add_argument('--verbose', '-v', action='count', default=0)
+parser.add_argument("--config", "-c", default="kontor-docker")
 parser.add_argument('--all', '-a', action='store_true')
 parser.add_argument('--limit', '-l', type=int, help='maximum number of links to check')
 parser.add_argument('--add-actor', action='store_true', help='add missing actors')
@@ -60,12 +63,17 @@ def get_logger(level: int) -> logging.Logger:
                 logger.setLevel(logging.CRITICAL)
     return logger
 
-def update_file(log: logging.Logger, media_file):
+def update_file(log: logging.Logger, media_file, api_data: Dict[str, Any]):
-    update = requests.put(f"http://127.0.0.1:8800/api/media/files/{media_file['id']}", json=media_file)
+    host = api_data["host"]
+    port = api_data["port"]
+    token = api_data['token']
+    headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
+    url = f"http://{host}:{port}/api/media/files/{media_file['id']}"
+    update = requests.put(url, headers=headers, json=media_file)
     log.debug(f"update status: {update.status_code}")
     log.debug(f"update result: {update.json()}")
 
-def get_actor_links(log: logging.Logger, media_file_url: str) -> list[str]:
+def get_actor_links(log: logging.Logger, media_file_url: str, api_data: Dict[str, Any]) -> list[str]:
     try:
         r = requests.get(media_file_url)
         soup = BeautifulSoup(r.content, "html.parser")
@@ -74,7 +82,7 @@ def get_actor_links(log: logging.Logger, media_file_url: str) -> list[str]:
             log.warning(f"{error404.get_text()}")
             media_file['url'] = None
             media_file['review'] = False
-            update_file(log, media_file)
+            update_file(log, media_file, api_data=api_data)
             return []
         anchors = soup.find_all('a', attrs={'href': re.compile("^https://.*pornstars/.*")})
         actor_links = []
@@ -91,19 +99,28 @@ def get_actor_links(log: logging.Logger, media_file_url: str) -> list[str]:
         log.warning(f"something went wrong: {error}")
         return []
 
-def get_media_files(all_files: bool)-> Any:
+def get_media_files(all_files: bool, api_data: Dict[str, Any])-> Any:
     files_url = ""
+    host = api_data["host"]
+    port = api_data["port"]
+    token = api_data['token']
+    headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
     if all_files:
-        files_url= "http://127.0.0.1:8800/api/media/files"
+        files_url= f"http://{host}:{port}/api/media/files"
     else:
-        files_url = "http://127.0.0.1:8800/api/media/files?review=true"
+        files_url = f"http://{host}:{port}/api/media/files?review=true"
-    response = requests.get(files_url)
+    response = requests.get(files_url, headers=headers)
     log.debug(f"Status: {response.status_code}")
     data = response.json()
     return data
 
-def update_media_file(item, log: logging.Logger) -> Any:
+def update_media_file(item, log: logging.Logger, api_data: Dict[str, Any]) -> Any:
-    update = requests.put(f"http://127.0.0.1:8800/api/media/files/{item['id']}", json=item)
+    host = api_data["host"]
+    port = api_data["port"]
+    token = api_data['token']
+    url: str = f"http://{host}:{port}/api/media/files/{item['id']}"
+    headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
+    update = requests.put(url, headers=headers, json=item)
     log.debug(f"update status: {update.status_code}")
     log.debug(f"update result: {update.json()}")
     return update.json()
@@ -112,9 +129,15 @@ def update_media_file_actors(mediafile: dict,
                              actor_id_list: list[dict[str, str]], 
                              actor_links: list[str], 
                              map_ids_actor: dict[str, str], 
-                             log: logging.Logger):
+                             log: logging.Logger, 
+                             api_data: Dict[str, Any]):
         media_file_id = mediafile['id']
-        actor_response = requests.put(f"http://127.0.0.1:8800/api/media/files/{media_file_id}/actors", json=actor_id_list)
+        host = api_data["host"]
+        port = api_data["port"]
+        token = api_data['token']
+        url: str = f"http://{host}:{port}/api/media/files/{media_file_id}/actors"
+        headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
+        actor_response = requests.put(url, headers=headers, json=actor_id_list)
         files_actor_list = actor_response.json()
         persisted_actor_links_count: int = len(files_actor_list)
         found_actor_links_count: int = len(actor_links)
@@ -129,15 +152,20 @@ def update_media_file_actors(mediafile: dict,
                 log.debug(f"check if actor({actor_id}) with {actor_url} in list")
                 if actor_url not in actor_links:
                     log.info(f"actor not found in links, delete relation {file_actor['id']}")
-                    delete_media_file_actor(file_actor['id'], log)
+                    delete_media_file_actor(file_actor['id'], log, api_data)
             mediafile['review'] = True
         else:
             mediafile['review'] = False
         log.debug(f"found {persisted_actor_links_count} actors")
         log.debug(f"found actors: {files_actor_list}")
 
-def delete_media_file_actor(media_actor_file_id: str, log: logging.Logger):
+def delete_media_file_actor(media_actor_file_id: str, log: logging.Logger, api_data: Dict[str, Any]):
-    delete_response = requests.delete(f"http://127.0.0.1:8800/api/media/actorfiles/{media_actor_file_id}")
+    host = api_data["host"]
+    port = api_data["port"]
+    token = api_data['token']
+    url: str = f"http://{host}:{port}/api/media/actorfiles/{media_actor_file_id}"
+    headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
+    delete_response = requests.delete(url, headers=headers)
     if delete_response.status_code == 204:
         log.info(f"actor file relation with id {media_actor_file_id} successfully deleted")
 
@@ -205,11 +233,16 @@ def get_persisted_actor(actor_url: str,
     log.info(f"found actor {actor_url} missing")
     return None
 
-def get_actors(log: logging.Logger):
+def get_actors(log: logging.Logger, api_data: Dict[str, Any]):
     actors_url = {}
     actors_id = {}
     actors_path = {}
-    response = requests.get("http://127.0.0.1:8800/api/media/actors")
+    host = api_data["host"]
+    port = api_data["port"]
+    token = api_data['token']
+    url: str = f"http://{host}:{port}/api/media/actors"
+    headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
+    response = requests.get(url, headers=headers)
     data = response.json()
     for media_actor in data:
         actor_id = media_actor['id']
@@ -238,9 +271,14 @@ def get_actor_name(actor_url: str, log: logging.Logger) -> str | None:
         log.warning(f"something went wrong: {error}")
         return None
 
-def create_actor(actor_url: str, actor_name: str, log: logging.Logger):
+def create_actor(actor_url: str, actor_name: str, log: logging.Logger, api_data: Dict[str, Any]):
     new_actor = { 'name': actor_name, 'url': actor_url}
-    actor_response = requests.post(f"http://127.0.0.1:8800/api/media/actors", json=new_actor)
+    host = api_data["host"]
+    port = api_data["port"]
+    token = api_data['token']
+    url: str = f"http://{host}:{port}/api/media/actors"
+    headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
+    actor_response = requests.post(url, headers=headers, json=new_actor)
     log.warning(f"add status: {actor_response.status_code}")
     if actor_response.status_code == 201:
         actor_data = actor_response.json()
@@ -253,8 +291,13 @@ if __name__ == '__main__':
     log = get_logger(args.verbose)
     log.warning('kontor.find_links started')
     log.debug('get all actors')
-    (actors_url, actors_id, actors_path)  = get_actors(log)
+    api_data = get_api_config(log, args.config)
-    data = get_media_files(args.all)
+    host = api_data["host"]
+    port = api_data["port"]
+    token = api_data['token']
+
+    (actors_url, actors_id, actors_path)  = get_actors(log, api_data=api_data)
+    data = get_media_files(args.all, api_data)
     entries_count = len(data)
     mediafile_index = 1
     log.debug(f"data: {len(data)}")
@@ -269,10 +312,10 @@ if __name__ == '__main__':
         if str(link) == "None":
             continue
         log.warning(f"{media_file['id']} - {str(link)}")
-        actor_links: list[str] = get_actor_links(log, link)
+        actor_links: list[str] = get_actor_links(log, link, api_data=api_data)
         actor_id_list = get_actor_ids(actor_links, actors_url, actors_id, actors_path, missing_actors, log)
-        update_media_file_actors(media_file, actor_id_list, actor_links, actors_id, log)
+        update_media_file_actors(media_file, actor_id_list, actor_links, actors_id, log, api_data=api_data)
-        result = update_media_file(media_file, log)
+        result = update_media_file(media_file, log, api_data=api_data)
         log.warning(f"processed {mediafile_index}/{entries_count}")
         if args.limit and args.limit <= mediafile_index:
             break
@@ -281,7 +324,7 @@ if __name__ == '__main__':
         log.info(f"{link}: {missing_actors[link]}")
         actor_name = get_actor_name(link, log)
         if actor_name and args.add_actor:
-            create_actor(link, actor_name, log)
+            create_actor(link, actor_name, log, api_data=api_data)
     log.info("Sort missing actors by occurence count:")
     sorted_missing = dict(sorted(missing_actors.items(), key=lambda item: item[1]))
     for key in sorted_missing: