diff --git a/kontor-scripts/config.py b/kontor-scripts/config.py index 80a268b..c5d05a1 100644 --- a/kontor-scripts/config.py +++ b/kontor-scripts/config.py @@ -1,11 +1,14 @@ """ Setup database connections """ +from logging import Logger import sqlite3 +from typing import Any, Dict import psycopg2 import logging.config from platformdirs import PlatformDirs from pathlib import Path +import requests import yaml @@ -54,3 +57,37 @@ def get_logger(level, config: str): logger.setLevel(logging.INFO) return logger +def get_api_config(log: Logger, config: str) -> Dict[str, Any]: + api_data: Dict[str, Any] = {} + token: str | None = None + host: str | None = None + port: int = 0 + dirs = PlatformDirs(config) + api_config = Path(dirs.user_config_dir, 'api.yaml') + with open(api_config, 'rt') as f: + api_data = yaml.safe_load(f.read()) + if not api_data: + log.fatal("API configuration is missing") + return api_data + host = api_data["host"] + port = api_data["login_port"] + if not token: + log.info("Call login first") + login_url = f"http://{host}:{port}/login" + login_data = {} + login_data['user'] = api_data["user"] + login_data['pass'] = api_data["pass"] + response = requests.post(login_url, data=login_data) + status = response.status_code + log.info(f"Status: {status}") + if status != 200: + log.fatal("authentication failed") + return api_data + data = response.json() + token = data['token'] + api_data['token'] = token + with open(api_config, 'w') as f: + yaml.dump(api_data, f) + else: + token = api_data['token'] + return api_data diff --git a/kontor-scripts/download.py b/kontor-scripts/download.py index b58c5c2..7ee338c 100644 --- a/kontor-scripts/download.py +++ b/kontor-scripts/download.py @@ -17,7 +17,7 @@ from platformdirs import PlatformDirs import requests import yaml -from config import get_logger +from config import get_api_config, get_logger parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("--verbose", "-v", action="count", default=0) @@ -106,9 +106,11 @@ def is_file_downloaded(media_file: dict, dir: Path) -> FileStatus: return FileStatus.UNKNOWN -def update_status(item_id: UUID, file_info: dict): - url: str = f"http://127.0.0.1:8800/api/media/files/{item_id}" - headers: Dict[str, str] = {"Authorization": f"Bearer {bearer_token}"} +def update_status(item_id: UUID, file_info: dict, api_data: Dict[str, Any]): + host = api_data["host"] + token = api_data['token'] + url: str = f"http://{host}:{port}/api/media/files/{item_id}" + headers: Dict[str, str] = {"Authorization": f"Bearer {token}"} update = requests.put(url, headers=headers, json=file_info) log.info(f"update status: {update.status_code}") log.info(f"update result: {update.json()}") @@ -132,39 +134,11 @@ def rename_file(file_info: dict): if __name__ == "__main__": log = get_logger(args.verbose, args.config) log.info("kontor.download started") - api_data: Dict[str, Any] = {} - token: str | None = None - host: str | None = None - port: int = 0 - dirs = PlatformDirs(args.config) - api_config = Path(dirs.user_config_dir, 'api.yaml') - with open(api_config, 'rt') as f: - api_data = yaml.safe_load(f.read()) - if not api_data: - log.fatal("API configuration is missing") - sys.exit(1) + api_data = get_api_config(log, args.config) host = api_data["host"] port = api_data["port"] - if not token: - log.info("Call login first") - login_url = f"http://{host}:{port}/login" - login_data = {} - login_data['user'] = api_data["user"] - login_data['pass'] = api_data["pass"] - response = requests.post(login_url, data=login_data) - status = response.status_code - log.info(f"Status: {status}") - if status != 200: - log.fatal("authentication failed") - sys.exit(0) - data = response.json() - token = data['token'] - api_data['token'] = token - with open(api_config, 'w') as f: - yaml.dump(api_data, f) - else: - token = api_data['token'] - url: str = f"http://{host}:8500/api/media/files?download=true" + token = api_data['token'] + url: str = f"http://{host}:{port}/api/media/files?download=true" headers: Dict[str, str] = {"Authorization": f"Bearer {token}"} response = requests.get(url, headers=headers) log.info(f"Status: {response.status_code}") @@ -186,15 +160,15 @@ if __name__ == "__main__": match download_status: case FileStatus.DOWNLOADED: rename_file(item) - update_status(file_id, item) + update_status(file_id, item, api_data) case FileStatus.RENAMED: log.info("update status") - update_status(file_id, item) + update_status(file_id, item, api_data) case FileStatus.UNKNOWN: download_file(link, item, args.dir) rename_file(item) log.info(f"{item}") - update_status(file_id, item) + update_status(file_id, item, api_data) log.warning(f"processed {mediafile_index}/{entries_count}") if args.limit and args.limit <= mediafile_index: break diff --git a/kontor-scripts/find_links.py b/kontor-scripts/find_links.py index c535357..1ef6854 100644 --- a/kontor-scripts/find_links.py +++ b/kontor-scripts/find_links.py @@ -3,15 +3,18 @@ download files with URLs from DB """ import logging.config import sys -from typing import Any +from typing import Any, Dict import requests import re from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter from bs4 import BeautifulSoup +from config import get_api_config + parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument('--verbose', '-v', action='count', default=0) +parser.add_argument("--config", "-c", default="kontor-docker") parser.add_argument('--all', '-a', action='store_true') parser.add_argument('--limit', '-l', type=int, help='maximum number of links to check') parser.add_argument('--add-actor', action='store_true', help='add missing actors') @@ -60,12 +63,17 @@ def get_logger(level: int) -> logging.Logger: logger.setLevel(logging.CRITICAL) return logger -def update_file(log: logging.Logger, media_file): - update = requests.put(f"http://127.0.0.1:8800/api/media/files/{media_file['id']}", json=media_file) +def update_file(log: logging.Logger, media_file, api_data: Dict[str, Any]): + host = api_data["host"] + port = api_data["port"] + token = api_data['token'] + headers: Dict[str, str] = {"Authorization": f"Bearer {token}"} + url = f"http://{host}:{port}/api/media/files/{media_file['id']}" + update = requests.put(url, headers=headers, json=media_file) log.debug(f"update status: {update.status_code}") log.debug(f"update result: {update.json()}") -def get_actor_links(log: logging.Logger, media_file_url: str) -> list[str]: +def get_actor_links(log: logging.Logger, media_file_url: str, api_data: Dict[str, Any]) -> list[str]: try: r = requests.get(media_file_url) soup = BeautifulSoup(r.content, "html.parser") @@ -74,7 +82,7 @@ def get_actor_links(log: logging.Logger, media_file_url: str) -> list[str]: log.warning(f"{error404.get_text()}") media_file['url'] = None media_file['review'] = False - update_file(log, media_file) + update_file(log, media_file, api_data=api_data) return [] anchors = soup.find_all('a', attrs={'href': re.compile("^https://.*pornstars/.*")}) actor_links = [] @@ -91,19 +99,28 @@ def get_actor_links(log: logging.Logger, media_file_url: str) -> list[str]: log.warning(f"something went wrong: {error}") return [] -def get_media_files(all_files: bool)-> Any: +def get_media_files(all_files: bool, api_data: Dict[str, Any])-> Any: files_url = "" + host = api_data["host"] + port = api_data["port"] + token = api_data['token'] + headers: Dict[str, str] = {"Authorization": f"Bearer {token}"} if all_files: - files_url= "http://127.0.0.1:8800/api/media/files" + files_url= f"http://{host}:{port}/api/media/files" else: - files_url = "http://127.0.0.1:8800/api/media/files?review=true" - response = requests.get(files_url) + files_url = f"http://{host}:{port}/api/media/files?review=true" + response = requests.get(files_url, headers=headers) log.debug(f"Status: {response.status_code}") data = response.json() return data -def update_media_file(item, log: logging.Logger) -> Any: - update = requests.put(f"http://127.0.0.1:8800/api/media/files/{item['id']}", json=item) +def update_media_file(item, log: logging.Logger, api_data: Dict[str, Any]) -> Any: + host = api_data["host"] + port = api_data["port"] + token = api_data['token'] + url: str = f"http://{host}:{port}/api/media/files/{item['id']}" + headers: Dict[str, str] = {"Authorization": f"Bearer {token}"} + update = requests.put(url, headers=headers, json=item) log.debug(f"update status: {update.status_code}") log.debug(f"update result: {update.json()}") return update.json() @@ -112,9 +129,15 @@ def update_media_file_actors(mediafile: dict, actor_id_list: list[dict[str, str]], actor_links: list[str], map_ids_actor: dict[str, str], - log: logging.Logger): + log: logging.Logger, + api_data: Dict[str, Any]): media_file_id = mediafile['id'] - actor_response = requests.put(f"http://127.0.0.1:8800/api/media/files/{media_file_id}/actors", json=actor_id_list) + host = api_data["host"] + port = api_data["port"] + token = api_data['token'] + url: str = f"http://{host}:{port}/api/media/files/{media_file_id}/actors" + headers: Dict[str, str] = {"Authorization": f"Bearer {token}"} + actor_response = requests.put(url, headers=headers, json=actor_id_list) files_actor_list = actor_response.json() persisted_actor_links_count: int = len(files_actor_list) found_actor_links_count: int = len(actor_links) @@ -129,15 +152,20 @@ def update_media_file_actors(mediafile: dict, log.debug(f"check if actor({actor_id}) with {actor_url} in list") if actor_url not in actor_links: log.info(f"actor not found in links, delete relation {file_actor['id']}") - delete_media_file_actor(file_actor['id'], log) + delete_media_file_actor(file_actor['id'], log, api_data) mediafile['review'] = True else: mediafile['review'] = False log.debug(f"found {persisted_actor_links_count} actors") log.debug(f"found actors: {files_actor_list}") -def delete_media_file_actor(media_actor_file_id: str, log: logging.Logger): - delete_response = requests.delete(f"http://127.0.0.1:8800/api/media/actorfiles/{media_actor_file_id}") +def delete_media_file_actor(media_actor_file_id: str, log: logging.Logger, api_data: Dict[str, Any]): + host = api_data["host"] + port = api_data["port"] + token = api_data['token'] + url: str = f"http://{host}:{port}/api/media/actorfiles/{media_actor_file_id}" + headers: Dict[str, str] = {"Authorization": f"Bearer {token}"} + delete_response = requests.delete(url, headers=headers) if delete_response.status_code == 204: log.info(f"actor file relation with id {media_actor_file_id} successfully deleted") @@ -205,11 +233,16 @@ def get_persisted_actor(actor_url: str, log.info(f"found actor {actor_url} missing") return None -def get_actors(log: logging.Logger): +def get_actors(log: logging.Logger, api_data: Dict[str, Any]): actors_url = {} actors_id = {} actors_path = {} - response = requests.get("http://127.0.0.1:8800/api/media/actors") + host = api_data["host"] + port = api_data["port"] + token = api_data['token'] + url: str = f"http://{host}:{port}/api/media/actors" + headers: Dict[str, str] = {"Authorization": f"Bearer {token}"} + response = requests.get(url, headers=headers) data = response.json() for media_actor in data: actor_id = media_actor['id'] @@ -238,9 +271,14 @@ def get_actor_name(actor_url: str, log: logging.Logger) -> str | None: log.warning(f"something went wrong: {error}") return None -def create_actor(actor_url: str, actor_name: str, log: logging.Logger): +def create_actor(actor_url: str, actor_name: str, log: logging.Logger, api_data: Dict[str, Any]): new_actor = { 'name': actor_name, 'url': actor_url} - actor_response = requests.post(f"http://127.0.0.1:8800/api/media/actors", json=new_actor) + host = api_data["host"] + port = api_data["port"] + token = api_data['token'] + url: str = f"http://{host}:{port}/api/media/actors" + headers: Dict[str, str] = {"Authorization": f"Bearer {token}"} + actor_response = requests.post(url, headers=headers, json=new_actor) log.warning(f"add status: {actor_response.status_code}") if actor_response.status_code == 201: actor_data = actor_response.json() @@ -253,8 +291,13 @@ if __name__ == '__main__': log = get_logger(args.verbose) log.warning('kontor.find_links started') log.debug('get all actors') - (actors_url, actors_id, actors_path) = get_actors(log) - data = get_media_files(args.all) + api_data = get_api_config(log, args.config) + host = api_data["host"] + port = api_data["port"] + token = api_data['token'] + + (actors_url, actors_id, actors_path) = get_actors(log, api_data=api_data) + data = get_media_files(args.all, api_data) entries_count = len(data) mediafile_index = 1 log.debug(f"data: {len(data)}") @@ -269,10 +312,10 @@ if __name__ == '__main__': if str(link) == "None": continue log.warning(f"{media_file['id']} - {str(link)}") - actor_links: list[str] = get_actor_links(log, link) + actor_links: list[str] = get_actor_links(log, link, api_data=api_data) actor_id_list = get_actor_ids(actor_links, actors_url, actors_id, actors_path, missing_actors, log) - update_media_file_actors(media_file, actor_id_list, actor_links, actors_id, log) - result = update_media_file(media_file, log) + update_media_file_actors(media_file, actor_id_list, actor_links, actors_id, log, api_data=api_data) + result = update_media_file(media_file, log, api_data=api_data) log.warning(f"processed {mediafile_index}/{entries_count}") if args.limit and args.limit <= mediafile_index: break @@ -281,7 +324,7 @@ if __name__ == '__main__': log.info(f"{link}: {missing_actors[link]}") actor_name = get_actor_name(link, log) if actor_name and args.add_actor: - create_actor(link, actor_name, log) + create_actor(link, actor_name, log, api_data=api_data) log.info("Sort missing actors by occurence count:") sorted_missing = dict(sorted(missing_actors.items(), key=lambda item: item[1])) for key in sorted_missing: