diff --git a/kontor-api/src/apis/version1/media/file.py b/kontor-api/src/apis/version1/media/file.py index 480c33d..de2a664 100644 --- a/kontor-api/src/apis/version1/media/file.py +++ b/kontor-api/src/apis/version1/media/file.py @@ -2,7 +2,10 @@ from typing import List from fastapi import APIRouter, status, HTTPException from src.core.log_conf import logger -from src.db.repository.media.actorfile import create_new_mediaactorfile +from src.db.repository.media.actorfile import ( + create_new_mediaactorfile, + delete_mediaactorfile, +) from src.db.repository.media.file import delete_mediafile, import_mediafile from src.db.session import SessionDep from src.schema.media.actor import MediaActorResponse, actor_to_response @@ -78,8 +81,11 @@ def delete_file(file_id: str, db: SessionDep): logger.info("delete MediaFile: %s", file_id) actor_files = mediafile.media_actor_files logger.info("MediaActorFiles links %s", len(actor_files)) - if len(actor_files) == 0: - delete_mediafile(db, mediafile.id) + if len(actor_files) > 0: + logger.info("delete MediaActor relations first") + for actor_file in actor_files: + delete_mediaactorfile(db, actor_file.id) + delete_mediafile(db, mediafile.id) @router.get("/files/{file_id}/actors", response_model=list[MediaActorResponse]) diff --git a/kontor-api/src/db/repository/media/actorfile.py b/kontor-api/src/db/repository/media/actorfile.py index 66d1892..43d3c8e 100644 --- a/kontor-api/src/db/repository/media/actorfile.py +++ b/kontor-api/src/db/repository/media/actorfile.py @@ -5,12 +5,16 @@ from sqlalchemy.orm import Session from src.core.log_conf import logger from src.db.models.media import MediaActorFile -from src.schema.media.actor import MediaActorModel from src.schema.media.actorfile import MediaActorFileModel -def create_new_mediaactorfile(db: Session, actor_id: str, file_id: str) -> MediaActorFile: - logger.info(f"create MediaActorFile with actor {actor_id} and file {file_id}") +def create_new_mediaactorfile( + db: Session, actor_id: str, file_id: str +) -> MediaActorFile: + """ + Create relation for MediaFile and MediaActor + """ + logger.info("create MediaActorFile with actor %s and file %s", actor_id, file_id) media_actor_file: MediaActorFile = MediaActorFile() media_actor_file.id = str(uuid.uuid4()) media_actor_file.created_date = datetime.now() @@ -23,17 +27,23 @@ def create_new_mediaactorfile(db: Session, actor_id: str, file_id: str) -> Media db.refresh(media_actor_file) return media_actor_file + def delete_mediaactorfile(db: Session, actorfile_id: str): - logger.info(f"delete MediaActorFile with id {actorfile_id}") + """ + Delete relation between MediaFile and MediaActor. + """ + logger.info("delete MediaActorFile with id %s", actorfile_id) media_actorfile = db.get(MediaActorFile, actorfile_id) db.delete(media_actorfile) db.commit() -def import_mediaactorfile(db: Session, new_actorfile: MediaActorFileModel) -> MediaActorFile: + +def import_mediaactorfile( + db: Session, new_actorfile: MediaActorFileModel +) -> MediaActorFile: """ Import MediaFile and set missing values with default ones. """ logger.info("import MediaActorFile with %s", new_actorfile) media_actor_file: MediaActorFile = MediaActorFile() return media_actor_file - \ No newline at end of file diff --git a/kontor-api/src/db/repository/media/file.py b/kontor-api/src/db/repository/media/file.py index 665e484..3185b3c 100644 --- a/kontor-api/src/db/repository/media/file.py +++ b/kontor-api/src/db/repository/media/file.py @@ -9,6 +9,9 @@ from src.schema.media.file import MediaFileModel def create_new_mediafile(link: str, db: Session) -> MediaFile: + """ + Create MediaFile with gievne URL. + """ logger.info("create MediaFile with url {link}") media_file: MediaFile = MediaFile() media_file.id = str(uuid.uuid4()) @@ -21,15 +24,20 @@ def create_new_mediafile(link: str, db: Session) -> MediaFile: db.add(media_file) db.commit() db.refresh(media_file) - logger.info(f"created {media_file}") + logger.info("created %s", media_file) return media_file + def delete_mediafile(db: Session, media_file_id: str): - logger.info(f"delete MediaFile with id {media_file_id}") + """ + Delete MediaFile with given ID from db. + """ + logger.info("delete MediaFile with id %s", media_file_id) media_file = db.get(MediaFile, media_file_id) db.delete(media_file) db.commit() + def import_mediafile(db: Session, new_file: MediaFileModel) -> MediaFile: """ import MediaActor and set missing values with defautl ones. diff --git a/kontor-scripts/api.py b/kontor-scripts/api.py index bb5780e..87dc3c3 100644 --- a/kontor-scripts/api.py +++ b/kontor-scripts/api.py @@ -83,7 +83,7 @@ class CredentialsValidationException(Exception): Raised when login failed or token is outdated. """ - + class EndPointNotAvailableException(Exception): """ Raised when calling an not existing endpoint. @@ -114,7 +114,7 @@ class Server: password: str timeout: int - def login(self, log: Logger, refresh_token: bool= False): + def login(self, log: Logger, refresh_token: bool = False): """ get token from server by calling login endpoint. """ @@ -177,14 +177,16 @@ class Server: raise EndPointNotAvailableException data = update.json() return data - + def create(self, log: Logger, table: str, new_item: dict): """ Create item in Kontor-API instance. """ url: str = f"{self.url}/{MAPPING[table]}" headers: Dict[str, str] = {"Authorization": f"Bearer {self.token}"} - create = requests.post(url, headers=headers, json=new_item, timeout=self.timeout) + create = requests.post( + url, headers=headers, json=new_item, timeout=self.timeout + ) log.info(f"Status: {create.status_code}") if create.status_code == 404: raise EndPointNotAvailableException @@ -192,7 +194,15 @@ class Server: log.fatal("Create Exception %s", create.json()) data = create.json() return data - + + def delete(self, log: Logger, table: str, item_id: str): + """ + Delete item in Kontor-API instance. + """ + url: str = f"{self.url}/{MAPPING[table]}/{item_id}" + headers: Dict[str, str] = {"Authorization": f"Bearer {self.token}"} + delete = requests.delete(url, headers=headers, timeout=self.timeout) + log.info(f"Status: {delete.status_code}") @dataclass diff --git a/kontor-scripts/check_kontor.py b/kontor-scripts/check_kontor.py index 4283227..ccf221f 100644 --- a/kontor-scripts/check_kontor.py +++ b/kontor-scripts/check_kontor.py @@ -1,40 +1,27 @@ """ Checks the database kontor """ -from dataclasses import dataclass -from enum import Enum, auto + from logging import Logger -from typing import Dict, List, Optional +from pathlib import Path +from typing import Any, Dict, List, Optional from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter from urllib.parse import urlparse +import click +from simple_term_menu import TerminalMenu -from api import Option, OptionType, Server, get_api_config, get_logger +from api import Server, get_api_config, get_logger parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument("--config", "-c", default="kontor-api") parser.add_argument("--dir", "-d", default="/data/media") +parser.add_argument("--add-dir", "-a", action="append") parser.add_argument("--dry-run", "-m", action="store_true") parser.add_argument("--server", "-s") args = parser.parse_args() -class StatusType(Enum): - UNKNOWN = auto() - FILE_NAME = auto() - FILE_ID = auto() - DUPLICATE = auto() - CLOUD_LINK = auto() - CLOUD_LINK_ID = auto() - -class FileStatus: - id: str | None = None - status_type: StatusType = StatusType.UNKNOWN - - def get_response(self, response: dict): - self.status_type = StatusType.FILE_NAME - self.id = response['id'] - def create_item_id_mapping(log: Logger, data_list: List[dict]) -> Dict[str, dict]: """ @@ -47,8 +34,32 @@ def create_item_id_mapping(log: Logger, data_list: List[dict]) -> Dict[str, dict return item_id_mapping -def check_duplicate_links(log: Logger, server: Server): - data = server.request(log=logger, table="media_file") +def remove_file(log: Logger, item_data: Dict[str, Any], media_dirs: List[str]): + """ + Delete file from path in dictionary. + """ + log.debug(item_data) + cloud_link = item_data["cloud_link"] + for file_dir in media_dirs: + log.info("look in %s", file_dir) + file_name = Path(cloud_link).name + media_file = Path(file_dir, file_name) + if media_file.exists(): + log.info("File to remove %s", media_file.absolute()) + media_file.unlink(missing_ok=True) + break + else: + log.info("File not found %s", media_file.absolute()) + + +def check_duplicate_links(log: Logger, server: Optional[Server], media_dirs: List[str]): + """ + Check if there are MediaFile URLs which only differ in hostname. + """ + if server is None: + log.info("no server selected") + return + data = server.request(log=log, table="media_file") mapping = create_item_id_mapping(log=log, data_list=data) visited_link_path: Dict[str, str] = {} duplicate_link_paths: Dict[str, List[str]] = {} @@ -60,7 +71,7 @@ def check_duplicate_links(log: Logger, server: Server): parsed_url = urlparse(link) link_path = parsed_url.path if link_path in visited_link_path: - log.info("duplicate url path found: %s", link_path) + log.debug("duplicate url path found: %s", link_path) if link_path in duplicate_link_paths: duplicate_link_paths[link_path].append(file_id) else: @@ -70,32 +81,42 @@ def check_duplicate_links(log: Logger, server: Server): else: visited_link_path[link_path] = file_id log.info("found %s duplicate links", len(duplicate_link_paths.keys())) - deletion_list: List[str] = [] - for key, value in duplicate_link_paths.items(): - if len(value) == 2: - log.info("%s:\n%s - %s\n%s - %s", key, value[0], mapping[value[0]]["url"], value[1], mapping[value[1]]["url"]) - if mapping[value[0]]["url"].startswith("https://xhamster"): - deletion_list.append(value[0]) - else: - deletion_list.append(value[1]) + for _, value in duplicate_link_paths.items(): + choices = [mapping[value[0]]["url"], mapping[value[1]]["url"], "Abbruch"] + menu = TerminalMenu( + choices, title="Choose an link to delete:", multi_select=False + ) + menu_choice = menu.show() + if isinstance(menu_choice, int): + if menu_choice == 2: + break + index: int = int(menu_choice) + server.delete(log=log, table="media_file", item_id=value[index]) + remove_file(log, mapping[value[index]], media_dirs) else: - log.info("found %s links", len(value)) - for key in deletion_list: - log.info("%s - %s", key, mapping[key]["url"]) + print("selection canceled") -if __name__ == '__main__': +if __name__ == "__main__": logger = get_logger(args.verbose, args.config) logger.info("kontor.check_kontor started") APICONFIG = get_api_config(logger, args.config) - server: Server = APICONFIG.server[0] + first_server: Optional[Server] = APICONFIG.get_server(args.server) + if not first_server: + SystemExit(2) + dirs: List[str] = args.add_dir + if dirs is None: + dirs = [args.dir] + else: + dirs.insert(0, args.dir) + logger.info(dirs) logger.info("kontor.check_kontor.check_duplicate_links") - check_duplicate_links(logger, server) - #logger.info("kontor.check_kontor.update_cloud_link_with_found_files") - #update_cloud_link_with_found_files(data_dir, mariadb_conn, args.dry_run) - #logger.info("kontor.check_kontor.get_ids_from_column_cloud_link") - #get_ids_from_column_cloud_link(link_list, mariadb_cursor) - #logger.info('found {} ids in column cloud_link'.format(len(link_list))) - #logger.info("kontor.check_kontor.checking_ids_from_cloud_link") - #checking_ids_from_cloud_link(link_list, mariadb_cursor) + check_duplicate_links(logger, first_server, dirs) + # logger.info("kontor.check_kontor.update_cloud_link_with_found_files") + # update_cloud_link_with_found_files(data_dir, mariadb_conn, args.dry_run) + # logger.info("kontor.check_kontor.get_ids_from_column_cloud_link") + # get_ids_from_column_cloud_link(link_list, mariadb_cursor) + # logger.info('found {} ids in column cloud_link'.format(len(link_list))) + # logger.info("kontor.check_kontor.checking_ids_from_cloud_link") + # checking_ids_from_cloud_link(link_list, mariadb_cursor) logger.info("kontor.check_kontor finished") diff --git a/kontor-scripts/pyproject.toml b/kontor-scripts/pyproject.toml index ca0b9dd..0e13cc8 100644 --- a/kontor-scripts/pyproject.toml +++ b/kontor-scripts/pyproject.toml @@ -11,6 +11,7 @@ maintainers = [ ] dependencies = [ "beautifulsoup4>=4.13.4", + "click>=8.1.8", "coverage>=7.8.0", "fastapi[standard]>=0.115.12", "pathlib>=1.0.1", @@ -21,6 +22,7 @@ dependencies = [ "python-qpid-proton>=0.40.0", "pyyaml>=6.0.2", "requests>=2.32.3", + "simple-term-menu>=1.6.6", "sqlalchemy>=2.0.40", "sqlmodel>=0.0.24", "stomp.py", diff --git a/kontor-scripts/sync.py b/kontor-scripts/sync.py index c8b94bc..8e2c1dd 100644 --- a/kontor-scripts/sync.py +++ b/kontor-scripts/sync.py @@ -88,7 +88,9 @@ if __name__ == "__main__": ) if len(server_list) > 1: for table, path in MAPPING.items(): - mapping = create_item_id_mapping(logger, export_data[server_list[1].name][table]) + mapping = create_item_id_mapping( + logger, export_data[server_list[1].name][table] + ) for item in export_data[server_list[0].name][table]: logger.debug("checking %s:%s", table, item["id"]) check_item_id = item["id"] @@ -99,10 +101,11 @@ if __name__ == "__main__": "checking values for %s != %s", item["id"], check_item["id"] ) logger.debug("diff: %s\n%s", item, check_item) - result = server_list[1].update( - logger, table, check_item_id, item - ) - logger.info("update result: %s", result) + if not args.dry_run: + result = server_list[1].update( + logger, table, check_item_id, item + ) + logger.info("update result: %s", result) else: logger.debug( "no changes for: %s(%s - %s)", @@ -112,9 +115,10 @@ if __name__ == "__main__": ) else: logger.info( - "item %s in %s missing: ", check_item_id, server_list[1].name, item + "item %s in %s missing: ", check_item_id, server_list[1].name ) - server_list[1].create(logger, table, item) + if not args.dry_run: + server_list[1].create(logger, table, item) logger.info("synchronization of %s finished", table) logger.info("all tables synchronized") else: diff --git a/kontor-scripts/uv.lock b/kontor-scripts/uv.lock index 46d69c1..aba3a99 100644 --- a/kontor-scripts/uv.lock +++ b/kontor-scripts/uv.lock @@ -324,6 +324,7 @@ version = "0.1.0" source = { virtual = "." } dependencies = [ { name = "beautifulsoup4" }, + { name = "click" }, { name = "coverage" }, { name = "fastapi", extra = ["standard"] }, { name = "pathlib" }, @@ -334,6 +335,7 @@ dependencies = [ { name = "python-qpid-proton" }, { name = "pyyaml" }, { name = "requests" }, + { name = "simple-term-menu" }, { name = "sqlalchemy" }, { name = "sqlmodel" }, { name = "stomp-py" }, @@ -342,6 +344,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "beautifulsoup4", specifier = ">=4.13.4" }, + { name = "click", specifier = ">=8.1.8" }, { name = "coverage", specifier = ">=7.8.0" }, { name = "fastapi", extras = ["standard"], specifier = ">=0.115.12" }, { name = "pathlib", specifier = ">=1.0.1" }, @@ -352,6 +355,7 @@ requires-dist = [ { name = "python-qpid-proton", specifier = ">=0.40.0" }, { name = "pyyaml", specifier = ">=6.0.2" }, { name = "requests", specifier = ">=2.32.3" }, + { name = "simple-term-menu", specifier = ">=1.6.6" }, { name = "sqlalchemy", specifier = ">=2.0.40" }, { name = "sqlmodel", specifier = ">=0.0.24" }, { name = "stomp-py" }, @@ -654,6 +658,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload_time = "2023-10-24T04:13:38.866Z" }, ] +[[package]] +name = "simple-term-menu" +version = "1.6.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/80/f0f10b4045628645a841d3d98b584a8699005ee03a211fc7c45f6c6f0e99/simple_term_menu-1.6.6.tar.gz", hash = "sha256:9813d36f5749d62d200a5599b1ec88469c71378312adc084c00c00bfbb383893", size = 35493, upload_time = "2024-12-02T16:31:50.639Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/09/21d993e394c1fe5c44cd90453d88ed44932da8dfca006e424c072d77d29b/simple_term_menu-1.6.6-py3-none-any.whl", hash = "sha256:c2a869efa7a9f7e4a9c25858b42ca6974034951c137d5e281f5339b06ed8c9c2", size = 27600, upload_time = "2024-12-02T16:31:48.934Z" }, +] + [[package]] name = "sniffio" version = "1.3.1"