fix problem when deleting MediaFile with MediaActor relations

kontor-api/src/apis/version1/media/file.py

@@ -2,7 +2,10 @@ from typing import List
 
 from fastapi import APIRouter, status, HTTPException
 from src.core.log_conf import logger
-from src.db.repository.media.actorfile import create_new_mediaactorfile
+from src.db.repository.media.actorfile import (
+    create_new_mediaactorfile,
+    delete_mediaactorfile,
+)
 from src.db.repository.media.file import delete_mediafile, import_mediafile
 from src.db.session import SessionDep
 from src.schema.media.actor import MediaActorResponse, actor_to_response
@@ -78,8 +81,11 @@ def delete_file(file_id: str, db: SessionDep):
     logger.info("delete MediaFile: %s", file_id)
     actor_files = mediafile.media_actor_files
     logger.info("MediaActorFiles links %s", len(actor_files))
-    if len(actor_files) == 0:
-        delete_mediafile(db, mediafile.id)
+    if len(actor_files) > 0:
+        logger.info("delete MediaActor relations first")
+        for actor_file in actor_files:
+            delete_mediaactorfile(db, actor_file.id)
+    delete_mediafile(db, mediafile.id)
 
 
 @router.get("/files/{file_id}/actors", response_model=list[MediaActorResponse])

kontor-api/src/db/repository/media/actorfile.py

@@ -5,12 +5,16 @@ from sqlalchemy.orm import Session
 
 from src.core.log_conf import logger
 from src.db.models.media import MediaActorFile
-from src.schema.media.actor import MediaActorModel
 from src.schema.media.actorfile import MediaActorFileModel
 
 
-def create_new_mediaactorfile(db: Session, actor_id: str, file_id: str) -> MediaActorFile:
-    logger.info(f"create MediaActorFile with actor {actor_id} and file {file_id}")
+def create_new_mediaactorfile(
+    db: Session, actor_id: str, file_id: str
+) -> MediaActorFile:
+    """
+    Create relation for MediaFile and MediaActor
+    """
+    logger.info("create MediaActorFile with actor %s and file %s", actor_id, file_id)
     media_actor_file: MediaActorFile = MediaActorFile()
     media_actor_file.id = str(uuid.uuid4())
     media_actor_file.created_date = datetime.now()
@@ -23,17 +27,23 @@ def create_new_mediaactorfile(db: Session, actor_id: str, file_id: str) -> Media
     db.refresh(media_actor_file)
     return media_actor_file
 
+
 def delete_mediaactorfile(db: Session, actorfile_id: str):
-    logger.info(f"delete MediaActorFile with id {actorfile_id}")
+    """
+    Delete relation between MediaFile and MediaActor.
+    """
+    logger.info("delete MediaActorFile with id %s", actorfile_id)
     media_actorfile = db.get(MediaActorFile, actorfile_id)
     db.delete(media_actorfile)
     db.commit()
 
-def import_mediaactorfile(db: Session, new_actorfile: MediaActorFileModel) -> MediaActorFile:
+
+def import_mediaactorfile(
+    db: Session, new_actorfile: MediaActorFileModel
+) -> MediaActorFile:
     """
     Import MediaFile and set missing values with default ones.
     """
     logger.info("import MediaActorFile with %s", new_actorfile)
     media_actor_file: MediaActorFile = MediaActorFile()
     return media_actor_file
-    

kontor-api/src/db/repository/media/file.py

@@ -9,6 +9,9 @@ from src.schema.media.file import MediaFileModel
 
 
 def create_new_mediafile(link: str, db: Session) -> MediaFile:
+    """
+    Create MediaFile with gievne URL.
+    """
     logger.info("create MediaFile with url {link}")
     media_file: MediaFile = MediaFile()
     media_file.id = str(uuid.uuid4())
@@ -21,15 +24,20 @@ def create_new_mediafile(link: str, db: Session) -> MediaFile:
     db.add(media_file)
     db.commit()
     db.refresh(media_file)
-    logger.info(f"created {media_file}")
+    logger.info("created %s", media_file)
     return media_file
 
+
 def delete_mediafile(db: Session, media_file_id: str):
-    logger.info(f"delete MediaFile with id {media_file_id}")
+    """
+    Delete MediaFile with given ID from db.
+    """
+    logger.info("delete MediaFile with id %s", media_file_id)
     media_file = db.get(MediaFile, media_file_id)
     db.delete(media_file)
     db.commit()
 
+
 def import_mediafile(db: Session, new_file: MediaFileModel) -> MediaFile:
     """
     import MediaActor and set missing values with defautl ones.

kontor-scripts/api.py

@@ -83,7 +83,7 @@ class CredentialsValidationException(Exception):
     Raised when login failed or token is outdated.
     """
 
-    
+
 class EndPointNotAvailableException(Exception):
     """
     Raised when calling an not existing endpoint.
@@ -114,7 +114,7 @@ class Server:
     password: str
     timeout: int
 
-    def login(self, log: Logger, refresh_token: bool= False):
+    def login(self, log: Logger, refresh_token: bool = False):
         """
         get token from server by calling login endpoint.
         """
@@ -177,14 +177,16 @@ class Server:
             raise EndPointNotAvailableException
         data = update.json()
         return data
-    
+
     def create(self, log: Logger, table: str, new_item: dict):
         """
         Create item in Kontor-API instance.
         """
         url: str = f"{self.url}/{MAPPING[table]}"
         headers: Dict[str, str] = {"Authorization": f"Bearer {self.token}"}
-        create = requests.post(url, headers=headers, json=new_item, timeout=self.timeout)
+        create = requests.post(
+            url, headers=headers, json=new_item, timeout=self.timeout
+        )
         log.info(f"Status: {create.status_code}")
         if create.status_code == 404:
             raise EndPointNotAvailableException
@@ -192,7 +194,15 @@ class Server:
             log.fatal("Create Exception %s", create.json())
         data = create.json()
         return data
-        
+
+    def delete(self, log: Logger, table: str, item_id: str):
+        """
+        Delete item in Kontor-API instance.
+        """
+        url: str = f"{self.url}/{MAPPING[table]}/{item_id}"
+        headers: Dict[str, str] = {"Authorization": f"Bearer {self.token}"}
+        delete = requests.delete(url, headers=headers, timeout=self.timeout)
+        log.info(f"Status: {delete.status_code}")
 
 
 @dataclass

kontor-scripts/check_kontor.py

@@ -1,40 +1,27 @@
 """
 Checks the database kontor
 """
-from dataclasses import dataclass
-from enum import Enum, auto
+
 from logging import Logger
-from typing import Dict, List, Optional
+from pathlib import Path
+from typing import Any, Dict, List, Optional
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
 from urllib.parse import urlparse
+import click
+from simple_term_menu import TerminalMenu
 
-from api import Option, OptionType, Server, get_api_config, get_logger
+from api import Server, get_api_config, get_logger
 
 
 parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
 parser.add_argument("--verbose", "-v", action="count", default=0)
 parser.add_argument("--config", "-c", default="kontor-api")
 parser.add_argument("--dir", "-d", default="/data/media")
+parser.add_argument("--add-dir", "-a", action="append")
 parser.add_argument("--dry-run", "-m", action="store_true")
 parser.add_argument("--server", "-s")
 args = parser.parse_args()
 
-class StatusType(Enum):
-    UNKNOWN = auto()
-    FILE_NAME = auto()
-    FILE_ID = auto()
-    DUPLICATE = auto()
-    CLOUD_LINK = auto()
-    CLOUD_LINK_ID = auto()
-
-class FileStatus:
-    id: str | None = None
-    status_type: StatusType = StatusType.UNKNOWN
-
-    def get_response(self, response: dict):
-        self.status_type = StatusType.FILE_NAME
-        self.id = response['id']
-
 
 def create_item_id_mapping(log: Logger, data_list: List[dict]) -> Dict[str, dict]:
     """
@@ -47,8 +34,32 @@ def create_item_id_mapping(log: Logger, data_list: List[dict]) -> Dict[str, dict
     return item_id_mapping
 
 
-def check_duplicate_links(log: Logger, server: Server):
-    data = server.request(log=logger, table="media_file")
+def remove_file(log: Logger, item_data: Dict[str, Any], media_dirs: List[str]):
+    """
+    Delete file from path in dictionary.
+    """
+    log.debug(item_data)
+    cloud_link = item_data["cloud_link"]
+    for file_dir in media_dirs:
+        log.info("look in %s", file_dir)
+        file_name = Path(cloud_link).name
+        media_file = Path(file_dir, file_name)
+        if media_file.exists():
+            log.info("File to remove %s", media_file.absolute())
+            media_file.unlink(missing_ok=True)
+            break
+        else:
+            log.info("File not found %s", media_file.absolute())
+
+
+def check_duplicate_links(log: Logger, server: Optional[Server], media_dirs: List[str]):
+    """
+    Check if there are MediaFile URLs which only differ in hostname.
+    """
+    if server is None:
+        log.info("no server selected")
+        return
+    data = server.request(log=log, table="media_file")
     mapping = create_item_id_mapping(log=log, data_list=data)
     visited_link_path: Dict[str, str] = {}
     duplicate_link_paths: Dict[str, List[str]] = {}
@@ -60,7 +71,7 @@ def check_duplicate_links(log: Logger, server: Server):
         parsed_url = urlparse(link)
         link_path = parsed_url.path
         if link_path in visited_link_path:
-            log.info("duplicate url path found: %s", link_path)
+            log.debug("duplicate url path found: %s", link_path)
             if link_path in duplicate_link_paths:
                 duplicate_link_paths[link_path].append(file_id)
             else:
@@ -70,32 +81,42 @@ def check_duplicate_links(log: Logger, server: Server):
         else:
             visited_link_path[link_path] = file_id
     log.info("found %s duplicate links", len(duplicate_link_paths.keys()))
-    deletion_list: List[str] = []
-    for key, value in duplicate_link_paths.items():
-        if len(value) == 2:
-            log.info("%s:\n%s - %s\n%s - %s", key, value[0], mapping[value[0]]["url"], value[1], mapping[value[1]]["url"])
-            if mapping[value[0]]["url"].startswith("https://xhamster"):
-                deletion_list.append(value[0])
-            else:
-                deletion_list.append(value[1])
+    for _, value in duplicate_link_paths.items():
+        choices = [mapping[value[0]]["url"], mapping[value[1]]["url"], "Abbruch"]
+        menu = TerminalMenu(
+            choices, title="Choose an link to delete:", multi_select=False
+        )
+        menu_choice = menu.show()
+        if isinstance(menu_choice, int):
+            if menu_choice == 2:
+                break
+            index: int = int(menu_choice)
+            server.delete(log=log, table="media_file", item_id=value[index])
+            remove_file(log, mapping[value[index]], media_dirs)
         else:
-            log.info("found %s links", len(value))
-    for key in deletion_list:
-        log.info("%s - %s", key, mapping[key]["url"])
+            print("selection canceled")
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     logger = get_logger(args.verbose, args.config)
     logger.info("kontor.check_kontor started")
     APICONFIG = get_api_config(logger, args.config)
-    server: Server = APICONFIG.server[0]
+    first_server: Optional[Server] = APICONFIG.get_server(args.server)
+    if not first_server:
+        SystemExit(2)
+    dirs: List[str] = args.add_dir
+    if dirs is None:
+        dirs = [args.dir]
+    else:
+        dirs.insert(0, args.dir)
+    logger.info(dirs)
     logger.info("kontor.check_kontor.check_duplicate_links")
-    check_duplicate_links(logger, server)
-    #logger.info("kontor.check_kontor.update_cloud_link_with_found_files")
-    #update_cloud_link_with_found_files(data_dir, mariadb_conn, args.dry_run)
-    #logger.info("kontor.check_kontor.get_ids_from_column_cloud_link")
-    #get_ids_from_column_cloud_link(link_list, mariadb_cursor)
-    #logger.info('found {} ids in column cloud_link'.format(len(link_list)))
-    #logger.info("kontor.check_kontor.checking_ids_from_cloud_link")
-    #checking_ids_from_cloud_link(link_list, mariadb_cursor)
+    check_duplicate_links(logger, first_server, dirs)
+    # logger.info("kontor.check_kontor.update_cloud_link_with_found_files")
+    # update_cloud_link_with_found_files(data_dir, mariadb_conn, args.dry_run)
+    # logger.info("kontor.check_kontor.get_ids_from_column_cloud_link")
+    # get_ids_from_column_cloud_link(link_list, mariadb_cursor)
+    # logger.info('found {} ids in column cloud_link'.format(len(link_list)))
+    # logger.info("kontor.check_kontor.checking_ids_from_cloud_link")
+    # checking_ids_from_cloud_link(link_list, mariadb_cursor)
     logger.info("kontor.check_kontor finished")

kontor-scripts/pyproject.toml

@@ -11,6 +11,7 @@ maintainers = [
 ]
 dependencies = [
     "beautifulsoup4>=4.13.4",
+    "click>=8.1.8",
     "coverage>=7.8.0",
     "fastapi[standard]>=0.115.12",
     "pathlib>=1.0.1",
@@ -21,6 +22,7 @@ dependencies = [
     "python-qpid-proton>=0.40.0",
     "pyyaml>=6.0.2",
     "requests>=2.32.3",
+    "simple-term-menu>=1.6.6",
     "sqlalchemy>=2.0.40",
     "sqlmodel>=0.0.24",
     "stomp.py",

kontor-scripts/sync.py

@@ -88,7 +88,9 @@ if __name__ == "__main__":
         )
     if len(server_list) > 1:
         for table, path in MAPPING.items():
-            mapping = create_item_id_mapping(logger, export_data[server_list[1].name][table])
+            mapping = create_item_id_mapping(
+                logger, export_data[server_list[1].name][table]
+            )
             for item in export_data[server_list[0].name][table]:
                 logger.debug("checking  %s:%s", table, item["id"])
                 check_item_id = item["id"]
@@ -99,10 +101,11 @@ if __name__ == "__main__":
                             "checking values for %s != %s", item["id"], check_item["id"]
                         )
                         logger.debug("diff: %s\n%s", item, check_item)
-                        result = server_list[1].update(
-                            logger, table, check_item_id, item
-                        )
-                        logger.info("update result: %s", result)
+                        if not args.dry_run:
+                            result = server_list[1].update(
+                                logger, table, check_item_id, item
+                            )
+                            logger.info("update result: %s", result)
                     else:
                         logger.debug(
                             "no changes for: %s(%s - %s)",
@@ -112,9 +115,10 @@ if __name__ == "__main__":
                         )
                 else:
                     logger.info(
-                        "item %s in %s missing: ", check_item_id, server_list[1].name, item
+                        "item %s in %s missing: ", check_item_id, server_list[1].name
                     )
-                    server_list[1].create(logger, table, item)
+                    if not args.dry_run:
+                        server_list[1].create(logger, table, item)
             logger.info("synchronization of %s finished", table)
         logger.info("all tables synchronized")
     else:

kontor-scripts/uv.lock

@@ -324,6 +324,7 @@ version = "0.1.0"
 source = { virtual = "." }
 dependencies = [
     { name = "beautifulsoup4" },
+    { name = "click" },
     { name = "coverage" },
     { name = "fastapi", extra = ["standard"] },
     { name = "pathlib" },
@@ -334,6 +335,7 @@ dependencies = [
     { name = "python-qpid-proton" },
     { name = "pyyaml" },
     { name = "requests" },
+    { name = "simple-term-menu" },
     { name = "sqlalchemy" },
     { name = "sqlmodel" },
     { name = "stomp-py" },
@@ -342,6 +344,7 @@ dependencies = [
 [package.metadata]
 requires-dist = [
     { name = "beautifulsoup4", specifier = ">=4.13.4" },
+    { name = "click", specifier = ">=8.1.8" },
     { name = "coverage", specifier = ">=7.8.0" },
     { name = "fastapi", extras = ["standard"], specifier = ">=0.115.12" },
     { name = "pathlib", specifier = ">=1.0.1" },
@@ -352,6 +355,7 @@ requires-dist = [
     { name = "python-qpid-proton", specifier = ">=0.40.0" },
     { name = "pyyaml", specifier = ">=6.0.2" },
     { name = "requests", specifier = ">=2.32.3" },
+    { name = "simple-term-menu", specifier = ">=1.6.6" },
     { name = "sqlalchemy", specifier = ">=2.0.40" },
     { name = "sqlmodel", specifier = ">=0.0.24" },
     { name = "stomp-py" },
@@ -654,6 +658,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload_time = "2023-10-24T04:13:38.866Z" },
 ]
 
+[[package]]
+name = "simple-term-menu"
+version = "1.6.6"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d8/80/f0f10b4045628645a841d3d98b584a8699005ee03a211fc7c45f6c6f0e99/simple_term_menu-1.6.6.tar.gz", hash = "sha256:9813d36f5749d62d200a5599b1ec88469c71378312adc084c00c00bfbb383893", size = 35493, upload_time = "2024-12-02T16:31:50.639Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/9c/09/21d993e394c1fe5c44cd90453d88ed44932da8dfca006e424c072d77d29b/simple_term_menu-1.6.6-py3-none-any.whl", hash = "sha256:c2a869efa7a9f7e4a9c25858b42ca6974034951c137d5e281f5339b06ed8c9c2", size = 27600, upload_time = "2024-12-02T16:31:48.934Z" },
+]
+
 [[package]]
 name = "sniffio"
 version = "1.3.1"