From c1d913e36bdf7bc4d3ae80bbb44a35d8480d9494 Mon Sep 17 00:00:00 2001 From: Thomas Peetz Date: Thu, 17 Sep 2026 16:37:02 +0200 Subject: [PATCH] add Python library diskcache --- kontor-scripts/api.py | 8 +++++++- kontor-scripts/check_kontor.py | 14 ++++++++------ kontor-scripts/pyproject.toml | 1 + kontor-scripts/uv.lock | 11 +++++++++++ 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/kontor-scripts/api.py b/kontor-scripts/api.py index 0a3a03b..f642918 100644 --- a/kontor-scripts/api.py +++ b/kontor-scripts/api.py @@ -4,6 +4,7 @@ from logging import Logger from pathlib import Path from typing import Dict, List, Optional from uuid import UUID +from diskcache import Index from platformdirs import PlatformDirs import requests @@ -142,12 +143,15 @@ class Server: self.token = str(token) self.token_type = str(token_type) - def request(self, log: Logger, table: str, param: Optional[Option] = None): + def request(self, log: Logger, table: str, index: Index, param: Optional[Option] = None): """ Requests data from Kontor-API instance by given table and optional parameters. """ if not param: url: str = f"{self.url}/{MAPPING[table]}" + if table in index: + log.info("use chached %s", table) + return index[table] else: url: str = f"{self.url}/{MAPPING[table]}{str(param)}" headers: Dict[str, str] = {"Authorization": f"Bearer {self.token}"} @@ -160,6 +164,8 @@ class Server: if response.status_code == 404: raise EndPointNotAvailableException data = response.json() + log.info("cache values of %s", table) + index[table] = data return data def update(self, log: Logger, table: str, item_id: UUID, file_info: dict): diff --git a/kontor-scripts/check_kontor.py b/kontor-scripts/check_kontor.py index 9cf4d07..0c909ff 100644 --- a/kontor-scripts/check_kontor.py +++ b/kontor-scripts/check_kontor.py @@ -9,6 +9,7 @@ from typing import Any, Dict, List, Optional from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter from urllib.parse import urlparse from simple_term_menu import TerminalMenu +from diskcache import Index from api import Server, get_api_config from log import get_logger @@ -55,14 +56,14 @@ def remove_file(log: Logger, item_data: Dict[str, Any], media_dirs: List[str]): log.info("File not found %s", media_file.absolute()) -def check_duplicate_links(log: Logger, server: Optional[Server], media_dirs: List[str]): +def check_duplicate_links(log: Logger, server: Optional[Server], media_dirs: List[str], index: Index): """ Check if there are MediaFile URLs which only differ in hostname. """ if server is None: log.info("no server selected") return - data = server.request(log=log, table="media_file") + data = server.request(log=log, table="media_file", index=index) mapping = create_item_id_mapping(log=log, data_list=data) visited_link_path: Dict[str, str] = {} duplicate_link_paths: Dict[str, List[str]] = {} @@ -100,7 +101,7 @@ def check_duplicate_links(log: Logger, server: Optional[Server], media_dirs: Lis print("selection canceled") -def check_media_dirs(log: Logger, server: Optional[Server], media_dirs: List[str]): +def check_media_dirs(log: Logger, server: Optional[Server], media_dirs: List[str], index: Index): """ Check if contents of directories match MediaFiles. """ @@ -108,7 +109,7 @@ def check_media_dirs(log: Logger, server: Optional[Server], media_dirs: List[str log.info("no server selected") return local_files = collect_files(media_dirs) - data = server.request(log=log, table="media_file") + data = server.request(log=log, table="media_file", index=index) for item in data: file_name = Path(item["cloud_link"]).name if file_name in local_files: @@ -143,10 +144,11 @@ if __name__ == "__main__": else: dirs.insert(0, args.dir) logger.info(dirs) + index = Index() logger.info("kontor.check_kontor.check_duplicate_links") - check_duplicate_links(logger, first_server, dirs) + check_duplicate_links(logger, first_server, dirs, index) logger.info("kontor.check_kontor.check_media_dirs") - check_media_dirs(logger, first_server, dirs) + check_media_dirs(logger, first_server, dirs, index) # logger.info("kontor.check_kontor.get_ids_from_column_cloud_link") # get_ids_from_column_cloud_link(link_list, mariadb_cursor) # logger.info('found {} ids in column cloud_link'.format(len(link_list))) diff --git a/kontor-scripts/pyproject.toml b/kontor-scripts/pyproject.toml index 378330d..3312b14 100644 --- a/kontor-scripts/pyproject.toml +++ b/kontor-scripts/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "sqlmodel>=0.0.24", "stomp-py", "kontor-model", + "diskcache>=5.6.3", ] [[tool.uv.index]] diff --git a/kontor-scripts/uv.lock b/kontor-scripts/uv.lock index 5354067..bb438a9 100644 --- a/kontor-scripts/uv.lock +++ b/kontor-scripts/uv.lock @@ -140,6 +140,15 @@ wheels = [ { url = "https://nexus.thpeetz.de/repository/pypi-proxy/packages/coverage/7.8.0/coverage-7.8.0-py3-none-any.whl", hash = "sha256:dbf364b4c5e7bae9250528167dfe40219b62e2d573c854d74be213e1e52069f7", size = 203435, upload-time = "2025-03-30T20:36:43.61Z" }, ] +[[package]] +name = "diskcache" +version = "5.6.3" +source = { registry = "https://nexus.thpeetz.de/repository/pypi-proxy/simple" } +sdist = { url = "https://nexus.thpeetz.de/repository/pypi-proxy/packages/diskcache/5.6.3/diskcache-5.6.3.tar.gz", hash = "sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc", size = 67916, upload-time = "2023-08-31T06:12:00.316Z" } +wheels = [ + { url = "https://nexus.thpeetz.de/repository/pypi-proxy/packages/diskcache/5.6.3/diskcache-5.6.3-py3-none-any.whl", hash = "sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19", size = 45550, upload-time = "2023-08-31T06:11:58.822Z" }, +] + [[package]] name = "dnspython" version = "2.7.0" @@ -338,6 +347,7 @@ dependencies = [ { name = "beautifulsoup4" }, { name = "click" }, { name = "coverage" }, + { name = "diskcache" }, { name = "fastapi", extra = ["standard"] }, { name = "kontor-model" }, { name = "msgspec", extra = ["toml", "yaml"] }, @@ -360,6 +370,7 @@ requires-dist = [ { name = "beautifulsoup4", specifier = ">=4.13.4" }, { name = "click", specifier = ">=8.1.8" }, { name = "coverage", specifier = ">=7.8.0" }, + { name = "diskcache", specifier = ">=5.6.3" }, { name = "fastapi", extras = ["standard"], specifier = ">=0.115.12" }, { name = "kontor-model" }, { name = "msgspec", extras = ["toml", "yaml"], specifier = ">=0.21.1" },