tpeetz/kontor
1
0
Fork 0
Code Issues 6 Pull Requests 2 Actions Packages Projects Releases 2 Wiki Activity

update kontor-api

Browse Source 
(cherry picked from commit f42735326b4dd490351cebb0fc751d62b3a187d0)
This commit is contained in:
Thomas Peetz
2026-01-28 17:20:11 +01:00
committed by Thomas Peetz
parent bb701a903d
commit 25fa07d517
1 changed files with 62 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kontor-scripts/import.py
+62 -31
View File
@@ -1,29 +1,30 @@
"""
import data from json file to PostgreSQL
"""
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from datetime import datetime, date
from logging import Logger
from typing import Any, Dict, List
import os
import json
import os
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
from datetime import date, datetime
from logging import Logger
from pathlib import Path
from typing import Any, Dict, List
import requests
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, Session
from db.models.base import Base
from db.models import registry
from psycopg2.errors import NotNullViolation
from sqlalchemy import create_engine
from sqlalchemy.orm import Session, sessionmaker
from config import get_api_config, get_logger
from db.models import registry
from db.models.base import Base
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--verbose', '-v', action='count', default=0)
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument("--config", "-c", default="kontor-docker")
parser.add_argument('--dry-run', '-m', action='store_true')
parser.add_argument('--cleanup', '-d', action='store_true')
parser.add_argument('--file', '-f', default='~/data.json')
parser.add_argument("--dry-run", "-m", action="store_true")
parser.add_argument("--cleanup", "-d", action="store_true")
parser.add_argument("--file", "-f", default="~/data.json")
args = parser.parse_args()
DB_USER: str = os.getenv("DB_USER", "kontor")
@@ -31,7 +32,9 @@ DB_PASSWORD: str = os.getenv("DB_PASSWORD", "kontor")
DB_SERVER: str = os.getenv("DB_SERVER", "127.0.0.1")
DB_PORT: int = int(os.getenv("DB_PORT", 5432))
DB_DBNAME: str = os.getenv("DB_DBNAME", "kontor")
DATABASE_URL: str = f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_SERVER}:{DB_PORT}/{DB_DBNAME}"
DATABASE_URL: str = (
f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_SERVER}:{DB_PORT}/{DB_DBNAME}"
)
def cleanup_database(db: Session, log, dry_run: bool):
@@ -46,6 +49,7 @@ def cleanup_database(db: Session, log, dry_run: bool):
db.delete(entry)
db.commit()
def load_data(filename: str, log) -> Dict[str, List[Dict[str, Any]]]:
log.debug("load_data")
import_file = Path(filename)
@@ -53,45 +57,54 @@ def load_data(filename: str, log) -> Dict[str, List[Dict[str, Any]]]:
log.info(f"File {filename} does not exist. Do nothing.")
raise FileNotFoundError()
log.info("read json file")
with open(filename, 'r') as json_file:
with open(filename, "r") as json_file:
json_load = json.load(json_file)
return json_load
def get_ids(items: List[Any]) -> List[str]:
result: List[str] = []
for item in items:
result.append(item.id)
return result
def update_item(db: Session, import_data: Dict[str, Any], item: Any, dry_run: bool, log):
for (key, value) in import_data.items():
def update_item(
db: Session, import_data: Dict[str, Any], item: Any, dry_run: bool, log
):
for key, value in import_data.items():
existing_value = getattr(item, str(key))
update: bool = has_changed(existing_value, value, log)
#if key == 'published_on':
# if key == 'published_on':
# log.info(f"{type(value)}:{value} != {type(existing_value)}:{existing_value} : {update}")
if update:
log.info(f"update {key}({existing_value}) with {value}")
if not dry_run:
if value == 'None':
if value == "None":
setattr(item, str(key), None)
else:
setattr(item, str(key), value)
db.add(item)
db.commit()
def has_changed(existing_data: Any, import_data: str, log) -> bool:
if existing_data is None and import_data == 'None':
if existing_data is None and import_data == "None":
return False
if isinstance(existing_data, str):
return existing_data != import_data
if isinstance(existing_data, date):
if len(import_data) > 19:
import_date = datetime.strptime(import_data, "%Y-%m-%d %H:%M:%S.%f")
log.debug(f"{type(existing_data)}:{existing_data} == {import_date} : {existing_data != import_date}")
log.debug(
f"{type(existing_data)}:{existing_data} == {import_date} : {existing_data != import_date}"
)
return existing_data != import_date
if len(import_data) > 10:
import_date = datetime.strptime(import_data, "%Y-%m-%d %H:%M:%S")
log.debug(f"{type(existing_data)}:{existing_data} == {import_date} : {existing_data != import_date}")
log.debug(
f"{type(existing_data)}:{existing_data} == {import_date} : {existing_data != import_date}"
)
return existing_data != import_date
return existing_data.strftime("%Y-%m-%d") != import_data
return existing_data != import_data
@@ -112,11 +125,12 @@ def item_import(db: Session, import_data: Dict[str, Any], dry_run: bool, log):
except Exception as error:
log.info(f"import failed: {error}")
def item_delete(table_name: str, item_id: str, api_data: Dict[str, Any], log: Logger):
log.info(f"delete item {item_id} from {table_name}")
host = api_data["host"]
port = api_data["port"]
token = api_data['token']
token = api_data["token"]
url = ""
match table_name:
case "media_file":
@@ -125,14 +139,16 @@ def item_delete(table_name: str, item_id: str, api_data: Dict[str, Any], log: Lo
url = f"http://{host}:{port}/api/media/actorfiles/{item_id}"
case "media_actor":
url = f"http://{host}:{port}/api/media/actors/{item_id}"
case "profile":
url = f"http://{host}:{port}/api/user/profile/{item_id}"
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
response = requests.delete(url, headers=headers)
log.debug(f"Status: {response.status_code}")
if __name__ == '__main__':
if __name__ == "__main__":
logger = get_logger(args.verbose, "kontor")
logger.info('kontor.import started')
logger.info("kontor.import started")
api_data = get_api_config(logger, args.config)
engine = create_engine(DATABASE_URL)
Base.metadata.create_all(bind=engine, checkfirst=True)
@@ -151,7 +167,7 @@ if __name__ == '__main__':
logger.debug(f"found {len(existing_items)} for table {tablename}")
import_items: List[Dict[str, Any]] = data[tablename]
for import_item in import_items:
item_id: str = import_item['id']
item_id: str = import_item["id"]
if item_id in existing_ids:
logger.debug(f"update {item_id}")
existing_item = db.get(model, item_id)
@@ -166,11 +182,26 @@ if __name__ == '__main__':
for item_id in existing_ids:
match tablename:
case "media_file":
item_delete(table_name=tablename, item_id=item_id, api_data=api_data, log=logger)
item_delete(
table_name=tablename,
item_id=item_id,
api_data=api_data,
log=logger,
)
case "media_actor_file":
item_delete(table_name=tablename, item_id=item_id, api_data=api_data, log=logger)
item_delete(
table_name=tablename,
item_id=item_id,
api_data=api_data,
log=logger,
)
case "media_actor":
item_delete(table_name=tablename, item_id=item_id, api_data=api_data, log=logger)
item_delete(
table_name=tablename,
item_id=item_id,
api_data=api_data,
log=logger,
)
case _:
logger.info("Method to remove remaining item not implemented")
logger.info('kontor.import finished')
logger.info("kontor.import finished")