setup kontor-schema

This commit is contained in:
Thomas Peetz
2025-04-21 21:45:06 +02:00
parent 75d3bf6ffb
commit 727e95e732
14 changed files with 50 additions and 542 deletions
-8
View File
@@ -1,8 +0,0 @@
from uuid import UUID
from pydantic import BaseModel
class SportResponse(BaseModel):
id: UUID
name: str
+14 -15
View File
@@ -1,4 +1,5 @@
import json
import logging
import uuid
from datetime import datetime
from enum import Enum, auto
@@ -43,11 +44,10 @@ class ExportType(Enum):
class KontorDB:
def __init__(self, db_engine: Any, log: Logger):
def __init__(self, db_engine: Any):
self.engine = db_engine
self.registry = {}
self.init_registry()
self.log = log
def init_registry(self):
self.registry[Card.__tablename__] = Card
@@ -131,7 +131,6 @@ class KontorDB:
def get_columns(self, table_name: str) -> dict:
columns = {}
order = 0
__session__ = sessionmaker(self.engine)
table_info = self.get_table_by_name(table_name)
_filters = {'table_id': table_info['id']}
@@ -192,7 +191,7 @@ class KontorDB:
if table in self.registry:
model = self.registry[table]
else:
self.log.info(f"table {table} is not registered")
logging.info(f"table {table} is not registered")
continue
__session__ = sessionmaker(self.engine)
with __session__() as session:
@@ -222,17 +221,17 @@ class KontorDB:
with open(export_file_name, "w") as dump_file:
dump_file.write(json_dump)
case "YAML":
export_file = Path(export_file_name)
pass
case "SQLite":
export_file = Path(export_file_name)
self.log.info(f"{len(results)} tables exported")
pass
logging.info(f"{len(results)} tables exported")
return results
def import_db(self, import_file_name: str) -> dict:
result = {}
import_file = Path(import_file_name)
if not import_file.exists():
self.log.info(f"File {import_file_name} does not exist. Do nothing.")
logging.info(f"File {import_file_name} does not exist. Do nothing.")
return result
match import_file.suffix:
case '.json':
@@ -240,7 +239,7 @@ class KontorDB:
with open(import_file_name, 'r') as json_file:
json_load = json.load(json_file)
for table in json_load:
self.log.info(f"{table}: {len(json_load[table])}")
logging.info(f"{table}: {len(json_load[table])}")
result[table] = self.import_table(table, json_load[table])
case '.yml':
print("read yaml file")
@@ -256,7 +255,7 @@ class KontorDB:
added = []
remaining = []
existing_ids = self.get_ids(table_name)
self.log.info(f"found {len(existing_ids)} existing ids for table {table_name}")
logging.info(f"found {len(existing_ids)} existing ids for table {table_name}")
for item in items:
current_id = item['id']
# print(f"import item: {item}")
@@ -269,7 +268,7 @@ class KontorDB:
changed = self.update_entry(table_name, current_id, item)
updated.append(item)
if changed:
self.log.info(f"{current_id} has changed")
logging.info(f"{current_id} has changed")
updated.append(item)
existing_ids.remove(current_id)
else:
@@ -277,7 +276,7 @@ class KontorDB:
self.add_entry(table_name, item)
added.append(item)
except IntegrityError as error:
self.log.info(f"Could not add item, due to: {error.detail}")
logging.info(f"Could not add item, due to: {error.detail}")
if len(existing_ids) > 0:
print(f"remaining items for {table_name}: {existing_ids}")
remaining.extend(existing_ids)
@@ -296,7 +295,7 @@ class KontorDB:
return existing_ids
def add_entry(self, table_name: str, update_item: dict):
self.log.debug(f"add entry to table {table_name} with {update_item}")
logging.debug(f"add entry to table {table_name} with {update_item}")
__session__ = sessionmaker(self.engine)
with __session__() as session:
add_item = self.registry[table_name]()
@@ -318,11 +317,11 @@ class KontorDB:
if type(existing_value) is not type(update_value):
existing_value = str(existing_value)
if existing_value != update_value:
self.log.info(f"{key} has changed: {existing_value} != {update_value}")
logging.info(f"{key} has changed: {existing_value} != {update_value}")
setattr(existing_item, key, update_value)
session.commit()
changed = True
self.log.info(f"update {key} with {update_value}")
logging.info(f"update {key} with {update_value}")
return changed
def add_link(self, link: str) -> dict:
+3 -2
View File
@@ -1,3 +1,4 @@
import logging
import re
import subprocess
from datetime import datetime
@@ -23,7 +24,7 @@ class MediaFile(Base, BaseMixin, BaseVideoMixin):
return f'{self.title}({self.id})'
def update_title(self) -> None:
print(f"update title for {self.url}")
logging.info(f"update title for {self.url}")
try:
r = requests.get(self.url)
soup = BeautifulSoup(r.content, "html.parser")
@@ -36,7 +37,7 @@ class MediaFile(Base, BaseMixin, BaseVideoMixin):
self.last_modified_date = datetime.now()
def download_file(self, download_dir: str, dl_tool: str):
print(f"download file for {self.url} to {download_dir}")
logging.info(f"download file for {self.url} to {download_dir}")
result = subprocess.run([dl_tool, self.url], cwd=download_dir, capture_output=True, text=True)
if result.returncode == 0:
output = result.stdout