tpeetz/kontor
1
0
Fork 0
Code Issues 6 Pull Requests 2 Actions Packages Projects Releases 2 Wiki Activity

setup kontor-schema

Browse Source
This commit is contained in:
Thomas Peetz
2025-04-21 21:45:06 +02:00
parent ee78af1abe
commit 6716103d0c
49 changed files with 1029 additions and 897 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kontor-api/src/models/media/file.py
+2 -2
View File
@@ -22,7 +22,7 @@ def get_file_details(mediafile: MediaFile) -> MediaFileResponse | None:
file_name=mediafile.file_name,
cloud_link=mediafile.cloud_link,
url=str(mediafile.url),
review=(mediafile.review == 1),
should_download=(mediafile.should_download == 1))
review=mediafile.review,
should_download=mediafile.should_download)
return response
kontor-api/src/models/tysc.py
-8
View File
@@ -1,8 +0,0 @@
from uuid import UUID
from pydantic import BaseModel
class SportResponse(BaseModel):
id: UUID
name: str
kontor-api/src/schema/__init__.py
+9 -21
View File
@@ -1,10 +1,9 @@
import logging.config
from pathlib import Path
import logging
import os
from typing import Annotated
import yaml
from fastapi import Depends
from platformdirs import PlatformDirs
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, Session
@@ -17,30 +16,19 @@ from .media import MediaFile, MediaArticle, MediaVideo
from .base import Base
from .database import KontorDB, ColumnEntry
dirs = PlatformDirs('kontor-docker')
logging_config = Path(dirs.user_config_dir, 'logging-config.yaml')
with open(logging_config, 'rt') as f:
configDict = yaml.safe_load(f.read())
logging.config.dictConfig(configDict)
logger = logging.getLogger('development')
logger.setLevel(logging.DEBUG)
database_config = Path(dirs.user_config_dir, 'database-config.yaml')
with open(database_config, 'rt') as f:
db_config = yaml.safe_load(f.read())
print(db_config)
connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format(
db_config['mariadb']['user'],
db_config['mariadb']['password'],
db_config['mariadb']['host'],
db_config['mariadb']['port'],
db_config['mariadb']['database']
os.environ.get('DB_USER', 'kontor'),
os.environ.get('DB_PASSWORD', 'kontor'),
os.environ.get('DB_HOST', 'mariadb'),
os.environ.get('DB_PORT', 3306),
os.environ.get('DB_NAME', 'kontor')
))
engine = create_engine(connect_string)
SessionLocal = sessionmaker(bind=engine)
Base.metadata.create_all(bind=engine, checkfirst=True)
def get_db():
logger.info("get_db")
logging.info("get_db")
with SessionLocal() as db:
yield db
kontor-api/src/schema/database.py
+14 -15
View File
@@ -1,4 +1,5 @@
import json
import logging
import uuid
from datetime import datetime
from enum import Enum, auto
@@ -38,11 +39,10 @@ class StatusType(Enum):
class KontorDB:
def __init__(self, db_engine: Any, log: Logger):
def __init__(self, db_engine: Any):
self.engine = db_engine
self.registry = {}
self.init_registry()
self.log = log
def init_registry(self):
self.registry[Card.__tablename__] = Card
@@ -126,7 +126,6 @@ class KontorDB:
def get_columns(self, table_name: str) -> dict:
columns = {}
order = 0
__session__ = sessionmaker(self.engine)
table_info = self.get_table_by_name(table_name)
_filters = {'table_id': table_info['id']}
@@ -187,7 +186,7 @@ class KontorDB:
if table in self.registry:
model = self.registry[table]
else:
self.log.info(f"table {table} is not registered")
logging.info(f"table {table} is not registered")
continue
__session__ = sessionmaker(self.engine)
with __session__() as session:
@@ -217,17 +216,17 @@ class KontorDB:
with open(export_file_name, "w") as dump_file:
dump_file.write(json_dump)
case "YAML":
export_file = Path(export_file_name)
pass
case "SQLite":
export_file = Path(export_file_name)
self.log.info(f"{len(results)} tables exported")
pass
logging.info(f"{len(results)} tables exported")
return results
def import_db(self, import_file_name: str) -> dict:
result = {}
import_file = Path(import_file_name)
if not import_file.exists():
self.log.info(f"File {import_file_name} does not exist. Do nothing.")
logging.info(f"File {import_file_name} does not exist. Do nothing.")
return result
match import_file.suffix:
case '.json':
@@ -235,7 +234,7 @@ class KontorDB:
with open(import_file_name, 'r') as json_file:
json_load = json.load(json_file)
for table in json_load:
self.log.info(f"{table}: {len(json_load[table])}")
logging.info(f"{table}: {len(json_load[table])}")
result[table] = self.import_table(table, json_load[table])
case '.yml':
print("read yaml file")
@@ -251,7 +250,7 @@ class KontorDB:
added = []
remaining = []
existing_ids = self.get_ids(table_name)
self.log.info(f"found {len(existing_ids)} existing ids for table {table_name}")
logging.info(f"found {len(existing_ids)} existing ids for table {table_name}")
for item in items:
current_id = item['id']
# print(f"import item: {item}")
@@ -264,7 +263,7 @@ class KontorDB:
changed = self.update_entry(table_name, current_id, item)
updated.append(item)
if changed:
self.log.info(f"{current_id} has changed")
logging.info(f"{current_id} has changed")
updated.append(item)
existing_ids.remove(current_id)
else:
@@ -272,7 +271,7 @@ class KontorDB:
self.add_entry(table_name, item)
added.append(item)
except IntegrityError as error:
self.log.info(f"Could not add item, due to: {error.detail}")
logging.info(f"Could not add item, due to: {error.detail}")
if len(existing_ids) > 0:
print(f"remaining items for {table_name}: {existing_ids}")
remaining.extend(existing_ids)
@@ -291,7 +290,7 @@ class KontorDB:
return existing_ids
def add_entry(self, table_name: str, update_item: dict):
self.log.debug(f"add entry to table {table_name} with {update_item}")
logging.debug(f"add entry to table {table_name} with {update_item}")
__session__ = sessionmaker(self.engine)
with __session__() as session:
add_item = self.registry[table_name]()
@@ -313,11 +312,11 @@ class KontorDB:
if type(existing_value) is not type(update_value):
existing_value = str(existing_value)
if existing_value != update_value:
self.log.info(f"{key} has changed: {existing_value} != {update_value}")
logging.info(f"{key} has changed: {existing_value} != {update_value}")
setattr(existing_item, key, update_value)
session.commit()
changed = True
self.log.info(f"update {key} with {update_value}")
logging.info(f"update {key} with {update_value}")
return changed
def add_link(self, link: str) -> dict:
kontor-api/src/schema/media.py
+3 -2
View File
@@ -1,3 +1,4 @@
import logging
import re
import subprocess
from datetime import datetime
@@ -22,7 +23,7 @@ class MediaFile(Base, BaseMixin, BaseVideoMixin):
return f'{self.title}({self.id})'
def update_title(self) -> None:
print(f"update title for {self.url}")
logging.info(f"update title for {self.url}")
try:
r = requests.get(self.url)
soup = BeautifulSoup(r.content, "html.parser")
@@ -35,7 +36,7 @@ class MediaFile(Base, BaseMixin, BaseVideoMixin):
self.last_modified_date = datetime.now()
def download_file(self, download_dir: str, dl_tool: str):
print(f"download file for {self.url} to {download_dir}")
logging.info(f"download file for {self.url} to {download_dir}")
result = subprocess.run([dl_tool, self.url], cwd=download_dir, capture_output=True, text=True)
if result.returncode == 0:
output = result.stdout