tpeetz/kontor
1
0
Fork 0
Code Issues 6 Pull Requests 2 Actions Packages Projects Releases 2 Wiki Activity

setup kontor-schema

Browse Source
This commit is contained in:
Thomas Peetz
2025-04-21 21:45:06 +02:00
parent ee78af1abe
commit 6716103d0c
49 changed files with 1029 additions and 897 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kontor-api/.coverage
BIN
View File
Binary file not shown.
kontor-api/Dockerfile
+12 -11
View File
@@ -22,18 +22,18 @@ RUN uv sync
FROM python:3.13-slim-bookworm AS production
# The following secrets are available during build time
RUN --mount=type=secret,id=DB_PASSWORD \
--mount=type=secret,id=DB_USER \
--mount=type=secret,id=DB_NAME \
--mount=type=secret,id=DB_HOST \
--mount=type=secret,id=DB_PORT \
DB_PASSWORD=/run/secrets/DB_PASSWORD \
DB_USER=$(cat /run/secrets/DB_USER) \
DB_NAME=$(cat /run/secrets/DB_NAME) \
DB_HOST=$(cat /run/secrets/DB_HOST) \
DB_PORT=$(cat /run/secrets/DB_PORT)
#RUN --mount=type=secret,id=DB_PASSWORD \
# --mount=type=secret,id=DB_USER \
# --mount=type=secret,id=DB_NAME \
# --mount=type=secret,id=DB_HOST \
# --mount=type=secret,id=DB_PORT \
# DB_PASSWORD=/run/secrets/DB_PASSWORD \
# DB_USER=$(cat /run/secrets/DB_USER) \
# DB_NAME=$(cat /run/secrets/DB_NAME) \
# DB_HOST=$(cat /run/secrets/DB_HOST) \
# DB_PORT=$(cat /run/secrets/DB_PORT)
# RUN --mount=type=secret,id=secret-key,target=secrets.json
#RUN --mount=type=secret,id=secret-key,target=secrets.json
RUN useradd --create-home appuser
USER appuser
@@ -42,6 +42,7 @@ WORKDIR /app
COPY /src src
COPY --from=builder /app/.venv .venv
COPY --from=builder /usr/lib/x86_64-linux-gnu/libmariadb.so.3 /usr/lib/x86_64-linux-gnu
# Set up environment variables for production
ENV PATH="/app/.venv/bin:$PATH"
kontor-api/Makefile
+5 -8
View File
@@ -3,15 +3,12 @@
clean:
find . -name '*.py[co]' -delete
virtualenv:
uv sync
@echo
@echo "VirtualENV Setup Complete. Now run: source env/bin/activate"
@echo
test:
pytest -v --cov --cov-report=term --cov-report=html:coverage-report
DB_HOST=localhost uv run pytest -v --cov --cov-report=term --cov-report=html:coverage-report
docker: clean
docker build -t kontor-api:latest .
docker build --target=production -t kontor-api -t kontor-api:0.1.0-SNAPSHOT .
dev:
uv run fastapi dev src/main.py --port 8008
kontor-api/src/models/media/file.py
+2 -2
View File
@@ -22,7 +22,7 @@ def get_file_details(mediafile: MediaFile) -> MediaFileResponse | None:
file_name=mediafile.file_name,
cloud_link=mediafile.cloud_link,
url=str(mediafile.url),
review=(mediafile.review == 1),
should_download=(mediafile.should_download == 1))
review=mediafile.review,
should_download=mediafile.should_download)
return response
kontor-api/src/models/tysc.py
-8
View File
@@ -1,8 +0,0 @@
from uuid import UUID
from pydantic import BaseModel
class SportResponse(BaseModel):
id: UUID
name: str
kontor-api/src/schema/__init__.py
+9 -21
View File
@@ -1,10 +1,9 @@
import logging.config
from pathlib import Path
import logging
import os
from typing import Annotated
import yaml
from fastapi import Depends
from platformdirs import PlatformDirs
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, Session
@@ -17,30 +16,19 @@ from .media import MediaFile, MediaArticle, MediaVideo
from .base import Base
from .database import KontorDB, ColumnEntry
dirs = PlatformDirs('kontor-docker')
logging_config = Path(dirs.user_config_dir, 'logging-config.yaml')
with open(logging_config, 'rt') as f:
configDict = yaml.safe_load(f.read())
logging.config.dictConfig(configDict)
logger = logging.getLogger('development')
logger.setLevel(logging.DEBUG)
database_config = Path(dirs.user_config_dir, 'database-config.yaml')
with open(database_config, 'rt') as f:
db_config = yaml.safe_load(f.read())
print(db_config)
connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format(
db_config['mariadb']['user'],
db_config['mariadb']['password'],
db_config['mariadb']['host'],
db_config['mariadb']['port'],
db_config['mariadb']['database']
os.environ.get('DB_USER', 'kontor'),
os.environ.get('DB_PASSWORD', 'kontor'),
os.environ.get('DB_HOST', 'mariadb'),
os.environ.get('DB_PORT', 3306),
os.environ.get('DB_NAME', 'kontor')
))
engine = create_engine(connect_string)
SessionLocal = sessionmaker(bind=engine)
Base.metadata.create_all(bind=engine, checkfirst=True)
def get_db():
logger.info("get_db")
logging.info("get_db")
with SessionLocal() as db:
yield db
kontor-api/src/schema/database.py
+14 -15
View File
@@ -1,4 +1,5 @@
import json
import logging
import uuid
from datetime import datetime
from enum import Enum, auto
@@ -38,11 +39,10 @@ class StatusType(Enum):
class KontorDB:
def __init__(self, db_engine: Any, log: Logger):
def __init__(self, db_engine: Any):
self.engine = db_engine
self.registry = {}
self.init_registry()
self.log = log
def init_registry(self):
self.registry[Card.__tablename__] = Card
@@ -126,7 +126,6 @@ class KontorDB:
def get_columns(self, table_name: str) -> dict:
columns = {}
order = 0
__session__ = sessionmaker(self.engine)
table_info = self.get_table_by_name(table_name)
_filters = {'table_id': table_info['id']}
@@ -187,7 +186,7 @@ class KontorDB:
if table in self.registry:
model = self.registry[table]
else:
self.log.info(f"table {table} is not registered")
logging.info(f"table {table} is not registered")
continue
__session__ = sessionmaker(self.engine)
with __session__() as session:
@@ -217,17 +216,17 @@ class KontorDB:
with open(export_file_name, "w") as dump_file:
dump_file.write(json_dump)
case "YAML":
export_file = Path(export_file_name)
pass
case "SQLite":
export_file = Path(export_file_name)
self.log.info(f"{len(results)} tables exported")
pass
logging.info(f"{len(results)} tables exported")
return results
def import_db(self, import_file_name: str) -> dict:
result = {}
import_file = Path(import_file_name)
if not import_file.exists():
self.log.info(f"File {import_file_name} does not exist. Do nothing.")
logging.info(f"File {import_file_name} does not exist. Do nothing.")
return result
match import_file.suffix:
case '.json':
@@ -235,7 +234,7 @@ class KontorDB:
with open(import_file_name, 'r') as json_file:
json_load = json.load(json_file)
for table in json_load:
self.log.info(f"{table}: {len(json_load[table])}")
logging.info(f"{table}: {len(json_load[table])}")
result[table] = self.import_table(table, json_load[table])
case '.yml':
print("read yaml file")
@@ -251,7 +250,7 @@ class KontorDB:
added = []
remaining = []
existing_ids = self.get_ids(table_name)
self.log.info(f"found {len(existing_ids)} existing ids for table {table_name}")
logging.info(f"found {len(existing_ids)} existing ids for table {table_name}")
for item in items:
current_id = item['id']
# print(f"import item: {item}")
@@ -264,7 +263,7 @@ class KontorDB:
changed = self.update_entry(table_name, current_id, item)
updated.append(item)
if changed:
self.log.info(f"{current_id} has changed")
logging.info(f"{current_id} has changed")
updated.append(item)
existing_ids.remove(current_id)
else:
@@ -272,7 +271,7 @@ class KontorDB:
self.add_entry(table_name, item)
added.append(item)
except IntegrityError as error:
self.log.info(f"Could not add item, due to: {error.detail}")
logging.info(f"Could not add item, due to: {error.detail}")
if len(existing_ids) > 0:
print(f"remaining items for {table_name}: {existing_ids}")
remaining.extend(existing_ids)
@@ -291,7 +290,7 @@ class KontorDB:
return existing_ids
def add_entry(self, table_name: str, update_item: dict):
self.log.debug(f"add entry to table {table_name} with {update_item}")
logging.debug(f"add entry to table {table_name} with {update_item}")
__session__ = sessionmaker(self.engine)
with __session__() as session:
add_item = self.registry[table_name]()
@@ -313,11 +312,11 @@ class KontorDB:
if type(existing_value) is not type(update_value):
existing_value = str(existing_value)
if existing_value != update_value:
self.log.info(f"{key} has changed: {existing_value} != {update_value}")
logging.info(f"{key} has changed: {existing_value} != {update_value}")
setattr(existing_item, key, update_value)
session.commit()
changed = True
self.log.info(f"update {key} with {update_value}")
logging.info(f"update {key} with {update_value}")
return changed
def add_link(self, link: str) -> dict:
kontor-api/src/schema/media.py
+3 -2
View File
@@ -1,3 +1,4 @@
import logging
import re
import subprocess
from datetime import datetime
@@ -22,7 +23,7 @@ class MediaFile(Base, BaseMixin, BaseVideoMixin):
return f'{self.title}({self.id})'
def update_title(self) -> None:
print(f"update title for {self.url}")
logging.info(f"update title for {self.url}")
try:
r = requests.get(self.url)
soup = BeautifulSoup(r.content, "html.parser")
@@ -35,7 +36,7 @@ class MediaFile(Base, BaseMixin, BaseVideoMixin):
self.last_modified_date = datetime.now()
def download_file(self, download_dir: str, dl_tool: str):
print(f"download file for {self.url} to {download_dir}")
logging.info(f"download file for {self.url} to {download_dir}")
result = subprocess.run([dl_tool, self.url], cwd=download_dir, capture_output=True, text=True)
if result.returncode == 0:
output = result.stdout
kontor-api/uv.lock
Generated
+44
View File
@@ -89,6 +89,35 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 },
]
[[package]]
name = "coverage"
version = "7.8.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/19/4f/2251e65033ed2ce1e68f00f91a0294e0f80c80ae8c3ebbe2f12828c4cd53/coverage-7.8.0.tar.gz", hash = "sha256:7a3d62b3b03b4b6fd41a085f3574874cf946cb4604d2b4d3e8dca8cd570ca501", size = 811872 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/f3/21/87e9b97b568e223f3438d93072479c2f36cc9b3f6b9f7094b9d50232acc0/coverage-7.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ac46d0c2dd5820ce93943a501ac5f6548ea81594777ca585bf002aa8854cacd", size = 211708 },
{ url = "https://files.pythonhosted.org/packages/75/be/882d08b28a0d19c9c4c2e8a1c6ebe1f79c9c839eb46d4fca3bd3b34562b9/coverage-7.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:771eb7587a0563ca5bb6f622b9ed7f9d07bd08900f7589b4febff05f469bea00", size = 211981 },
{ url = "https://files.pythonhosted.org/packages/7a/1d/ce99612ebd58082fbe3f8c66f6d8d5694976c76a0d474503fa70633ec77f/coverage-7.8.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42421e04069fb2cbcbca5a696c4050b84a43b05392679d4068acbe65449b5c64", size = 245495 },
{ url = "https://files.pythonhosted.org/packages/dc/8d/6115abe97df98db6b2bd76aae395fcc941d039a7acd25f741312ced9a78f/coverage-7.8.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:554fec1199d93ab30adaa751db68acec2b41c5602ac944bb19187cb9a41a8067", size = 242538 },
{ url = "https://files.pythonhosted.org/packages/cb/74/2f8cc196643b15bc096d60e073691dadb3dca48418f08bc78dd6e899383e/coverage-7.8.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aaeb00761f985007b38cf463b1d160a14a22c34eb3f6a39d9ad6fc27cb73008", size = 244561 },
{ url = "https://files.pythonhosted.org/packages/22/70/c10c77cd77970ac965734fe3419f2c98665f6e982744a9bfb0e749d298f4/coverage-7.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:581a40c7b94921fffd6457ffe532259813fc68eb2bdda60fa8cc343414ce3733", size = 244633 },
{ url = "https://files.pythonhosted.org/packages/38/5a/4f7569d946a07c952688debee18c2bb9ab24f88027e3d71fd25dbc2f9dca/coverage-7.8.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f319bae0321bc838e205bf9e5bc28f0a3165f30c203b610f17ab5552cff90323", size = 242712 },
{ url = "https://files.pythonhosted.org/packages/bb/a1/03a43b33f50475a632a91ea8c127f7e35e53786dbe6781c25f19fd5a65f8/coverage-7.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:04bfec25a8ef1c5f41f5e7e5c842f6b615599ca8ba8391ec33a9290d9d2db3a3", size = 244000 },
{ url = "https://files.pythonhosted.org/packages/6a/89/ab6c43b1788a3128e4d1b7b54214548dcad75a621f9d277b14d16a80d8a1/coverage-7.8.0-cp313-cp313-win32.whl", hash = "sha256:dd19608788b50eed889e13a5d71d832edc34fc9dfce606f66e8f9f917eef910d", size = 214195 },
{ url = "https://files.pythonhosted.org/packages/12/12/6bf5f9a8b063d116bac536a7fb594fc35cb04981654cccb4bbfea5dcdfa0/coverage-7.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:a9abbccd778d98e9c7e85038e35e91e67f5b520776781d9a1e2ee9d400869487", size = 214998 },
{ url = "https://files.pythonhosted.org/packages/2a/e6/1e9df74ef7a1c983a9c7443dac8aac37a46f1939ae3499424622e72a6f78/coverage-7.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:18c5ae6d061ad5b3e7eef4363fb27a0576012a7447af48be6c75b88494c6cf25", size = 212541 },
{ url = "https://files.pythonhosted.org/packages/04/51/c32174edb7ee49744e2e81c4b1414ac9df3dacfcb5b5f273b7f285ad43f6/coverage-7.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:95aa6ae391a22bbbce1b77ddac846c98c5473de0372ba5c463480043a07bff42", size = 212767 },
{ url = "https://files.pythonhosted.org/packages/e9/8f/f454cbdb5212f13f29d4a7983db69169f1937e869a5142bce983ded52162/coverage-7.8.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e013b07ba1c748dacc2a80e69a46286ff145935f260eb8c72df7185bf048f502", size = 256997 },
{ url = "https://files.pythonhosted.org/packages/e6/74/2bf9e78b321216d6ee90a81e5c22f912fc428442c830c4077b4a071db66f/coverage-7.8.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d766a4f0e5aa1ba056ec3496243150698dc0481902e2b8559314368717be82b1", size = 252708 },
{ url = "https://files.pythonhosted.org/packages/92/4d/50d7eb1e9a6062bee6e2f92e78b0998848a972e9afad349b6cdde6fa9e32/coverage-7.8.0-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad80e6b4a0c3cb6f10f29ae4c60e991f424e6b14219d46f1e7d442b938ee68a4", size = 255046 },
{ url = "https://files.pythonhosted.org/packages/40/9e/71fb4e7402a07c4198ab44fc564d09d7d0ffca46a9fb7b0a7b929e7641bd/coverage-7.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:b87eb6fc9e1bb8f98892a2458781348fa37e6925f35bb6ceb9d4afd54ba36c73", size = 256139 },
{ url = "https://files.pythonhosted.org/packages/49/1a/78d37f7a42b5beff027e807c2843185961fdae7fe23aad5a4837c93f9d25/coverage-7.8.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:d1ba00ae33be84066cfbe7361d4e04dec78445b2b88bdb734d0d1cbab916025a", size = 254307 },
{ url = "https://files.pythonhosted.org/packages/58/e9/8fb8e0ff6bef5e170ee19d59ca694f9001b2ec085dc99b4f65c128bb3f9a/coverage-7.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f3c38e4e5ccbdc9198aecc766cedbb134b2d89bf64533973678dfcf07effd883", size = 255116 },
{ url = "https://files.pythonhosted.org/packages/56/b0/d968ecdbe6fe0a863de7169bbe9e8a476868959f3af24981f6a10d2b6924/coverage-7.8.0-cp313-cp313t-win32.whl", hash = "sha256:379fe315e206b14e21db5240f89dc0774bdd3e25c3c58c2c733c99eca96f1ada", size = 214909 },
{ url = "https://files.pythonhosted.org/packages/87/e9/d6b7ef9fecf42dfb418d93544af47c940aa83056c49e6021a564aafbc91f/coverage-7.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:2e4b6b87bb0c846a9315e3ab4be2d52fac905100565f4b92f02c445c8799e257", size = 216068 },
{ url = "https://files.pythonhosted.org/packages/59/f1/4da7717f0063a222db253e7121bd6a56f6fb1ba439dcc36659088793347c/coverage-7.8.0-py3-none-any.whl", hash = "sha256:dbf364b4c5e7bae9250528167dfe40219b62e2d573c854d74be213e1e52069f7", size = 203435 },
]
[[package]]
name = "dnspython"
version = "2.7.0"
@@ -275,6 +304,7 @@ dependencies = [
{ name = "pathlib" },
{ name = "platformdirs" },
{ name = "pytest" },
{ name = "pytest-cov" },
{ name = "pyyaml" },
{ name = "requests" },
{ name = "sqlalchemy" },
@@ -290,6 +320,7 @@ requires-dist = [
{ name = "pathlib", specifier = ">=1.0.1" },
{ name = "platformdirs", specifier = ">=4.3.7" },
{ name = "pytest", specifier = "==7.4.0" },
{ name = "pytest-cov", specifier = ">=6.1.1" },
{ name = "pyyaml", specifier = ">=6.0.2" },
{ name = "requests", specifier = ">=2.32.3" },
{ name = "sqlalchemy", specifier = ">=2.0.40" },
@@ -461,6 +492,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/33/b2/741130cbcf2bbfa852ed95a60dc311c9e232c7ed25bac3d9b8880a8df4ae/pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32", size = 323580 },
]
[[package]]
name = "pytest-cov"
version = "6.1.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "coverage" },
{ name = "pytest" },
]
sdist = { url = "https://files.pythonhosted.org/packages/25/69/5f1e57f6c5a39f81411b550027bf72842c4567ff5fd572bed1edc9e4b5d9/pytest_cov-6.1.1.tar.gz", hash = "sha256:46935f7aaefba760e716c2ebfbe1c216240b9592966e7da99ea8292d4d3e2a0a", size = 66857 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/28/d0/def53b4a790cfb21483016430ed828f64830dd981ebe1089971cd10cab25/pytest_cov-6.1.1-py3-none-any.whl", hash = "sha256:bddf29ed2d0ab6f4df17b4c55b0a657287db8684af9c42ea546b21b1041b3dde", size = 23841 },
]
[[package]]
name = "python-dotenv"
version = "1.1.0"