From bfccca72a15abef296999172c66836b64be55913 Mon Sep 17 00:00:00 2001 From: Thomas Peetz Date: Wed, 23 Apr 2025 10:08:22 +0200 Subject: [PATCH] copy schema to kontor-api --- kontor-api/.coverage | Bin 53248 -> 53248 bytes kontor-api/src/kontor_api.egg-info/PKG-INFO | 1 - .../src/kontor_api.egg-info/SOURCES.txt | 9 + .../src/kontor_api.egg-info/requires.txt | 1 - .../src/kontor_api.egg-info/top_level.txt | 1 + kontor-api/src/models/comics/artist.py | 2 +- kontor-api/src/models/comics/comic.py | 2 +- kontor-api/src/models/media/file.py | 2 +- kontor-api/src/routers/__init__.py | 2 +- kontor-api/src/routers/comic.py | 2 +- kontor-api/src/routers/media.py | 2 +- kontor-api/src/routers/tysc.py | 2 +- kontor-api/src/schema/__init__.py | 0 kontor-api/src/schema/admin.py | 77 ++++ kontor-api/src/schema/base.py | 30 ++ kontor-api/src/schema/bookshelf.py | 50 +++ kontor-api/src/schema/comic.py | 99 +++++ kontor-api/src/schema/database.py | 396 ++++++++++++++++++ kontor-api/src/schema/media.py | 92 ++++ kontor-api/src/schema/metadata.py | 41 ++ kontor-api/src/schema/tysc.py | 99 +++++ kontor-api/uv.lock | 19 - 22 files changed, 901 insertions(+), 28 deletions(-) create mode 100644 kontor-api/src/schema/__init__.py create mode 100644 kontor-api/src/schema/admin.py create mode 100644 kontor-api/src/schema/base.py create mode 100644 kontor-api/src/schema/bookshelf.py create mode 100644 kontor-api/src/schema/comic.py create mode 100644 kontor-api/src/schema/database.py create mode 100644 kontor-api/src/schema/media.py create mode 100644 kontor-api/src/schema/metadata.py create mode 100644 kontor-api/src/schema/tysc.py diff --git a/kontor-api/.coverage b/kontor-api/.coverage index d004fb77d1d141bab5c243857ef10af22a26ee23..44d6ca261d7b921a433a3387e3cb7b53e796297f 100644 GIT binary patch delta 884 zcmX|-O-NKx6vy9V-hH3<-Sg(-X}YpV7v@(OVpM86YR>4i3>uWCaxn`voG}rD84H6} z3Kx}v8p~yiidhW>PPNJc3tH=@MNydIqQtV!)frdk{QviV&b{}SUi73FJXqC-e#}u|8TeR>UIavYEbW68nMM7@~d55qw<=p0VvLN-r_ zbOPwO`+IVsGQv=iedP&ziH*v?uc{*qVx!4;B8hjmQuM_Qjr>usu69`{y~P;c>@$do zPHDj@mkmA;Zs21PKhe=YSf7hHq(wt1K4n~uB?fVEyUK_A+SLw%z9q)9yAeZ1U}*`u z0qndzMwjebT55f=R5Q<3m_lRLMRu2U(I3{VnTEG;2Z2RQrB#`d#cEpM<#sBFD;BLS z;L^E2dKz&#vY*SB+xTL%T{(;#u+fOqqaYhGlnWgpa9+X3qjHN}hX$|5WB6`%K|xL; zhn&nqPBwoWZ5D`R%3t(%6Dy=3m4jHBZ1Vo-Nu=Twb>K+w{wsk T$;?%tv%s~ya0Y=dtj7NX;|=v! delta 1188 zcmZ9KUuauZ9LMjuH~0U&=XdizcOY&9eQDP=o3xc~P1dBVHt8Q2OlTKFwq`E4G_H+K zK}@FTDkyFq)`AbF`k-%aBcKz;X!o`~II0hO7~9a72r5{7Sj#Z;cS=)&=i&T*=X<`t z-|ybfz13x@x-4Dw#Cu)+tN^#*eK-P?-li95!rrj2+Gp$z>ksQ2D{t*Fe>IoQ21dG<*vDGoa9{OF)K1g&ekm@ntJ1&d-7AE%F=EENkU@tCl&w}~SU&=aN7>+`4c zXHKC!Xkm*!_TIQE#!Oyd(2C<8F zr}d}xt#!)Gm|t0Y&EM)>lLvLSHr>xQMs3-B$%E{pnHP||Ln7?rbd+<)FR(T}EQen9 zAZ~s;k@k`x%N;)~2evB)j2)_ewvvhd59;(Vt4_s;)4H|O`UMs1h9*l27v8meyL zDPZ*;xLg4f)7|D&GhD5FS!>i{hJixJ`03v#5B8h7&W-x9hcTjPm{N@>Dv$WZ6pXu+ zhldoUsVIufRhh6ypFJnJaFnSRJ2W^AIhcZ@2-5{ts>xy3=6.0.2 Requires-Dist: requests>=2.32.3 Requires-Dist: sqlalchemy>=2.0.40 Requires-Dist: sqlmodel>=0.0.24 -Requires-Dist: kontor.schema>=0.1.0 diff --git a/kontor-api/src/kontor_api.egg-info/SOURCES.txt b/kontor-api/src/kontor_api.egg-info/SOURCES.txt index 69b723b..f0db090 100644 --- a/kontor-api/src/kontor_api.egg-info/SOURCES.txt +++ b/kontor-api/src/kontor_api.egg-info/SOURCES.txt @@ -19,4 +19,13 @@ src/routers/__init__.py src/routers/comic.py src/routers/media.py src/routers/tysc.py +src/schema/__init__.py +src/schema/admin.py +src/schema/base.py +src/schema/bookshelf.py +src/schema/comic.py +src/schema/database.py +src/schema/media.py +src/schema/metadata.py +src/schema/tysc.py tests/test_main.py \ No newline at end of file diff --git a/kontor-api/src/kontor_api.egg-info/requires.txt b/kontor-api/src/kontor_api.egg-info/requires.txt index 15c93d0..a638783 100644 --- a/kontor-api/src/kontor_api.egg-info/requires.txt +++ b/kontor-api/src/kontor_api.egg-info/requires.txt @@ -10,4 +10,3 @@ pyyaml>=6.0.2 requests>=2.32.3 sqlalchemy>=2.0.40 sqlmodel>=0.0.24 -kontor.schema>=0.1.0 diff --git a/kontor-api/src/kontor_api.egg-info/top_level.txt b/kontor-api/src/kontor_api.egg-info/top_level.txt index 579e547..db2075f 100644 --- a/kontor-api/src/kontor_api.egg-info/top_level.txt +++ b/kontor-api/src/kontor_api.egg-info/top_level.txt @@ -2,3 +2,4 @@ __init__ main models routers +schema diff --git a/kontor-api/src/models/comics/artist.py b/kontor-api/src/models/comics/artist.py index 3094b6d..345debe 100644 --- a/kontor-api/src/models/comics/artist.py +++ b/kontor-api/src/models/comics/artist.py @@ -1,7 +1,7 @@ from typing import List, Dict from uuid import UUID -from kontor_schema import Artist +from src.schema.comic import Artist from pydantic import BaseModel diff --git a/kontor-api/src/models/comics/comic.py b/kontor-api/src/models/comics/comic.py index 427d35c..c84cee1 100644 --- a/kontor-api/src/models/comics/comic.py +++ b/kontor-api/src/models/comics/comic.py @@ -1,7 +1,7 @@ from typing import List, Dict from uuid import UUID -from kontor_schema import Comic +from src.schema.comic import Comic from pydantic import BaseModel diff --git a/kontor-api/src/models/media/file.py b/kontor-api/src/models/media/file.py index daf1eaa..af01717 100644 --- a/kontor-api/src/models/media/file.py +++ b/kontor-api/src/models/media/file.py @@ -1,6 +1,6 @@ from uuid import UUID -from kontor_schema import MediaFile +from src.schema.media import MediaFile from pydantic import BaseModel diff --git a/kontor-api/src/routers/__init__.py b/kontor-api/src/routers/__init__.py index 08c7afb..5bc1995 100644 --- a/kontor-api/src/routers/__init__.py +++ b/kontor-api/src/routers/__init__.py @@ -3,7 +3,7 @@ import os from typing import Annotated from fastapi import Depends -from kontor_schema import Base +from src.schema.base import Base from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker, Session diff --git a/kontor-api/src/routers/comic.py b/kontor-api/src/routers/comic.py index 4831c65..583df7b 100644 --- a/kontor-api/src/routers/comic.py +++ b/kontor-api/src/routers/comic.py @@ -1,12 +1,12 @@ from uuid import UUID from typing import List from fastapi import APIRouter, HTTPException, status -from kontor_schema import Comic, Artist from sqlalchemy import select from src.models.comics.comic import ComicResponse, ComicDetailsResponse, get_comic_details from src.models.comics.artist import ArtistCreation, ArtistDetailResponse, ArtistResponse, get_artist_details from src.routers import SessionDep +from src.schema.comic import Comic, Artist router = APIRouter( prefix="/comic", diff --git a/kontor-api/src/routers/media.py b/kontor-api/src/routers/media.py index 4046d82..a933325 100644 --- a/kontor-api/src/routers/media.py +++ b/kontor-api/src/routers/media.py @@ -3,11 +3,11 @@ from typing import List from uuid import uuid4, UUID from fastapi import APIRouter, status, HTTPException -from kontor_schema import MediaFile from sqlalchemy import select, Sequence from src.models.media.file import MediaFileResponse, Link, get_file_details from src.routers import SessionDep +from src.schema.media import MediaFile router = APIRouter( prefix="/media", diff --git a/kontor-api/src/routers/tysc.py b/kontor-api/src/routers/tysc.py index 62eaef5..63342a7 100644 --- a/kontor-api/src/routers/tysc.py +++ b/kontor-api/src/routers/tysc.py @@ -1,9 +1,9 @@ from typing import List from fastapi import APIRouter -from kontor_schema import Sport from src.models.tysc.sport import SportResponse from src.routers import SessionDep +from src.schema.tysc import Sport router = APIRouter( prefix="/tysc", diff --git a/kontor-api/src/schema/__init__.py b/kontor-api/src/schema/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/kontor-api/src/schema/admin.py b/kontor-api/src/schema/admin.py new file mode 100644 index 0000000..2cbfff4 --- /dev/null +++ b/kontor-api/src/schema/admin.py @@ -0,0 +1,77 @@ +from datetime import datetime + +from sqlalchemy import Column, ForeignKey, Integer, String, Boolean +from sqlalchemy.orm import relationship, mapped_column, Mapped + +from .base import Base, BaseMixin + + +class User(Base, BaseMixin): + __tablename__ = 'user' + first_name = Column(String(255)) + last_name = Column(String(255)) + user_name = Column(String(255), nullable=False) + email = Column(String(255)) + password = Column(String(255)) + enabled = Column(Boolean) + matrix = relationship("AuthorizationMatrix") + tokens = relationship("Token") + + def get_full_name(self) -> str: + full_name = "" + if self.first_name is not None: + full_name += self.first_name + if self.last_name is not None: + if len(full_name) > 0: + full_name += " " + full_name += self.last_name + return full_name + + +class Token(Base, BaseMixin): + __tablename__ = "token" + token = Column(String(255), nullable=False, unique=True) + name = Column(String(255)) + last_used_date: Mapped[datetime] = mapped_column() + enabled = Column(Boolean) + user_id = Column(String(255), ForeignKey("user.id"), nullable=False) + user = relationship("User", back_populates="tokens") + + +class Role(Base, BaseMixin): + __tablename__ = "role" + name = Column(String(255), nullable=False) + matrix = relationship("AuthorizationMatrix") + + +class AuthorizationMatrix(Base, BaseMixin): + __tablename__ = "authorization_matrix" + user_id = Column(String, ForeignKey("user.id"), nullable=False) + user = relationship("User", back_populates="matrix") + role_id = Column(String, ForeignKey("role.id"), nullable=False) + role = relationship("Role", back_populates="matrix") + + +class ModuleData(Base, BaseMixin): + __tablename__ = "module_data" + module_name = Column(String(255), nullable=False) + import_data = Column(Boolean) + + +class MailAccount(Base, BaseMixin): + __tablename__ = "mail_account" + host = Column(String(255)) + port = Column(Integer) + protocol = Column(String(255)) + user_name = Column(String(255)) + password = Column(String(255)) + start_tls = Column(Boolean) + + +class Mail(Base, BaseMixin): + __tablename__ = "mail" + folder: Mapped[str] = mapped_column() + subject: Mapped[str] = mapped_column() + body: Mapped[str] = mapped_column() + sent_date: Mapped[datetime] = mapped_column() + received_date: Mapped[datetime] = mapped_column() diff --git a/kontor-api/src/schema/base.py b/kontor-api/src/schema/base.py new file mode 100644 index 0000000..5ef8183 --- /dev/null +++ b/kontor-api/src/schema/base.py @@ -0,0 +1,30 @@ +import uuid +from datetime import datetime + +from sqlalchemy import func, Column, String, Boolean +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column + + +class Base(DeclarativeBase): + pass + + +class BaseMixin: + id = Column(String(255), primary_key=True, default=uuid.uuid4()) + # id: Mapped[str] = mapped_column(primary_key=True, default=uuid.uuid4()) + # created_date = Column(DateTime) + created_date: Mapped[datetime] = mapped_column(default=func.now()) + # last_modified_date = Column(DateTime) + last_modified_date: Mapped[datetime] = mapped_column(default=func.now()) + # version = Column(Integer) + version: Mapped[int] = mapped_column(default=0) + + +class BaseVideoMixin: + cloud_link = Column(String(255)) + file_name = Column(String(255)) + path = Column(String(255)) + review = Column(Boolean) + title = Column(String(255)) + url = Column(String(255), unique=True) + should_download = Column(Boolean) diff --git a/kontor-api/src/schema/bookshelf.py b/kontor-api/src/schema/bookshelf.py new file mode 100644 index 0000000..91e0ae4 --- /dev/null +++ b/kontor-api/src/schema/bookshelf.py @@ -0,0 +1,50 @@ +from sqlalchemy import Column, ForeignKey, Integer, String +from sqlalchemy.orm import relationship + +from .base import Base, BaseMixin + + +class Article(Base, BaseMixin): + __tablename__ = 'article' + title = Column(String(length=255), unique=True) + article_authors = relationship("ArticleAuthor") + + +class Author(Base, BaseMixin): + __tablename__ = 'author' + first_name = Column(String(255)) + last_name = Column(String(255)) + article_authors = relationship("ArticleAuthor") + book_authors = relationship("BookAuthor") + + +class BookshelfPublisher(Base, BaseMixin): + __tablename__ = 'bookshelf_publisher' + name = Column(String(length=255), unique=True) + books = relationship("Book") + + +class Book(Base, BaseMixin): + __tablename__ = 'book' + isbn = Column(String(255), unique=True) + title = Column(String(255)) + year = Column(Integer, nullable=False) + publisher_id = Column(String, ForeignKey('bookshelf_publisher.id'), nullable=False) + publisher = relationship('BookshelfPublisher', back_populates="books") + book_authors = relationship("BookAuthor") + + +class ArticleAuthor(Base, BaseMixin): + __tablename__ = 'article_author' + article_id = Column(String, ForeignKey('article.id'), nullable=False) + article = relationship('Article', back_populates="article_authors") + author_id = Column(String, ForeignKey('author.id'), nullable=False) + author = relationship('Author', back_populates="article_authors") + + +class BookAuthor(Base, BaseMixin): + __tablename__ = 'book_author' + author_id = Column(String, ForeignKey('author.id'), nullable=False) + author = relationship('Author', back_populates="book_authors") + book_id = Column(String, ForeignKey('book.id'), nullable=False) + book = relationship('Book', back_populates="book_authors") diff --git a/kontor-api/src/schema/comic.py b/kontor-api/src/schema/comic.py new file mode 100644 index 0000000..45cb8c3 --- /dev/null +++ b/kontor-api/src/schema/comic.py @@ -0,0 +1,99 @@ +from sqlalchemy import Column, ForeignKey, Integer, String, Boolean +from sqlalchemy.orm import relationship + +from .base import Base, BaseMixin + + +class Publisher(Base, BaseMixin): + __tablename__ = "publisher" + name = Column(String(length=255), unique=True) + comics = relationship("Comic") + + def __repr__(self): + return f'Publisher({self.id} {self.name})' + + def __str__(self): + return self.__repr__() + + +class Comic(Base, BaseMixin): + __tablename__ = 'comic' + title = Column(String(length=255), unique=True) + publisher_id = Column(String, ForeignKey('publisher.id'), nullable=False) + publisher = relationship("Publisher", back_populates="comics") + current_order = Column(Boolean) + completed = Column(Boolean) + issues = relationship("Issue") + story_arcs = relationship("StoryArc") + trade_paperbacks = relationship("TradePaperback") + volumes = relationship("Volume") + comic_works = relationship("ComicWork") + + def __repr__(self): + return f'Comic({self.id} {self.version} {self.title} {self.publisher.name})' + + def __str__(self): + return f'{self.title}({self.id})' + + +class Volume(Base, BaseMixin): + __tablename__ = "volume" + name = Column(String(length=255), nullable=False) + comic_id = Column(String, ForeignKey("comic.id"), nullable=False) + comic = relationship("Comic", back_populates="volumes") + issues = relationship("Issue") + + +class TradePaperback(Base, BaseMixin): + __tablename__ = "trade_paperback" + name = Column(String(length=255), nullable=False) + issue_start = Column(Integer) + issue_end = Column(Integer) + comic_id = Column(String, ForeignKey("comic.id"), nullable=False) + comic = relationship("Comic", back_populates="trade_paperbacks") + + +class StoryArc(Base, BaseMixin): + __tablename__ = "story_arc" + name = Column(String(length=255), nullable=False) + comic_id = Column(String, ForeignKey("comic.id"), nullable=False) + comic = relationship("Comic", back_populates="story_arcs") + + +class Issue(Base, BaseMixin): + __tablename__ = "issue" + issue_number = Column(String(255)) + in_stock = Column(Boolean) + is_read = Column(Boolean) + comic_id = Column(String, ForeignKey("comic.id"), nullable=False) + comic = relationship("Comic", back_populates="issues") + volume_id = Column(String, ForeignKey("volume.id"), nullable=True) + volume = relationship("Volume", back_populates="issues") + + +class Artist(Base, BaseMixin): + __tablename__ = "artist" + name = Column(String(length=255), nullable=False) + comic_works = relationship("ComicWork") + + +class WorkType(Base, BaseMixin): + __tablename__ = "worktype" + name = Column(String(length=255), nullable=False, unique=True) + comic_works = relationship("ComicWork") + + def __repr__(self): + return f'Worktype({self.id} {self.version} {self.name} {len(self.comic_works)})' + + def __str__(self): + return f'{self.name}({self.id})' + + +class ComicWork(Base, BaseMixin): + __tablename__ = "comic_work" + comic_id = Column(String, ForeignKey("comic.id"), nullable=False) + comic = relationship("Comic", back_populates="comic_works") + artist_id = Column(String, ForeignKey("artist.id"), nullable=False) + artist = relationship("Artist", back_populates="comic_works") + work_type_id = Column(String, ForeignKey("worktype.id"), nullable=False) + work_type = relationship("WorkType", back_populates="comic_works") diff --git a/kontor-api/src/schema/database.py b/kontor-api/src/schema/database.py new file mode 100644 index 0000000..54d3596 --- /dev/null +++ b/kontor-api/src/schema/database.py @@ -0,0 +1,396 @@ +import json +import uuid +from datetime import datetime +from enum import Enum, auto +from logging import Logger +from pathlib import Path + +from sqlalchemy import Engine, select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import sessionmaker + +from .tysc import Card, CardSet, Rooster, Team, FieldPosition, Player, Vendor, Sport +from .comic import Issue, TradePaperback, StoryArc, Volume, ComicWork, Artist, Comic, Publisher, WorkType +from .bookshelf import ArticleAuthor, BookAuthor, BookshelfPublisher, Article, Book, Author +from .admin import Mail, MailAccount, ModuleData, Role, User, Token, AuthorizationMatrix +from .metadata import MetaDataTable, MetaDataColumn +from .media import MediaVideo, MediaArticle, MediaFile, MediaActor, MediaActorFile + + +class ColumnEntry(Enum): + COLUMN_NAME = 'column' + COLUMN_LABEL = 'label' + COLUMN_ORDER = 'order' + COLUMN_REF_COLUMN = 'ref_column' + COLUMN_TYPE = 'type' + COLUMN_WIDGET = 'widget' + + +class StatusType(Enum): + UNKNOWN = auto() + FILE_NAME = auto() + FILE_ID = auto() + DUPLICATE = auto() + CLOUD_LINK = auto() + CLOUD_LINK_ID = auto() + + +class ExportType(Enum): + JSON = "JSON" + YAML = "YAML" + SQLITE = "SQLite" + + +class KontorDB: + + def __init__(self, db_engine: Engine, log: Logger): + self.engine = db_engine + self.registry = {} + self.init_registry() + self.log = log + + def init_registry(self): + self.registry[Card.__tablename__] = Card + self.registry[CardSet.__tablename__] = CardSet + self.registry[Rooster.__tablename__] = Rooster + self.registry[Team.__tablename__] = Team + self.registry[FieldPosition.__tablename__] = FieldPosition + self.registry[Player.__tablename__] = Player + self.registry[Vendor.__tablename__] = Vendor + self.registry[Sport.__tablename__] = Sport + self.registry[Issue.__tablename__] = Issue + self.registry[TradePaperback.__tablename__] = TradePaperback + self.registry[StoryArc.__tablename__] = StoryArc + self.registry[Volume.__tablename__] = Volume + self.registry[ComicWork.__tablename__] = ComicWork + self.registry[Artist.__tablename__] = Artist + self.registry[Comic.__tablename__] = Comic + self.registry[Publisher.__tablename__] = Publisher + self.registry[WorkType.__tablename__] = WorkType + self.registry[ArticleAuthor.__tablename__] = ArticleAuthor + self.registry[BookAuthor.__tablename__] = BookAuthor + self.registry[BookshelfPublisher.__tablename__] = BookshelfPublisher + self.registry[Article.__tablename__] = Article + self.registry[Book.__tablename__] = Book + self.registry[Author.__tablename__] = Author + self.registry[MediaFile.__tablename__] = MediaFile + self.registry[MediaActor.__tablename__] = MediaActor + self.registry[MediaActorFile.__tablename__] = MediaActorFile + self.registry[MediaArticle.__tablename__] = MediaArticle + self.registry[MediaVideo.__tablename__] = MediaVideo + self.registry[MetaDataColumn.__tablename__] = MetaDataColumn + self.registry[MetaDataTable.__tablename__] = MetaDataTable + self.registry[AuthorizationMatrix.__tablename__] = AuthorizationMatrix + self.registry[Token.__tablename__] = Token + self.registry[User.__tablename__] = User + self.registry[Role.__tablename__] = Role + self.registry[ModuleData.__tablename__] = ModuleData + self.registry[MailAccount.__tablename__] = MailAccount + self.registry[Mail.__tablename__] = Mail + + def get_table_names(self) -> list: + result = [] + __session__ = sessionmaker(self.engine) + with __session__() as session: + tables = session.scalars(select(MetaDataTable)).all() + result = [table.table_name for table in tables] + return result + + def get_table_by_name(self, table_name: str) -> dict: + result = {} + __session__ = sessionmaker(self.engine) + _filter = {'table_name': table_name} + with __session__() as session: + table = session.query(MetaDataTable).filter_by(**_filter).one() + result['id'] = table.id + result['table_name'] = table.table_name + return result + + def get_column_meta_data(self, table_name: str, view_only=True) -> dict: + meta_data = {} + order = 0 + __session__ = sessionmaker(self.engine) + columns = list() + table_info = self.get_table_by_name(table_name) + _filters = {'table_id': table_info['id']} + if view_only: + _filters['is_shown'] = True + with __session__() as session: + columns = session.query(MetaDataColumn).filter_by(**_filters).all() + for column in columns: + # self.log.info("get_column_meta_data: %s %s %d", column.column_name, column.column_label, column.column_order) + meta_data[order] = { + ColumnEntry.COLUMN_NAME: column.column_name, + ColumnEntry.COLUMN_LABEL: column.column_label, + ColumnEntry.COLUMN_ORDER: column.column_order, + ColumnEntry.COLUMN_REF_COLUMN: column.ref_column, + ColumnEntry.COLUMN_TYPE: column.column_type + } + order += 1 + return meta_data + + def get_columns(self, table_name: str) -> dict: + columns = {} + __session__ = sessionmaker(self.engine) + table_info = self.get_table_by_name(table_name) + _filters = {'table_id': table_info['id']} + with __session__() as session: + for column in session.query(MetaDataColumn).filter_by(**_filters).all(): + columns[column.column_name] = { + ColumnEntry.COLUMN_ORDER: column.column_order, + ColumnEntry.COLUMN_TYPE: column.column_type + } + return columns + + def get_filters(self, table_name: str) -> dict: + _filter_map = {} + __session__ = sessionmaker(self.engine) + table_info = self.get_table_by_name(table_name) + _filters = {'table_id': table_info['id'], 'show_filter': True} + with __session__() as session: + for column in session.query(MetaDataColumn).filter_by(**_filters).all(): + _filter_map[column.column_name] = { + ColumnEntry.COLUMN_LABEL: column.filter_label, + ColumnEntry.COLUMN_WIDGET: None + } + return _filter_map + + def data(self, table_name: str, columns: dict, filters: dict) -> list: + data = [] + __session__ = sessionmaker(self.engine) + table = self.registry[table_name] + with __session__() as session: + entries = [] + if len(filters) == 0: + entries = session.scalars(select(table)).all() + else: + entries = session.scalars(select(table).filter_by(**filters)).all() + for entry in entries: + # self.log.info("data: %s", entry) + row = [] + for order in columns.keys(): + column_name = columns[order][ColumnEntry.COLUMN_NAME] + ref_column = columns[order][ColumnEntry.COLUMN_REF_COLUMN] + if str(column_name).endswith("_id"): + ref_table = column_name[:-3] + ref = getattr(entry, ref_table) + value = getattr(ref, ref_column) + row.append(value) + else: + row.append(getattr(entry, column_name)) + data.append(row) + # self.log.info("data: %s", data) + return data + + def export_db(self, export_type: ExportType, export_file_name: str) -> dict: + results = {} + db = {} + export_table_list = self.get_table_names() + for table in export_table_list: + columns = self.get_column_meta_data(table, view_only=False) + if table in self.registry: + model = self.registry[table] + else: + self.log.info(f"table {table} is not registered") + continue + __session__ = sessionmaker(self.engine) + with __session__() as session: + rows = session.query(model).all() + entries = [] + for row in rows: + # print(row) + entry = {} + for order in columns: + # print(columns[order]) + column_name = columns[order][ColumnEntry.COLUMN_NAME] + # print(f"get value {column_name} from {row} of table {table}") + try: + value = getattr(row, column_name) + if isinstance(value, datetime): + entry[column_name] = str(value) + else: + entry[column_name] = value + except AttributeError: + pass + entries.append(entry) + db[table] = entries + results[table] = len(entries) + match export_type: + case ExportType.JSON: + json_dump = json.dumps(db, indent=4) + with open(export_file_name, "w") as dump_file: + dump_file.write(json_dump) + case ExportType.YAML: + pass + case ExportType.SQLITE: + pass + self.log.info(f"{len(results)} tables exported") + return results + + def import_db(self, import_file_name: str) -> dict: + result = {} + import_file = Path(import_file_name) + if not import_file.exists(): + self.log.info(f"File {import_file_name} does not exist. Do nothing.") + return result + match import_file.suffix: + case '.json': + print("read json file") + with open(import_file_name, 'r') as json_file: + json_load = json.load(json_file) + for table in json_load: + self.log.info(f"{table}: {len(json_load[table])}") + result[table] = self.import_table(table, json_load[table]) + case '.yml': + print("read yaml file") + case '.yaml': + print("read yaml file") + case '.db': + print("read sqlite file") + return result + + def import_table(self, table_name: str, items:list) -> dict: + result = {} + updated = [] + added = [] + remaining = [] + existing_ids = self.get_ids(table_name) + self.log.info(f"found {len(existing_ids)} existing ids for table {table_name}") + for item in items: + current_id = item['id'] + # print(f"import item: {item}") + found_item = None + __session__ = sessionmaker(self.engine) + with __session__() as session: + found_item = session.get(self.registry[table_name], current_id) + # print(f"found item: {found_item}") + if found_item is not None: + changed = self.update_entry(table_name, current_id, item) + updated.append(item) + if changed: + self.log.info(f"{current_id} has changed") + updated.append(item) + existing_ids.remove(current_id) + else: + try: + self.add_entry(table_name, item) + added.append(item) + except IntegrityError as error: + self.log.info(f"Could not add item, due to: {error.detail}") + if len(existing_ids) > 0: + print(f"remaining items: {existing_ids}") + remaining.extend(existing_ids) + result['updated'] = updated + result['added'] = added + result['remaining'] = remaining + return result + + def get_ids(self, table_name: str) -> list: + existing_ids = [] + __session__ = sessionmaker(self.engine) + with __session__() as session: + items = session.query(self.registry[table_name]).all() + for item in items: + existing_ids.append(getattr(item, 'id')) + return existing_ids + + def add_entry(self, table_name: str, update_item: dict): + self.log.debug(f"add entry to table {table_name} with {update_item}") + __session__ = sessionmaker(self.engine) + with __session__() as session: + add_item = self.registry[table_name]() + for key in update_item.keys(): + update_value = update_item[key] + setattr(add_item, key, update_value) + session.add(add_item) + session.commit() + + def update_entry(self, table_name, current_id, update_item: dict) -> bool: + # self.log.info("update entry to table %s", table_name) + __session__ = sessionmaker(self.engine) + with __session__() as session: + existing_item = session.query(self.registry[table_name]).get(current_id) + changed = False + for key in update_item.keys(): + update_value = update_item[key] + existing_value = getattr(existing_item, key) + if type(existing_value) is not type(update_value): + existing_value = str(existing_value) + if existing_value != update_value: + self.log.info(f"{key} has changed: {existing_value} != {update_value}") + setattr(existing_item, key, update_value) + session.commit() + changed = True + self.log.info(f"update {key} with {update_value}") + return changed + + def add_link(self, link: str) -> dict: + result = {} + __session__ = sessionmaker(self.engine) + with __session__() as session: + media_file = MediaFile() + media_file.id = str(uuid.uuid4()) + media_file.created_date = datetime.now() + media_file.last_modified_date = datetime.now() + media_file.version = 0 + media_file.url = link + media_file.review = 1 + media_file.should_download = 1 + try: + session.add(media_file) + session.commit() + result['added'] = {'url': media_file.url, 'title': media_file.title, 'review': media_file.review, 'download': media_file.should_download} + except IntegrityError as error: + session.rollback() + result['error'] = error.orig + return result + + def update_titles(self) -> dict: + update_list = {} + __session__ = sessionmaker(self.engine) + _filter = { 'review': True} + with __session__() as session: + links = session.query(MediaFile).filter_by(**_filter).all() + for link in links: + url = link.url + if url is None: + continue + link.update_title() + session.commit() + update_list[link.id] = link.title + return update_list + + def get_download_list(self) -> list: + download_list = [] + __session__ = sessionmaker(self.engine) + _filter = { 'should_download': True} + with __session__() as session: + links = session.query(MediaFile).filter_by(**_filter).all() + for link in links: + url = link.url + if url is None: + continue + download_list.append(link.id) + return download_list + + def download_file(self, entry_id: str, download_dir = "/data/media", dl_tool = "yt-dlp") -> str: + __session__ = sessionmaker(self.engine) + with __session__() as session: + link = session.query(MediaFile).get(entry_id) + link.download_file(download_dir, dl_tool) + session.commit() + file_name = link.file_name + return file_name + + def delete_entries(self): + for (table_name, table) in self.registry.items(): + # self.log.info("delete entries from table %s", table_name) + __session__ = sessionmaker(self.engine) + with __session__() as session: + items = session.query(table).all() + for item in items: + session.delete(item) + session.commit() + + def check_files(self): + pass diff --git a/kontor-api/src/schema/media.py b/kontor-api/src/schema/media.py new file mode 100644 index 0000000..d3a9723 --- /dev/null +++ b/kontor-api/src/schema/media.py @@ -0,0 +1,92 @@ +import re +import subprocess +from datetime import datetime +from pathlib import Path + +import requests +from bs4 import BeautifulSoup +from sqlalchemy import Column, String, ForeignKey, Boolean +from sqlalchemy.orm import relationship + +from .base import Base, BaseMixin, BaseVideoMixin + + +class MediaFile(Base, BaseMixin, BaseVideoMixin): + __tablename__ = 'media_file' + media_actor_files = relationship("MediaActorFile") + + def __repr__(self): + return f'MediaFile({self.id} {self.title} {self.title})' + + def __str__(self): + return f'{self.title}({self.id})' + + def update_title(self) -> None: + print(f"update title for {self.url}") + try: + r = requests.get(self.url) + soup = BeautifulSoup(r.content, "html.parser") + title = soup.title.string + self.title = title + self.review = 0 + except: + self.title = None + self.review = 1 + self.last_modified_date = datetime.now() + + def download_file(self, download_dir: str, dl_tool: str): + print(f"download file for {self.url} to {download_dir}") + result = subprocess.run([dl_tool, self.url], cwd=download_dir, capture_output=True, text=True) + if result.returncode == 0: + output = result.stdout + output = re.sub(' +', ' ', output) + lines_list = output.splitlines() + file_name = self.__parse_output__(lines_list) + if file_name is None: + self.review = True + self.should_download = True + self.file_name = None + else: + download_file = Path(file_name) + self.should_download = False + self.file_name = download_file.name + self.cloud_link = str(download_file.absolute()) + self.last_modified_date = datetime.now() + + def __parse_output__(self, lines_list): + self.file_name = None + for line in lines_list: + if 'has already been downloaded' in line: + end_len = len(' has already been downloaded') + self.file_name = line[11:-end_len] + if 'Destination' in line: + line_len = len(line) + start_len = len('[download] Destination: ') + file_len = line_len - start_len + self.file_name = line[-file_len:] + return self.file_name + + +class MediaActor(Base, BaseMixin): + __tablename__ = 'media_actor' + name = Column(String(255)) + media_actor_files = relationship("MediaActorFile") + + +class MediaActorFile(Base, BaseMixin): + __tablename__ = 'media_actor_file' + media_actor_id = Column(String(255), ForeignKey("media_actor.id"), nullable=False) + media_actor = relationship("MediaActor", back_populates="media_actor_files") + media_file_id = Column(String(255), ForeignKey("media_file.id"), nullable=False) + media_file = relationship("MediaFile", back_populates="media_actor_files") + + +class MediaArticle(Base, BaseMixin): + __tablename__ = 'media_article' + review = Column(Boolean) + title = Column(String(255)) + url = Column(String(255), unique=True) + + +class MediaVideo(Base, BaseMixin, BaseVideoMixin): + __tablename__ = 'media_video' diff --git a/kontor-api/src/schema/metadata.py b/kontor-api/src/schema/metadata.py new file mode 100644 index 0000000..9ac5aa4 --- /dev/null +++ b/kontor-api/src/schema/metadata.py @@ -0,0 +1,41 @@ +from sqlalchemy import Column, String, ForeignKey, Integer, Boolean +from sqlalchemy.orm import relationship + +from .base import Base, BaseMixin + + +class MetaDataTable(Base, BaseMixin): + __tablename__ = 'meta_data_table' + table_name = Column(String(255), unique=True) + table_columns = relationship("MetaDataColumn") + + def __repr__(self): + return f'MetaDataTable({self.id} {self.table_name})' + + def __str__(self): + return f'{self.table_name}({self.id})' + + +class MetaDataColumn(Base, BaseMixin): + __tablename__ = 'meta_data_column' + column_name = Column(String(255), nullable=False) + column_sync_name = Column(String(255)) + column_type = Column(String(255)) + column_modifier = Column(String(255), nullable=True) + column_order = Column(Integer) + table_id = Column(String, ForeignKey('meta_data_table.id')) + table = relationship("MetaDataTable", back_populates="table_columns") + column_label = Column(String(255)) + filter_label = Column(String(255)) + is_shown = Column(Boolean) + show_filter = Column(Boolean) + ref_column = Column(String, nullable=True) + + def __repr__(self): + if self.column_name is None: + return f'MetaDataColumn({self.id} {self.table.table_name}.__)' + else: + return f'MetaDataColumn({self.id} {self.table.table_name}.{self.column_name})' + + def __str__(self): + return f'{self.column_name}({self.id})' diff --git a/kontor-api/src/schema/tysc.py b/kontor-api/src/schema/tysc.py new file mode 100644 index 0000000..5ab4baa --- /dev/null +++ b/kontor-api/src/schema/tysc.py @@ -0,0 +1,99 @@ +from sqlalchemy import Column, Integer, String, ForeignKey, UniqueConstraint, Boolean +from sqlalchemy.orm import relationship + +from .base import Base, BaseMixin + + +class Sport(Base, BaseMixin): + __tablename__ = "sport" + __table_args__ = ( + UniqueConstraint("name"), + ) + name = Column(String(255), nullable=False, index=True, unique=True) + teams = relationship("Team") + positions = relationship("FieldPosition") + + +class Team(Base, BaseMixin): + __tablename__ = "team" + name = Column(String(255), nullable=False, index=True, unique=True) + short_name = Column(String(255), nullable=False, ) + sport_id = Column(String, ForeignKey("sport.id"), nullable=False) + sport = relationship("Sport", back_populates="teams") + roosters = relationship("Rooster") + + +class FieldPosition(Base, BaseMixin): + __tablename__ = "field_position" + __table_args__ = ( + UniqueConstraint("name", "sport_id"), + UniqueConstraint("short_name", "sport_id"), + ) + name = Column(String(255), nullable=False, index=True) + short_name = Column(String(255), nullable=False) + sport_id = Column(String, ForeignKey("sport.id"), nullable=False, index=True) + sport = relationship("Sport", back_populates="positions") + roosters = relationship("Rooster") + + +class Player(Base, BaseMixin): + __tablename__ = "player" + __table_args__ = ( + UniqueConstraint("first_name", "last_name"), + ) + first_name = Column(String(255), nullable=False, index=True) + last_name = Column(String(255), nullable=False, index=True) + roosters = relationship("Rooster") + + def get_full_name(self) -> str: + return f"{self.last_name}, {self.first_name}" + + +class Rooster(Base, BaseMixin): + __tablename__ = "rooster" + __table_args__ = ( + UniqueConstraint("year", "team_id", "player_id", "position_id"), + ) + year = Column(Integer) + team_id = Column(String, ForeignKey("team.id"), nullable=False, index=True) + team = relationship("Team", back_populates="roosters") + player_id = Column(String, ForeignKey("player.id"), nullable=False, index=True) + player = relationship("Player", back_populates="roosters") + position_id = Column(String, ForeignKey("field_position.id"), nullable=False, index=True) + position = relationship("FieldPosition", back_populates="roosters") + cards = relationship("Card") + + +class Vendor(Base, BaseMixin): + __tablename__ = "vendor" + name = Column(String(255), nullable=False, unique=True, index=True) + card_sets = relationship("CardSet") + cards = relationship("Card") + + +class CardSet(Base, BaseMixin): + __tablename__ = "card_set" + __table_args__ = ( + UniqueConstraint("name", "vendor_id"), + ) + name = Column(String(255), index=True) + parallel_set = Column(Boolean) + insert_set = Column(Boolean) + vendor_id = Column(String, ForeignKey("vendor.id"), nullable=False, index=True) + vendor = relationship("Vendor", back_populates="card_sets") + cards = relationship("Card") + + +class Card(Base, BaseMixin): + __tablename__ = "card" + __table_args__ = ( + UniqueConstraint("card_number", "year", "vendor_id", "card_set_id"), + ) + card_number = Column(Integer, index=True) + year = Column(Integer, index=True) + card_set_id = Column(String, ForeignKey("card_set.id"), nullable=False) + card_set = relationship("CardSet", back_populates="cards") + rooster_id = Column(String, ForeignKey("rooster.id"), nullable=False) + rooster = relationship("Rooster", back_populates="cards") + vendor_id = Column(String, ForeignKey("vendor.id"), nullable=False) + vendor = relationship("Vendor", back_populates="cards") diff --git a/kontor-api/uv.lock b/kontor-api/uv.lock index a09337f..f156c50 100644 --- a/kontor-api/uv.lock +++ b/kontor-api/uv.lock @@ -300,7 +300,6 @@ dependencies = [ { name = "beautifulsoup4" }, { name = "fastapi", extra = ["standard"] }, { name = "httpx" }, - { name = "kontor-schema" }, { name = "mariadb" }, { name = "pathlib" }, { name = "platformdirs" }, @@ -317,7 +316,6 @@ requires-dist = [ { name = "beautifulsoup4", specifier = ">=4.13.4" }, { name = "fastapi", extras = ["standard"], specifier = ">=0.115.12" }, { name = "httpx", specifier = "==0.24.1" }, - { name = "kontor-schema", directory = "../kontor-schema" }, { name = "mariadb", specifier = ">=1.1.12" }, { name = "pathlib", specifier = ">=1.0.1" }, { name = "platformdirs", specifier = ">=4.3.7" }, @@ -329,23 +327,6 @@ requires-dist = [ { name = "sqlmodel", specifier = ">=0.0.24" }, ] -[[package]] -name = "kontor-schema" -version = "0.1.0" -source = { directory = "../kontor-schema" } -dependencies = [ - { name = "beautifulsoup4" }, - { name = "requests" }, - { name = "sqlalchemy" }, -] - -[package.metadata] -requires-dist = [ - { name = "beautifulsoup4", specifier = ">=4.13.4" }, - { name = "requests", specifier = ">=2.32.3" }, - { name = "sqlalchemy", specifier = ">=2.0.40" }, -] - [[package]] name = "mariadb" version = "1.1.12"