diff --git a/kontor-api/src/db/repository/comic.py b/kontor-api/src/db/repository/comic.py index 53f3a12..a20fccc 100644 --- a/kontor-api/src/db/repository/comic.py +++ b/kontor-api/src/db/repository/comic.py @@ -1,13 +1,14 @@ import uuid from datetime import datetime -from typing import List, Type +from typing import List, Type, AnyStr from sqlalchemy.orm import Session +from src.core.log_conf import logger from src.db.models.comic import Artist, Comic, Issue, WorkType from src.schema.comics.artist import ArtistDetailResponse from src.schema.comics.issue import IssueDetailsResponse -from src.webapps.comic.forms import AddWorktypeForm +from src.schema.comics.worktype import AddWorkType def get_artist_details(artist: Artist) -> ArtistDetailResponse: @@ -42,7 +43,7 @@ def get_issue_details(issue: Issue) -> IssueDetailsResponse: ) return response -def create_new_worktype(work: AddWorktypeForm, db: Session) -> WorkType: +def create_new_worktype(work: AddWorkType, db: Session) -> WorkType: worktype = WorkType() worktype.id = str(uuid.uuid4()) worktype.created_date = datetime.now() @@ -51,5 +52,15 @@ def create_new_worktype(work: AddWorktypeForm, db: Session) -> WorkType: db.add(worktype) db.commit() db.refresh(worktype) - print(worktype) + logger.info(f"create_new_worktype: {worktype}") + return worktype + + +def update_worktype(work: AddWorkType, worktype_id: AnyStr, db: Session) -> WorkType: + logger.info("update worktype") + worktype = db.get(WorkType, worktype_id) + worktype.name = work.worktype + db.add(worktype) + db.commit() + db.refresh(worktype) return worktype diff --git a/kontor-api/src/templates/comic/add_worktype.html b/kontor-api/src/templates/comic/add_worktype.html deleted file mode 100644 index 0055747..0000000 --- a/kontor-api/src/templates/comic/add_worktype.html +++ /dev/null @@ -1,28 +0,0 @@ -{% extends "shared/base.html" %} - - -{% block title %} - Add a Video Link -{% endblock %} - -{% block content %} -
-
-
- {% for error in errors %} -
  • {{error}}
    • - {% endfor %} -
    -
    - -
    -

    Add a WorkType

    -
    -
    - -
    - -
    -
    -
    -{% endblock %} diff --git a/kontor-api/src/templates/comic/worktype_detail.html b/kontor-api/src/templates/comic/worktype_detail.html index 178d65a..a3fa9ed 100644 --- a/kontor-api/src/templates/comic/worktype_detail.html +++ b/kontor-api/src/templates/comic/worktype_detail.html @@ -58,3 +58,15 @@ {% endblock %} + +{% block scripts %} + +{% endblock %} diff --git a/kontor-api/src/templates/components/artist_cards.html b/kontor-api/src/templates/components/artist_cards.html index 53192af..f0f5584 100644 --- a/kontor-api/src/templates/components/artist_cards.html +++ b/kontor-api/src/templates/components/artist_cards.html @@ -1,6 +1,7 @@
    {{obj.name}}
    +

    Link : {{obj.weblink}}

    Read more
    diff --git a/kontor-api/src/webapps/comic/route_comics.py b/kontor-api/src/webapps/comic/route_comics.py index 196bbdc..f099ac7 100644 --- a/kontor-api/src/webapps/comic/route_comics.py +++ b/kontor-api/src/webapps/comic/route_comics.py @@ -10,7 +10,8 @@ from src.db.repository.comics.comic import update_comic from src.schema.comics.comic import ComicSchema from src.webapps.comic.forms.comic import ValidateComicForm -from src.db.repository.comic import create_new_worktype +from src.db.repository.comic import create_new_worktype, update_worktype +from src.main import logger from src.schema.comics.worktype import AddWorkType from src.webapps.comic.forms import AddWorktypeForm diff --git a/kontor-scripts/schema/metadata.py b/kontor-scripts/db/models/metadata.py similarity index 96% rename from kontor-scripts/schema/metadata.py rename to kontor-scripts/db/models/metadata.py index 42dd88b..915c4a9 100644 --- a/kontor-scripts/schema/metadata.py +++ b/kontor-scripts/db/models/metadata.py @@ -1,7 +1,7 @@ from sqlalchemy import Column, String, ForeignKey, Integer, Boolean from sqlalchemy.orm import relationship -from .base import Base, BaseMixin +from db.models.base import Base, BaseMixin class MetaDataTable(Base, BaseMixin): diff --git a/kontor-scripts/db/repository/metadata.py b/kontor-scripts/db/repository/metadata.py new file mode 100644 index 0000000..0e0558b --- /dev/null +++ b/kontor-scripts/db/repository/metadata.py @@ -0,0 +1,28 @@ + +from typing import List +from sqlalchemy.orm import Session + +from db.models.metadata import MetaDataColumn, MetaDataTable +from db.schemas.metadata import MetaDataTableResponse, MetaDataColumnResponse + + +def get_tables(db: Session) -> List[MetaDataTableResponse]: + tables = db.query(MetaDataTable).all() + results: List[MetaDataTableResponse] = [MetaDataTableResponse(id=table.id, name=table.table_name) for table in tables] + return results + + +def get_columns_for_table(db: Session, table: MetaDataTableResponse)-> List[MetaDataColumnResponse]: + columns = db.query(MetaDataColumn).filter_by(table_id = table.id).all() + results: List[MetaDataColumnResponse] = [] + for column in columns: + result: MetaDataColumnResponse = MetaDataColumnResponse( + id=str(column.id), + name=column.column_name, + label=column.column_label, + order=column.column_order, + ref_column=column.ref_column, + column_type=column.column_type) + results.append(result) + return results + diff --git a/kontor-scripts/db/schemas/metadata.py b/kontor-scripts/db/schemas/metadata.py new file mode 100644 index 0000000..8986fa9 --- /dev/null +++ b/kontor-scripts/db/schemas/metadata.py @@ -0,0 +1,15 @@ +from pydantic import BaseModel, PositiveInt + + +class MetaDataTableResponse(BaseModel): + id: str + name: str + + +class MetaDataColumnResponse(BaseModel): + id: str + name: str + label: str + order: PositiveInt + ref_column: str | None + column_type: str diff --git a/kontor-scripts/json_to_postgres.py b/kontor-scripts/json_to_postgres.py index 3aa0738..2616313 100644 --- a/kontor-scripts/json_to_postgres.py +++ b/kontor-scripts/json_to_postgres.py @@ -7,7 +7,6 @@ from typing import Dict, List from config import get_logger, get_database_cursors import json -import psycopg2 from psycopg2.sql import SQL parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) diff --git a/kontor-scripts/schema/__init__.py b/kontor-scripts/schema/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/kontor-scripts/schema/admin.py b/kontor-scripts/schema/admin.py deleted file mode 100644 index 4d57232..0000000 --- a/kontor-scripts/schema/admin.py +++ /dev/null @@ -1,77 +0,0 @@ -from datetime import datetime - -from sqlalchemy import Column, ForeignKey, Integer, String, Boolean -from sqlalchemy.orm import relationship, mapped_column, Mapped - -from .base import Base, BaseMixin - - -class User(Base, BaseMixin): - __tablename__ = 'user' - first_name = Column(String(255)) - last_name = Column(String(255)) - user_name = Column(String(255), nullable=False) - email = Column(String(255)) - password = Column(String(255)) - enabled = Column(BIT(1)) - matrix = relationship("AuthorizationMatrix") - tokens = relationship("Token") - - def get_full_name(self) -> str: - full_name = "" - if self.first_name is not None: - full_name += self.first_name - if self.last_name is not None: - if len(full_name) > 0: - full_name += " " - full_name += self.last_name - return full_name - - -class Token(Base, BaseMixin): - __tablename__ = "token" - token = Column(String(255), nullable=False, unique=True) - name = Column(String(255)) - last_used_date: Mapped[datetime] = mapped_column() - enabled = Column(BIT(1)) - user_id = Column(String(255), ForeignKey("user.id"), nullable=False) - user = relationship("User", back_populates="tokens") - - -class Role(Base, BaseMixin): - __tablename__ = "role" - name = Column(String(255), nullable=False) - matrix = relationship("AuthorizationMatrix") - - -class AuthorizationMatrix(Base, BaseMixin): - __tablename__ = "authorization_matrix" - user_id = Column(String, ForeignKey("user.id"), nullable=False) - user = relationship("User", back_populates="matrix") - role_id = Column(String, ForeignKey("role.id"), nullable=False) - role = relationship("Role", back_populates="matrix") - - -class ModuleData(Base, BaseMixin): - __tablename__ = "module_data" - module_name = Column(String, nullable=False) - import_data = Column(Boolean) - - -class MailAccount(Base, BaseMixin): - __tablename__ = "mail_account" - host = Column(String) - port = Column(Integer) - protocol = Column(String) - user_name = Column(String) - password = Column(String) - start_tls = Column(Boolean) - - -class Mail(Base, BaseMixin): - __tablename__ = "mail" - folder: Mapped[str] = mapped_column() - subject: Mapped[str] = mapped_column() - body: Mapped[str] = mapped_column() - sent_date: Mapped[datetime] = mapped_column() - received_date: Mapped[datetime] = mapped_column() diff --git a/kontor-scripts/schema/base.py b/kontor-scripts/schema/base.py deleted file mode 100644 index 84d00ef..0000000 --- a/kontor-scripts/schema/base.py +++ /dev/null @@ -1,30 +0,0 @@ -import uuid -from datetime import datetime - -from sqlalchemy import func, Column, String, Boolean -from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column - - -class Base(DeclarativeBase): - pass - - -class BaseMixin: - #id = Column(String, primary_key=True, default=uuid.uuid4) - id: Mapped[str] = mapped_column(primary_key=True, default=uuid.uuid4) - # created_date = Column(DateTime) - created_date: Mapped[datetime] = mapped_column(default=func.now()) - # last_modified_date = Column(DateTime) - last_modified_date: Mapped[datetime] = mapped_column(default=func.now()) - # version = Column(Integer) - version: Mapped[int] = mapped_column(default=0) - - -class BaseVideoMixin: - cloud_link = Column(String) - file_name = Column(String) - path = Column(String) - review = Column(Boolean) - title = Column(String) - url = Column(String, unique=True) - should_download = Column(Boolean) diff --git a/kontor-scripts/schema/bookshelf.py b/kontor-scripts/schema/bookshelf.py deleted file mode 100644 index 2830ee2..0000000 --- a/kontor-scripts/schema/bookshelf.py +++ /dev/null @@ -1,50 +0,0 @@ -from sqlalchemy import Column, ForeignKey, Integer, String -from sqlalchemy.orm import relationship - -from .base import Base, BaseMixin - - -class Article(Base, BaseMixin): - __tablename__ = 'article' - title = Column(String, unique=True) - article_authors = relationship("ArticleAuthor") - - -class Author(Base, BaseMixin): - __tablename__ = 'author' - first_name = Column(String) - last_name = Column(String) - article_authors = relationship("ArticleAuthor") - book_authors = relationship("BookAuthor") - - -class BookshelfPublisher(Base, BaseMixin): - __tablename__ = 'bookshelf_publisher' - name = Column(String, unique=True) - books = relationship("Book") - - -class Book(Base, BaseMixin): - __tablename__ = 'book' - isbn = Column(String, unique=True) - title = Column(String) - year = Column(Integer, nullable=False) - publisher_id = Column(String, ForeignKey('bookshelf_publisher.id'), nullable=False) - publisher = relationship('BookshelfPublisher', back_populates="books") - book_authors = relationship("BookAuthor") - - -class ArticleAuthor(Base, BaseMixin): - __tablename__ = 'article_author' - article_id = Column(String, ForeignKey('article.id'), nullable=False) - article = relationship('Article', back_populates="article_authors") - author_id = Column(String, ForeignKey('author.id'), nullable=False) - author = relationship('Author', back_populates="article_authors") - - -class BookAuthor(Base, BaseMixin): - __tablename__ = 'book_author' - author_id = Column(String, ForeignKey('author.id'), nullable=False) - author = relationship('Author', back_populates="book_authors") - book_id = Column(String, ForeignKey('book.id'), nullable=False) - book = relationship('Book', back_populates="book_authors") diff --git a/kontor-scripts/schema/comic.py b/kontor-scripts/schema/comic.py deleted file mode 100644 index 9108605..0000000 --- a/kontor-scripts/schema/comic.py +++ /dev/null @@ -1,99 +0,0 @@ -from sqlalchemy import Column, ForeignKey, Integer, String, Boolean -from sqlalchemy.orm import relationship - -from .base import Base, BaseMixin - - -class Publisher(Base, BaseMixin): - __tablename__ = "publisher" - name = Column(String, unique=True) - comics = relationship("Comic") - - def __repr__(self): - return f'Publisher({self.id} {self.name})' - - def __str__(self): - return self.__repr__() - - -class Comic(Base, BaseMixin): - __tablename__ = 'comic' - title = Column(String, unique=True) - publisher_id = Column(String, ForeignKey('publisher.id'), nullable=False) - publisher = relationship("Publisher", back_populates="comics") - current_order = Column(Boolean) - completed = Column(Boolean) - issues = relationship("Issue") - story_arcs = relationship("StoryArc") - trade_paperbacks = relationship("TradePaperback") - volumes = relationship("Volume") - comic_works = relationship("ComicWork") - - def __repr__(self): - return f'Comic({self.id} {self.version} {self.title} {self.publisher.name})' - - def __str__(self): - return f'{self.title}({self.id})' - - -class Volume(Base, BaseMixin): - __tablename__ = "volume" - name = Column(String, nullable=False) - comic_id = Column(String, ForeignKey("comic.id"), nullable=False) - comic = relationship("Comic", back_populates="volumes") - issues = relationship("Issue") - - -class TradePaperback(Base, BaseMixin): - __tablename__ = "trade_paperback" - name = Column(String, nullable=False) - issue_start = Column(Integer) - issue_end = Column(Integer) - comic_id = Column(String, ForeignKey("comic.id"), nullable=False) - comic = relationship("Comic", back_populates="trade_paperbacks") - - -class StoryArc(Base, BaseMixin): - __tablename__ = "story_arc" - name = Column(String, nullable=False) - comic_id = Column(String, ForeignKey("comic.id"), nullable=False) - comic = relationship("Comic", back_populates="story_arcs") - - -class Issue(Base, BaseMixin): - __tablename__ = "issue" - issue_number = Column(String) - in_stock = Column(Boolean) - is_read = Column(Boolean) - comic_id = Column(String, ForeignKey("comic.id"), nullable=False) - comic = relationship("Comic", back_populates="issues") - volume_id = Column(String, ForeignKey("volume.id"), nullable=True) - volume = relationship("Volume", back_populates="issues") - - -class Artist(Base, BaseMixin): - __tablename__ = "artist" - name = Column(String, nullable=False) - comic_works = relationship("ComicWork") - - -class WorkType(Base, BaseMixin): - __tablename__ = "worktype" - name = Column(String, nullable=False, unique=True) - comic_works = relationship("ComicWork") - - def __repr__(self): - return f'Worktype({self.id} {self.version} {self.name} {len(self.comic_works)})' - - def __str__(self): - return f'{self.name}({self.id})' - - -class ComicWork(Base, BaseMixin): - __tablename__ = "comic_work" - comic_id = Column(String, ForeignKey("comic.id"), nullable=False) - comic = relationship("Comic", back_populates="comic_works") - artist_id = Column(String, ForeignKey("artist.id"), nullable=False) - artist = relationship("Artist", back_populates="comic_works") - work_type_id = Column(String, ForeignKey("worktype.id"), nullable=False) - work_type = relationship("WorkType", back_populates="comic_works") diff --git a/kontor-scripts/schema/database.py b/kontor-scripts/schema/database.py deleted file mode 100644 index 87f9669..0000000 --- a/kontor-scripts/schema/database.py +++ /dev/null @@ -1,399 +0,0 @@ -import json -import uuid -from datetime import datetime -from enum import Enum, auto -from logging import Logger -from pathlib import Path -from typing import Any - -from sqlalchemy import UUID, select -from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import sessionmaker - -from .tysc import Card, CardSet, Rooster, Team, FieldPosition, Player, Vendor, Sport -from .comic import Issue, TradePaperback, StoryArc, Volume, ComicWork, Artist, Comic, Publisher, WorkType -from .bookshelf import ArticleAuthor, BookAuthor, BookshelfPublisher, Article, Book, Author -from .admin import Mail, MailAccount, ModuleData, Role, User, Token, AuthorizationMatrix -from .metadata import MetaDataTable, MetaDataColumn -from .media import MediaVideo, MediaArticle, MediaFile, MediaActor, MediaActorFile - - -class ColumnEntry(Enum): - COLUMN_NAME = 'column' - COLUMN_LABEL = 'label' - COLUMN_ORDER = 'order' - COLUMN_REF_COLUMN = 'ref_column' - COLUMN_TYPE = 'type' - COLUMN_WIDGET = 'widget' - - -class StatusType(Enum): - UNKNOWN = auto() - FILE_NAME = auto() - FILE_ID = auto() - DUPLICATE = auto() - CLOUD_LINK = auto() - CLOUD_LINK_ID = auto() - - -class ExportType(Enum): - JSON = "JSON" - YAML = "YAML" - SQLITE = "SQLite" - - -class KontorDB: - - def __init__(self, db_engine: Any, log: Logger): - self.engine = db_engine - self.registry = {} - self.init_registry() - self.log = log - - def init_registry(self): - self.registry[Card.__tablename__] = Card - self.registry[CardSet.__tablename__] = CardSet - self.registry[Rooster.__tablename__] = Rooster - self.registry[Team.__tablename__] = Team - self.registry[FieldPosition.__tablename__] = FieldPosition - self.registry[Player.__tablename__] = Player - self.registry[Vendor.__tablename__] = Vendor - self.registry[Sport.__tablename__] = Sport - self.registry[Issue.__tablename__] = Issue - self.registry[TradePaperback.__tablename__] = TradePaperback - self.registry[StoryArc.__tablename__] = StoryArc - self.registry[Volume.__tablename__] = Volume - self.registry[ComicWork.__tablename__] = ComicWork - self.registry[Artist.__tablename__] = Artist - self.registry[Comic.__tablename__] = Comic - self.registry[Publisher.__tablename__] = Publisher - self.registry[WorkType.__tablename__] = WorkType - self.registry[ArticleAuthor.__tablename__] = ArticleAuthor - self.registry[BookAuthor.__tablename__] = BookAuthor - self.registry[BookshelfPublisher.__tablename__] = BookshelfPublisher - self.registry[Article.__tablename__] = Article - self.registry[Book.__tablename__] = Book - self.registry[Author.__tablename__] = Author - self.registry[MediaFile.__tablename__] = MediaFile - self.registry[MediaActor.__tablename__] = MediaActor - self.registry[MediaActorFile.__tablename__] = MediaActorFile - self.registry[MediaArticle.__tablename__] = MediaArticle - self.registry[MediaVideo.__tablename__] = MediaVideo - self.registry[MetaDataColumn.__tablename__] = MetaDataColumn - self.registry[MetaDataTable.__tablename__] = MetaDataTable - self.registry[AuthorizationMatrix.__tablename__] = AuthorizationMatrix - self.registry[Token.__tablename__] = Token - self.registry[User.__tablename__] = User - self.registry[Role.__tablename__] = Role - self.registry[ModuleData.__tablename__] = ModuleData - self.registry[MailAccount.__tablename__] = MailAccount - self.registry[Mail.__tablename__] = Mail - - def get_table_names(self) -> list: - result = [] - __session__ = sessionmaker(self.engine) - with __session__() as session: - tables = session.scalars(select(MetaDataTable)).all() - result = [table.table_name for table in tables] - return result - - def get_table_by_name(self, table_name: str) -> dict: - result = {} - __session__ = sessionmaker(self.engine) - _filter = {'table_name': table_name} - with __session__() as session: - table = session.query(MetaDataTable).filter_by(**_filter).one() - result['id'] = table.id - result['table_name'] = table.table_name - return result - - def get_column_meta_data(self, table_name: str, view_only=True) -> dict: - meta_data = {} - order = 0 - __session__ = sessionmaker(self.engine) - columns = list() - table_info = self.get_table_by_name(table_name) - _filters = {'table_id': table_info['id']} - if view_only: - _filters['is_shown'] = True - with __session__() as session: - columns = session.query(MetaDataColumn).filter_by(**_filters).all() - for column in columns: - # self.log.info("get_column_meta_data: %s %s %d", column.column_name, column.column_label, column.column_order) - meta_data[order] = { - ColumnEntry.COLUMN_NAME: column.column_name, - ColumnEntry.COLUMN_LABEL: column.column_label, - ColumnEntry.COLUMN_ORDER: column.column_order, - ColumnEntry.COLUMN_REF_COLUMN: column.ref_column, - ColumnEntry.COLUMN_TYPE: column.column_type - } - order += 1 - return meta_data - - def get_columns(self, table_name: str) -> dict: - columns = {} - __session__ = sessionmaker(self.engine) - table_info = self.get_table_by_name(table_name) - _filters = {'table_id': table_info['id']} - with __session__() as session: - for column in session.query(MetaDataColumn).filter_by(**_filters).all(): - columns[column.column_name] = { - ColumnEntry.COLUMN_ORDER: column.column_order, - ColumnEntry.COLUMN_TYPE: column.column_type - } - return columns - - def get_filters(self, table_name: str) -> dict: - _filter_map = {} - __session__ = sessionmaker(self.engine) - table_info = self.get_table_by_name(table_name) - _filters = {'table_id': table_info['id'], 'show_filter': True} - with __session__() as session: - for column in session.query(MetaDataColumn).filter_by(**_filters).all(): - _filter_map[column.column_name] = { - ColumnEntry.COLUMN_LABEL: column.filter_label, - ColumnEntry.COLUMN_WIDGET: None - } - return _filter_map - - def data(self, table_name: str, columns: dict, filters: dict) -> list: - data = [] - __session__ = sessionmaker(self.engine) - table = self.registry[table_name] - with __session__() as session: - entries = [] - if len(filters) == 0: - entries = session.scalars(select(table)).all() - else: - entries = session.scalars(select(table).filter_by(**filters)).all() - for entry in entries: - # self.log.info("data: %s", entry) - row = [] - for order in columns.keys(): - column_name = columns[order][ColumnEntry.COLUMN_NAME] - ref_column = columns[order][ColumnEntry.COLUMN_REF_COLUMN] - if str(column_name).endswith("_id"): - ref_table = column_name[:-3] - ref = getattr(entry, ref_table) - value = getattr(ref, ref_column) - row.append(value) - else: - row.append(getattr(entry, column_name)) - data.append(row) - # self.log.info("data: %s", data) - return data - - def export_db(self, export_type: ExportType, export_file_name: str) -> dict: - results = {} - db = {} - export_table_list = self.get_table_names() - for table in export_table_list: - columns = self.get_column_meta_data(table, view_only=False) - if table in self.registry: - model = self.registry[table] - else: - self.log.info(f"table {table} is not registered") - continue - __session__ = sessionmaker(self.engine) - with __session__() as session: - rows = session.query(model).all() - entries = [] - for row in rows: - # print(row) - entry = {} - for order in columns: - # print(columns[order]) - column_name = columns[order][ColumnEntry.COLUMN_NAME] - # print(f"get value {column_name} from {row} of table {table}") - try: - value = getattr(row, column_name) - if isinstance(value, datetime): - entry[column_name] = str(value) - else: - entry[column_name] = value - except AttributeError: - pass - entries.append(entry) - db[table] = entries - results[table] = len(entries) - match export_type: - case ExportType.JSON: - json_dump = json.dumps(db, indent=4) - with open(export_file_name, "w") as dump_file: - dump_file.write(json_dump) - case ExportType.YAML: - pass - case ExportType.SQLITE: - pass - self.log.info(f"{len(results)} tables exported") - return results - - def import_db(self, import_file_name: str) -> dict: - result = {} - import_file = Path(import_file_name) - if not import_file.exists(): - self.log.info(f"File {import_file_name} does not exist. Do nothing.") - return result - match import_file.suffix: - case '.json': - print("read json file") - with open(import_file_name, 'r') as json_file: - json_load = json.load(json_file) - for table in json_load: - self.log.info(f"{table}: {len(json_load[table])}") - result[table] = self.import_table(table, json_load[table]) - case '.yml': - print("read yaml file") - case '.yaml': - print("read yaml file") - case '.db': - print("read sqlite file") - return result - - def import_table(self, table_name: str, items: list) -> dict: - result = {} - updated = [] - added = [] - remaining = [] - existing_ids = self.get_ids(table_name) - self.log.info(f"found {len(existing_ids)} existing ids for table {table_name}") - for item in items: - current_id = item['id'] - # print(f"import item: {item}") - found_item = None - __session__ = sessionmaker(self.engine) - with __session__() as session: - found_item = session.get(self.registry[table_name], current_id) - # print(f"found item: {found_item}") - if found_item is not None: - changed = self.update_entry(table_name, current_id, item) - updated.append(item) - if changed: - self.log.info(f"{current_id} has changed") - updated.append(item) - existing_ids.remove(current_id) - else: - try: - self.add_entry(table_name, item) - added.append(item) - except IntegrityError as error: - self.log.info(f"Could not add item, due to: {error.detail}") - if len(existing_ids) > 0: - print(f"remaining items for {table_name}: {existing_ids}") - remaining.extend(existing_ids) - result['updated'] = updated - result['added'] = added - result['remaining'] = remaining - return result - - def get_ids(self, table_name: str) -> list: - existing_ids = [] - __session__ = sessionmaker(self.engine) - with __session__() as session: - items = session.query(self.registry[table_name]).all() - for item in items: - existing_ids.append(getattr(item, 'id')) - return existing_ids - - def add_entry(self, table_name: str, update_item: dict): - self.log.debug(f"add entry to table {table_name} with {update_item}") - __session__ = sessionmaker(self.engine) - with __session__() as session: - add_item = self.registry[table_name]() - for key in update_item.keys(): - update_value = update_item[key] - setattr(add_item, key, update_value) - session.add(add_item) - session.commit() - - def update_entry(self, table_name, current_id, update_item: dict) -> bool: - # self.log.info("update entry to table %s", table_name) - __session__ = sessionmaker(self.engine) - with __session__() as session: - existing_item = session.query(self.registry[table_name]).get(current_id) - changed = False - for key in update_item.keys(): - update_value = update_item[key] - existing_value = getattr(existing_item, key) - if type(existing_value) is not type(update_value): - existing_value = str(existing_value) - if existing_value != update_value: - self.log.info(f"{key} has changed: {existing_value} != {update_value}") - setattr(existing_item, key, update_value) - session.commit() - changed = True - self.log.info(f"update {key} with {update_value}") - return changed - - def add_link(self, link: str) -> dict: - result = {} - __session__ = sessionmaker(self.engine) - with __session__() as session: - media_file = MediaFile() - media_file.id = str(uuid.uuid4()) - media_file.created_date = datetime.now() - media_file.last_modified_date = datetime.now() - media_file.version = 0 - media_file.url = link - media_file.review = 1 - media_file.should_download = 1 - try: - session.add(media_file) - session.commit() - result['added'] = {'url': media_file.url, 'title': media_file.title, 'review': media_file.review, - 'download': media_file.should_download} - except IntegrityError as error: - session.rollback() - result['error'] = error.orig - return result - - def update_titles(self) -> dict: - update_list = {} - __session__ = sessionmaker(self.engine) - _filter = {'review': True} - with __session__() as session: - links = session.query(MediaFile).filter_by(**_filter).all() - self.log.info("%d entries found for updating titles", len(links)) - for link in links: - url = link.url - if url is None: - continue - link.update_title() - session.commit() - update_list[link.id] = link.title - return update_list - - def get_download_list(self) -> list[UUID]: - download_list = [] - __session__ = sessionmaker(self.engine) - _filter = {'should_download': True} - with __session__() as session: - links = session.query(MediaFile).filter_by(**_filter).all() - for link in links: - url = link.url - if url is None: - continue - download_list.append(link.id) - return download_list - - def download_file(self, entry_id: str, download_dir="/data/media", dl_tool="yt-dlp") -> str: - __session__ = sessionmaker(self.engine) - with __session__() as session: - link = session.query(MediaFile).get(entry_id) - link.download_file(download_dir, dl_tool) - session.commit() - file_name = link.file_name - return file_name - - def delete_entries(self): - for (table_name, table) in self.registry.items(): - # self.log.info("delete entries from table %s", table_name) - __session__ = sessionmaker(self.engine) - with __session__() as session: - items = session.query(table).all() - for item in items: - session.delete(item) - session.commit() - - def check_files(self): - pass diff --git a/kontor-scripts/schema/media.py b/kontor-scripts/schema/media.py deleted file mode 100644 index 979e8e1..0000000 --- a/kontor-scripts/schema/media.py +++ /dev/null @@ -1,99 +0,0 @@ -import re -import subprocess -from datetime import datetime -from pathlib import Path - -import requests -from bs4 import BeautifulSoup -from sqlalchemy import Boolean, Column, False_, String, ForeignKey -from sqlalchemy.orm import relationship - -from .base import Base, BaseMixin, BaseVideoMixin - - -class MediaFile(Base, BaseMixin, BaseVideoMixin): - __tablename__ = 'media_file' - media_actor_files = relationship("MediaActorFile") - - def __repr__(self): - return f'MediaFile({self.id} {self.title} {self.title})' - - def __str__(self): - return f'{self.title}({self.id})' - - def update_title(self) -> None: - print(f"update title for {self.url}") - try: - r = requests.get(self.url) - soup = BeautifulSoup(r.content, "html.parser") - title = soup.title.string - self.title = title - self.review = False_ - except: - self.title = None - self.review = True - self.last_modified_date = datetime.now() - - def download_file(self, download_dir: str, dl_tool: str): - print(f"download file for {self.url} to {download_dir}") - result = subprocess.run([dl_tool, self.url], cwd=download_dir, capture_output=True, text=True) - if result.returncode == 0: - output = result.stdout - output = re.sub(' +', ' ', output) - lines_list = output.splitlines() - file_name = self.__parse_output__(lines_list) - if file_name is None: - self.review = True - self.should_download = True - self.file_name = None - else: - download_file = Path(file_name) - self.should_download = False_ - self.file_name = download_file.name - self.cloud_link = str(download_file.absolute()) - self.last_modified_date = datetime.now() - - def __parse_output__(self, lines_list): - self.file_name = None - for line in lines_list: - if 'has already been downloaded' in line: - end_len = len(' has already been downloaded') - self.file_name = line[11:-end_len] - if 'Destination' in line: - line_len = len(line) - start_len = len('[download] Destination: ') - file_len = line_len - start_len - self.file_name = line[-file_len:] - return self.file_name - - -class MediaActor(Base, BaseMixin): - __tablename__ = 'media_actor' - name = Column(String) - media_actor_files = relationship("MediaActorFile") - - -class MediaActorFile(Base, BaseMixin): - __tablename__ = 'media_actor_file' - media_actor_id = Column(String, ForeignKey("media_actor.id"), nullable=False) - media_actor = relationship("MediaActor", back_populates="media_actor_files") - media_file_id = Column(String, ForeignKey("media_file.id"), nullable=True) - media_file = relationship("MediaFile", back_populates="media_actor_files") - - -class MediaArticle(Base, BaseMixin): - __tablename__ = 'media_article' - review = Column(Boolean) - title = Column(String) - url = Column(String, unique=True) - - -class MediaVideo(Base, BaseMixin): - __tablename__ = 'media_video' - cloud_link = Column(String) - file_name = Column(String) - path = Column(String) - review = Column(Boolean) - title = Column(String) - url = Column(String, unique=True) - should_download = Column(Boolean) diff --git a/kontor-scripts/schema/tysc.py b/kontor-scripts/schema/tysc.py deleted file mode 100644 index 1bc500e..0000000 --- a/kontor-scripts/schema/tysc.py +++ /dev/null @@ -1,99 +0,0 @@ -from sqlalchemy import Column, Integer, String, ForeignKey, UniqueConstraint, Boolean -from sqlalchemy.orm import relationship - -from .base import Base, BaseMixin - - -class Sport(Base, BaseMixin): - __tablename__ = "sport" - __table_args__ = ( - UniqueConstraint("name"), - ) - name = Column(String, nullable=False, index=True, unique=True) - teams = relationship("Team") - positions = relationship("FieldPosition") - - -class Team(Base, BaseMixin): - __tablename__ = "team" - name = Column(String, nullable=False, index=True, unique=True) - short_name = Column(String, nullable=False, ) - sport_id = Column(String, ForeignKey("sport.id"), nullable=False) - sport = relationship("Sport", back_populates="teams") - roosters = relationship("Rooster") - - -class FieldPosition(Base, BaseMixin): - __tablename__ = "field_position" - __table_args__ = ( - UniqueConstraint("name", "sport_id"), - UniqueConstraint("short_name", "sport_id"), - ) - name = Column(String, nullable=False, index=True) - short_name = Column(String, nullable=False) - sport_id = Column(String, ForeignKey("sport.id"), nullable=False, index=True) - sport = relationship("Sport", back_populates="positions") - roosters = relationship("Rooster") - - -class Player(Base, BaseMixin): - __tablename__ = "player" - __table_args__ = ( - UniqueConstraint("first_name", "last_name"), - ) - first_name = Column(String, nullable=False, index=True) - last_name = Column(String, nullable=False, index=True) - roosters = relationship("Rooster") - - def get_full_name(self) -> str: - return f"{self.last_name}, {self.first_name}" - - -class Rooster(Base, BaseMixin): - __tablename__ = "rooster" - __table_args__ = ( - UniqueConstraint("year", "team_id", "player_id", "position_id"), - ) - year = Column(Integer) - team_id = Column(String, ForeignKey("team.id"), nullable=False, index=True) - team = relationship("Team", back_populates="roosters") - player_id = Column(String, ForeignKey("player.id"), nullable=False, index=True) - player = relationship("Player", back_populates="roosters") - position_id = Column(String, ForeignKey("field_position.id"), nullable=False, index=True) - position = relationship("FieldPosition", back_populates="roosters") - cards = relationship("Card") - - -class Vendor(Base, BaseMixin): - __tablename__ = "vendor" - name = Column(String, nullable=False, unique=True, index=True) - card_sets = relationship("CardSet") - cards = relationship("Card") - - -class CardSet(Base, BaseMixin): - __tablename__ = "card_set" - __table_args__ = ( - UniqueConstraint("name", "vendor_id"), - ) - name = Column(String, index=True) - parallel_set = Column(Boolean) - insert_set = Column(Boolean) - vendor_id = Column(String, ForeignKey("vendor.id"), nullable=False, index=True) - vendor = relationship("Vendor", back_populates="card_sets") - cards = relationship("Card") - - -class Card(Base, BaseMixin): - __tablename__ = "card" - __table_args__ = ( - UniqueConstraint("card_number", "year", "vendor_id", "card_set_id"), - ) - card_number = Column(Integer, index=True) - year = Column(Integer, index=True) - card_set_id = Column(String, ForeignKey("card_set.id"), nullable=False) - card_set = relationship("CardSet", back_populates="cards") - rooster_id = Column(String, ForeignKey("rooster.id"), nullable=False) - rooster = relationship("Rooster", back_populates="cards") - vendor_id = Column(String, ForeignKey("vendor.id"), nullable=False) - vendor = relationship("Vendor", back_populates="cards")