tpeetz/kontor
1
0
Fork 0
Code Issues 6 Pull Requests 2 Actions Packages Projects Releases 2 Wiki Activity

setup kontor-schema

Browse Source
This commit is contained in:
Thomas Peetz
2025-04-21 21:45:06 +02:00
parent ee78af1abe
commit 6716103d0c
49 changed files with 1029 additions and 897 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/schema/__init__.py
-9
View File
@@ -1,9 +0,0 @@
from .admin import User, Token, Role, AuthorizationMatrix, ModuleData, MailAccount, Mail
from .bookshelf import Article, Book, Author, BookshelfPublisher, ArticleAuthor, BookAuthor
from .comic import Comic, Artist, Publisher, Issue, StoryArc, TradePaperback, Volume, ComicWork, WorkType
from .metadata import MetaDataTable, MetaDataColumn
from .tysc import Card, CardSet, Sport, Team, FieldPosition, Rooster, Player, Vendor
from .media import MediaFile, MediaArticle, MediaVideo
from .base import Base
from .database import KontorDB, ColumnEntry
scripts/schema/database.py
+13 -12
View File
@@ -35,6 +35,7 @@ class StatusType(Enum):
CLOUD_LINK = auto()
CLOUD_LINK_ID = auto()
class ExportType(Enum):
JSON = "JSON"
YAML = "YAML"
@@ -131,7 +132,6 @@ class KontorDB:
def get_columns(self, table_name: str) -> dict:
columns = {}
order = 0
__session__ = sessionmaker(self.engine)
table_info = self.get_table_by_name(table_name)
_filters = {'table_id': table_info['id']}
@@ -183,7 +183,7 @@ class KontorDB:
# self.log.info("data: %s", data)
return data
def export_db(self, export_type: str, export_file_name: str) -> dict:
def export_db(self, export_type: ExportType, export_file_name: str) -> dict:
results = {}
db = {}
export_table_list = self.get_table_names()
@@ -217,14 +217,14 @@ class KontorDB:
db[table] = entries
results[table] = len(entries)
match export_type:
case "JSON":
case ExportType.JSON:
json_dump = json.dumps(db, indent=4)
with open(export_file_name, "w") as dump_file:
dump_file.write(json_dump)
case "YAML":
export_file = Path(export_file_name)
case "SQLite":
export_file = Path(export_file_name)
case ExportType.YAML:
pass
case ExportType.SQLITE:
pass
self.log.info(f"{len(results)} tables exported")
return results
@@ -250,7 +250,7 @@ class KontorDB:
print("read sqlite file")
return result
def import_table(self, table_name: str, items:list) -> dict:
def import_table(self, table_name: str, items: list) -> dict:
result = {}
updated = []
added = []
@@ -340,7 +340,8 @@ class KontorDB:
try:
session.add(media_file)
session.commit()
result['added'] = {'url': media_file.url, 'title': media_file.title, 'review': media_file.review, 'download': media_file.should_download}
result['added'] = {'url': media_file.url, 'title': media_file.title, 'review': media_file.review,
'download': media_file.should_download}
except IntegrityError as error:
session.rollback()
result['error'] = error.orig
@@ -349,7 +350,7 @@ class KontorDB:
def update_titles(self) -> dict:
update_list = {}
__session__ = sessionmaker(self.engine)
_filter = { 'review': True}
_filter = {'review': True}
with __session__() as session:
links = session.query(MediaFile).filter_by(**_filter).all()
self.log.info("%d entries found for updating titles", len(links))
@@ -365,7 +366,7 @@ class KontorDB:
def get_download_list(self) -> list[UUID]:
download_list = []
__session__ = sessionmaker(self.engine)
_filter = { 'should_download': True}
_filter = {'should_download': True}
with __session__() as session:
links = session.query(MediaFile).filter_by(**_filter).all()
for link in links:
@@ -375,7 +376,7 @@ class KontorDB:
download_list.append(link.id)
return download_list
def download_file(self, entry_id: str, download_dir = "/data/media", dl_tool = "yt-dlp") -> str:
def download_file(self, entry_id: str, download_dir="/data/media", dl_tool="yt-dlp") -> str:
__session__ = sessionmaker(self.engine)
with __session__() as session:
link = session.query(MediaFile).get(entry_id)
scripts/schema/media.py
+9 -10
View File
@@ -5,8 +5,7 @@ from pathlib import Path
import requests
from bs4 import BeautifulSoup
from sqlalchemy import Column, String, ForeignKey
from sqlalchemy.dialects.mysql import BIT
from sqlalchemy import Boolean, Column, False_, String, ForeignKey
from sqlalchemy.orm import relationship
from .base import Base, BaseMixin, BaseVideoMixin
@@ -29,10 +28,10 @@ class MediaFile(Base, BaseMixin, BaseVideoMixin):
soup = BeautifulSoup(r.content, "html.parser")
title = soup.title.string
self.title = title
self.review = 0
self.review = False_
except:
self.title = None
self.review = 1
self.review = True
self.last_modified_date = datetime.now()
def download_file(self, download_dir: str, dl_tool: str):
@@ -44,12 +43,12 @@ class MediaFile(Base, BaseMixin, BaseVideoMixin):
lines_list = output.splitlines()
file_name = self.__parse_output__(lines_list)
if file_name is None:
self.review = 1
self.should_download = 1
self.review = True
self.should_download = True
self.file_name = None
else:
download_file = Path(file_name)
self.should_download = 0
self.should_download = False_
self.file_name = download_file.name
self.cloud_link = str(download_file.absolute())
self.last_modified_date = datetime.now()
@@ -84,7 +83,7 @@ class MediaActorFile(Base, BaseMixin):
class MediaArticle(Base, BaseMixin):
__tablename__ = 'media_article'
review = Column(BIT(1))
review = Column(Boolean)
title = Column(String(255))
url = Column(String(255), unique=True)
@@ -94,7 +93,7 @@ class MediaVideo(Base, BaseMixin):
cloud_link = Column(String(255))
file_name = Column(String(255))
path = Column(String(255))
review = Column(BIT(1))
review = Column(Boolean)
title = Column(String(255))
url = Column(String(255), unique=True)
should_download = Column(BIT(1))
should_download = Column(Boolean)