tpeetz/kontor
1
0
Fork 0
Code Issues 6 Pull Requests 2 Actions Packages Projects Releases 2 Wiki Activity

setup kontor-schema

Browse Source
This commit is contained in:
Thomas Peetz
2025-04-21 21:45:06 +02:00
parent 75d3bf6ffb
commit 727e95e732
14 changed files with 50 additions and 542 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kontor-scripts/download.py
+8 -8
View File
@@ -61,19 +61,19 @@ def __parse_output__(lines_list: list[str]) -> str | None:
return file_name
def is_file_downloaded(item: dict, dir: Path) -> FileStatus:
file_name_as_title = f"{item['file_name']}"
file_title = Path(dir, file_name_as_title, ".mp4")
def is_file_downloaded(media_file: dict, dir: Path) -> FileStatus:
file_name_as_title = f"{media_file['file_name']}"
file_title = Path(dir, f"{file_name_as_title}.mp4")
if file_title.exists():
log.info(f"{file_name_as_title} has been downloaded")
item['should_download'] = 0
media_file['should_download'] = False
return FileStatus.DOWNLOADED
file_name_as_id = f"{item['id']}"
file_with_id_as_name = Path(dir, file_name_as_id, ".mp4")
file_name_as_id = f"{media_file['id']}"
file_with_id_as_name = Path(dir, f"{file_name_as_id}.mp4")
if file_with_id_as_name.exists():
log.info(f"{file_with_id_as_name} has been downloaded and renamed")
item['cloud_link'] = file_with_id_as_name
item['should_download'] = 0
media_file['cloud_link'] = file_with_id_as_name
media_file['should_download'] = False
return FileStatus.RENAMED
log.info("could not find file - start download")
return FileStatus.UNKNOWN
kontor-scripts/export.py
+3 -3
View File
@@ -11,12 +11,12 @@ from pathlib import Path
from schema import Base, KontorDB
from config import get_logger
from schema.database import ExportType
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--verbose', '-v', action='count', default=0)
parser.add_argument('--config', '-c', default='kontor-docker')
parser.add_argument('--recreate-db', action='store_true')
parser.add_argument('--file', '-f', default='~/data.json')
parser.add_argument('--file', '-f', default='data.json')
args = parser.parse_args()
@@ -38,5 +38,5 @@ if __name__ == '__main__':
Base.metadata.create_all(bind=engine, checkfirst=True)
__session__ = sessionmaker(bind=engine)
kontor_db = KontorDB(engine, logger)
kontor_db.export_db("JSON", args.file)
kontor_db.export_db(ExportType.JSON, args.file)
logger.info('kontor.export finished')
kontor-scripts/json_to_mariadb.py
-1
View File
@@ -15,7 +15,6 @@ args = parser.parse_args()
def copy_data(mariadb_conn, data_file: Path, log):
mariadb_cursor = mariadb_conn.cursor()
result = {}
import_file = Path(data_file)
if not import_file.exists():
log.info(f"File {data_file} does not exist. Do nothing.")
kontor-scripts/requirements-dev.txt
-8
View File
@@ -1,8 +0,0 @@
-r requirements.txt
pytest
pytest-cov
coverage
twine>=1.11.0
setuptools>=38.6.0
wheel>=0.31.0
kontor-scripts/schema/__init__.py
-9
View File
@@ -1,9 +0,0 @@
from .admin import User, Token, Role, AuthorizationMatrix, ModuleData, MailAccount, Mail
from .bookshelf import Article, Book, Author, BookshelfPublisher, ArticleAuthor, BookAuthor
from .comic import Comic, Artist, Publisher, Issue, StoryArc, TradePaperback, Volume, ComicWork, WorkType
from .metadata import MetaDataTable, MetaDataColumn
from .tysc import Card, CardSet, Sport, Team, FieldPosition, Rooster, Player, Vendor
from .media import MediaFile, MediaArticle, MediaVideo
from .base import Base
from .database import KontorDB, ColumnEntry
kontor-scripts/schema/database.py
+13 -12
View File
@@ -35,6 +35,7 @@ class StatusType(Enum):
CLOUD_LINK = auto()
CLOUD_LINK_ID = auto()
class ExportType(Enum):
JSON = "JSON"
YAML = "YAML"
@@ -131,7 +132,6 @@ class KontorDB:
def get_columns(self, table_name: str) -> dict:
columns = {}
order = 0
__session__ = sessionmaker(self.engine)
table_info = self.get_table_by_name(table_name)
_filters = {'table_id': table_info['id']}
@@ -183,7 +183,7 @@ class KontorDB:
# self.log.info("data: %s", data)
return data
def export_db(self, export_type: str, export_file_name: str) -> dict:
def export_db(self, export_type: ExportType, export_file_name: str) -> dict:
results = {}
db = {}
export_table_list = self.get_table_names()
@@ -217,14 +217,14 @@ class KontorDB:
db[table] = entries
results[table] = len(entries)
match export_type:
case "JSON":
case ExportType.JSON:
json_dump = json.dumps(db, indent=4)
with open(export_file_name, "w") as dump_file:
dump_file.write(json_dump)
case "YAML":
export_file = Path(export_file_name)
case "SQLite":
export_file = Path(export_file_name)
case ExportType.YAML:
pass
case ExportType.SQLITE:
pass
self.log.info(f"{len(results)} tables exported")
return results
@@ -250,7 +250,7 @@ class KontorDB:
print("read sqlite file")
return result
def import_table(self, table_name: str, items:list) -> dict:
def import_table(self, table_name: str, items: list) -> dict:
result = {}
updated = []
added = []
@@ -340,7 +340,8 @@ class KontorDB:
try:
session.add(media_file)
session.commit()
result['added'] = {'url': media_file.url, 'title': media_file.title, 'review': media_file.review, 'download': media_file.should_download}
result['added'] = {'url': media_file.url, 'title': media_file.title, 'review': media_file.review,
'download': media_file.should_download}
except IntegrityError as error:
session.rollback()
result['error'] = error.orig
@@ -349,7 +350,7 @@ class KontorDB:
def update_titles(self) -> dict:
update_list = {}
__session__ = sessionmaker(self.engine)
_filter = { 'review': True}
_filter = {'review': True}
with __session__() as session:
links = session.query(MediaFile).filter_by(**_filter).all()
self.log.info("%d entries found for updating titles", len(links))
@@ -365,7 +366,7 @@ class KontorDB:
def get_download_list(self) -> list[UUID]:
download_list = []
__session__ = sessionmaker(self.engine)
_filter = { 'should_download': True}
_filter = {'should_download': True}
with __session__() as session:
links = session.query(MediaFile).filter_by(**_filter).all()
for link in links:
@@ -375,7 +376,7 @@ class KontorDB:
download_list.append(link.id)
return download_list
def download_file(self, entry_id: str, download_dir = "/data/media", dl_tool = "yt-dlp") -> str:
def download_file(self, entry_id: str, download_dir="/data/media", dl_tool="yt-dlp") -> str:
__session__ = sessionmaker(self.engine)
with __session__() as session:
link = session.query(MediaFile).get(entry_id)
kontor-scripts/schema/media.py
+9 -10
View File
@@ -5,8 +5,7 @@ from pathlib import Path
import requests
from bs4 import BeautifulSoup
from sqlalchemy import Column, String, ForeignKey
from sqlalchemy.dialects.mysql import BIT
from sqlalchemy import Boolean, Column, False_, String, ForeignKey
from sqlalchemy.orm import relationship
from .base import Base, BaseMixin, BaseVideoMixin
@@ -29,10 +28,10 @@ class MediaFile(Base, BaseMixin, BaseVideoMixin):
soup = BeautifulSoup(r.content, "html.parser")
title = soup.title.string
self.title = title
self.review = 0
self.review = False_
except:
self.title = None
self.review = 1
self.review = True
self.last_modified_date = datetime.now()
def download_file(self, download_dir: str, dl_tool: str):
@@ -44,12 +43,12 @@ class MediaFile(Base, BaseMixin, BaseVideoMixin):
lines_list = output.splitlines()
file_name = self.__parse_output__(lines_list)
if file_name is None:
self.review = 1
self.should_download = 1
self.review = True
self.should_download = True
self.file_name = None
else:
download_file = Path(file_name)
self.should_download = 0
self.should_download = False_
self.file_name = download_file.name
self.cloud_link = str(download_file.absolute())
self.last_modified_date = datetime.now()
@@ -84,7 +83,7 @@ class MediaActorFile(Base, BaseMixin):
class MediaArticle(Base, BaseMixin):
__tablename__ = 'media_article'
review = Column(BIT(1))
review = Column(Boolean)
title = Column(String(255))
url = Column(String(255), unique=True)
@@ -94,7 +93,7 @@ class MediaVideo(Base, BaseMixin):
cloud_link = Column(String(255))
file_name = Column(String(255))
path = Column(String(255))
review = Column(BIT(1))
review = Column(Boolean)
title = Column(String(255))
url = Column(String(255), unique=True)
should_download = Column(BIT(1))
should_download = Column(Boolean)