tpeetz/kontor
1
0
Fork 0
Code Issues 6 Pull Requests 2 Actions Packages Projects Releases 2 Wiki Activity

evaluate sqlmodel

Browse Source
This commit is contained in:
Thomas Peetz
2025-04-13 16:16:10 +02:00
parent a43e2c806c
commit b14a267b5b
107 changed files with 2517 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/kontor-cli/kontor/main.py
+1 -1
View File
@@ -15,7 +15,7 @@ CONFIG = init_defaults('kontor', 'mariadb', 'media')
CONFIG['mariadb']['user'] = 'kontor'
CONFIG['mariadb']['password'] = 'kontor'
CONFIG['mariadb']['host'] = '127.0.0.1'
CONFIG['mariadb']['port'] = '3306'
CONFIG['mariadb']['port'] = '3316'
CONFIG['mariadb']['database'] = 'kontor'
CONFIG['media']['yt-dlp'] = '/home/tpeetz/bin/yt-dlp'
CONFIG['media']['dir'] = '/data/media'
python/kontor-gui/.gitignore
-9
View File
@@ -1,9 +0,0 @@
deployment/
venv/
kontor.bin
bin/
include/
lib/
lib64/
lib64
env/
python/kontor-gui/gui/__init__.py
View File
python/kontor-gui/gui/comic_window.py
-66
View File
@@ -1,66 +0,0 @@
from PySide6.QtCore import Signal, QSortFilterProxyModel
from PySide6.QtWidgets import QMainWindow, QWidget, QVBoxLayout, QTabWidget, QMenu, QTableView, QMdiSubWindow, \
QHeaderView
from gui.model_config import KontorModelConfig
from gui.table_model import KontorTableModel
class ComicWindow(QMdiSubWindow):
closed = Signal()
def __init__(self, main_window):
super().__init__()
self.data_views = list()
self._main_window = main_window
self.log = main_window.log
self._init_gui()
self.tick = main_window.tick
self.cross = main_window.cross
def _init_gui(self):
self.setWindowTitle("Comics")
self.setWidget(QWidget())
layout = QVBoxLayout()
self.tabs = QTabWidget()
self.tabs.addTab(self.generate_data_tab("comic"), "Comics")
self.tabs.addTab(self.generate_data_tab("publisher"), "Publisher")
self.tabs.currentChanged.connect(self._tab_changed)
layout.addWidget(self.tabs)
self.setLayout(layout)
self.setWidget(self.tabs)
def closeEvent(self, event):
self.closed.emit()
super().closeEvent(event)
self._main_window.remove_sub_window('comic')
def refresh(self):
# self.log.info("refresh")
self.data_views[self.tabs.currentIndex()].refresh()
def _tab_changed(self, tab_index):
self.data_views[tab_index].refresh()
def update_status(self, message):
self._main_window.update_status(message)
def generate_data_tab(self, table_name):
data_tab = QWidget()
table_config = KontorModelConfig(self._main_window.kontor_db, self, table_name)
model = KontorTableModel(table_config)
layout = QVBoxLayout()
self.data_views.append(model)
data_tab.setLayout(layout)
table_view = QTableView()
header = table_view.horizontalHeader()
header.setSectionResizeMode(QHeaderView.ResizeMode.ResizeToContents)
proxy_model = QSortFilterProxyModel()
proxy_model.setSourceModel(model)
table_view.setSortingEnabled(True)
table_view.setModel(proxy_model)
layout.addLayout(table_config.get_filter_layout())
layout.addWidget(table_view)
model.refresh()
return data_tab
python/kontor-gui/gui/data_view.py
-12
View File
@@ -1,12 +0,0 @@
from abc import ABC, abstractmethod
class DataViewMeta(ABC):
@abstractmethod
def get_header(self):
pass
class ComicView(DataViewMeta):
def get_header(self):
pass
python/kontor-gui/gui/data_view_model.py
-32
View File
@@ -1,32 +0,0 @@
from typing import List
from PySide6.QtCore import QModelIndex, QAbstractTableModel
from PySide6.QtGui import Qt
from gui.data_view import DataViewMeta
class DataViewModel(QAbstractTableModel):
def __init__(self):
super().__init__()
self.main_window = None
self._config = None
self._data = List[DataViewMeta]
def rowCount(self, parent=QModelIndex()):
return len(self._data)
def columnCount(self, parent=QModelIndex()):
return 0
def headerData(self, section, orientation, role=Qt.ItemDataRole.DisplayRole):
return None
def data(self, index, role=Qt.ItemDataRole.DisplayRole):
return None
def setData(self, index, value, role=Qt.ItemDataRole.EditRole):
return False
def flags(self, index):
return None
python/kontor-gui/gui/dialogs.py
-106
View File
@@ -1,106 +0,0 @@
from pathlib import Path
from PySide6.QtWidgets import QDialog, QDialogButtonBox, QVBoxLayout, QLabel, QHBoxLayout, QPushButton, QFileDialog, \
QCheckBox, QComboBox
class ExportKontorDialog(QDialog):
def __init__(self, parent=None, kontor_db=None):
super().__init__(parent)
self.parent = parent
self.kontor_db = kontor_db
self.file_name = "data.json"
self.tables = []
self._table_options = {}
self.export_options = {"JSON": {"ext": ".json"}, "YAML": {"ext": ".yaml"}, "SQLite": {"ext": ".db"}}
self.current_export_type = "JSON"
buttons = (QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
self.buttonBox = QDialogButtonBox(buttons)
self.buttonBox.accepted.connect(self.accept)
self.buttonBox.rejected.connect(self.reject)
layout = QVBoxLayout()
self.label = QLabel()
self.label.setText("Export DB to data.json")
self.combo_box = QComboBox()
self.combo_box.addItems(["JSON", "YAML", "SQLite"])
self.combo_box.currentTextChanged.connect(self.change_export_type)
file_layout = QHBoxLayout()
file_layout.addWidget(self.label)
file_layout.addWidget(self.combo_box)
file_button = QPushButton("Select file")
file_button.clicked.connect(self.select_file)
file_layout.addWidget(file_button)
layout.addLayout(file_layout)
for table_name in self.kontor_db.get_table_names():
check_box = QCheckBox(table_name)
check_box.setChecked(True)
self.tables.append(table_name)
self._table_options[table_name] = check_box
check_box.stateChanged.connect(self.change_selection)
layout.addWidget(check_box)
layout.addWidget(self.buttonBox)
self.setLayout(layout)
def change_selection(self):
self.tables.clear()
for (name, box) in self._table_options.items():
if box.isChecked():
self.tables.append(name)
def change_export_type(self, text):
self.current_export_type = text
self.label.setText(f'Export DB to data.{self.export_options[text]["ext"]}')
def select_file(self):
file_dialog = QFileDialog()
file_dialog.setFileMode(QFileDialog.FileMode.AnyFile)
file_dialog.setDefaultSuffix(self.export_options[self.current_export_type]["ext"])
file_dialog.setNameFilter(f'*{self.export_options[self.current_export_type]["ext"]}')
if file_dialog.exec():
self.file_name = file_dialog.selectedFiles()[0]
export_file = Path(self.file_name)
self.file_name = export_file.with_suffix(self.export_options[self.current_export_type]["ext"])
self.label.setText(f"Export DB to {self.file_name}")
def get_tables_to_export(self) -> list:
return self.tables
class ImportKontorDialog(QDialog):
def __init__(self, parent=None):
super().__init__(parent)
self.file_name = None
QBtn = (QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
self.buttonBox = QDialogButtonBox(QBtn)
self.buttonBox.accepted.connect(self.accept)
self.buttonBox.rejected.connect(self.reject)
self.label = QLabel()
self.label.setText("Import DB from data.json")
layout = QVBoxLayout()
file_layout = QHBoxLayout()
file_layout.addWidget(self.label)
file_button = QPushButton("Select file")
file_button.clicked.connect(self.select_file)
file_layout.addWidget(file_button)
layout.addLayout(file_layout)
layout.addWidget(self.buttonBox)
self.setLayout(layout)
def select_file(self):
file_dialog = QFileDialog()
file_dialog.setFileMode(QFileDialog.FileMode.ExistingFile)
if file_dialog.exec():
self.file_name = file_dialog.selectedFiles()[0]
self.label.setText(f"Import DB from {self.file_name}")
python/kontor-gui/gui/main_window.py
-239
View File
@@ -1,239 +0,0 @@
from PySide6.QtCore import Qt, QThreadPool
from PySide6.QtGui import QAction, QIcon, QGuiApplication
from PySide6.QtWidgets import QWidget, QVBoxLayout, QMenu, QMessageBox, QTabWidget, QTableView, QProgressBar, QMdiArea
from PySide6.QtWidgets import QLabel, QMainWindow
from sqlalchemy import Engine
from kontor_schema import KontorDB
from .comic_window import ComicWindow
from .media_window import MediaWindow
from .meta_data_window import MetaDataWindow
from .progress import ProgressUpdate
from .dialogs import ExportKontorDialog, ImportKontorDialog
from .model_config import KontorModelConfig
from .table_model import KontorTableModel
from .worker import VideoDownloader
class MainWindow(QMainWindow):
def __init__(self, engine: Engine, log):
super().__init__()
self.downloader = None
self.tick = QIcon('res/tick.png')
self.cross = QIcon('res/cross.png')
self.import_icon = QIcon("res/application-import.png")
self.export_icon = QIcon("res/application-export.png")
self.circle_icon = QIcon("res/arrow-circle-double.png")
self.data = []
self.filter = {}
self.kontor_db = KontorDB(engine, log)
self.log = log
self._subwindows = {}
self.media_dir = "/data/media"
self.dl_tool = "yt-dlp"
self._setup_ui()
def _setup_ui(self):
self.setWindowTitle("Kontor")
self.setMinimumSize(1200, 800)
self._create_actions()
self.mdi_area = QMdiArea()
self.setCentralWidget(self.mdi_area)
self.mdi_area.setObjectName('mdi_area')
self.mdi_area.setHorizontalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAsNeeded)
self.mdi_area.setVerticalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAsNeeded)
self._create_menubar()
self._create_toolbars()
self.status_progress = QProgressBar()
self.progress_update = ProgressUpdate(self.status_progress)
self._create_statusbar()
center_point = QGuiApplication.screens()[0].geometry().center()
self.move(center_point - self.frameGeometry().center())
def _create_actions(self):
self.newAction = QAction("&New", self)
self.aboutAction = QAction("&Über...", self)
self.aboutAction.triggered.connect(self.about)
self.showComicWindow = QAction("&Comic Window", self)
self.showComicWindow.triggered.connect(self.show_comic_window)
self.showTyscWindow = QAction("TYSC Window", self)
self.showMediaWindow = QAction("&Media Window", self)
self.showMediaWindow.triggered.connect(self.show_media_window)
self.showMetaDataWindow = QAction("Meta Data Window", self)
self.showMetaDataWindow.triggered.connect(self.show_meta_data_window)
self.importAction = QAction(self.import_icon, "&Import", self)
self.importAction.triggered.connect(self.import_from_file)
self.exportAction = QAction(self.export_icon, "&Export", self)
self.exportAction.triggered.connect(self.export_to_file)
self.refreshAction = QAction(self.circle_icon, "&Refresh", self)
self.refreshAction.triggered.connect(self.refresh)
self.updateTitleAction = QAction("&Update Titles", self)
self.updateTitleAction.triggered.connect(self.update_title)
self.downloadAction = QAction("&Download Videos", self)
self.downloadAction.triggered.connect(self.start_download)
self.checkFileAction = QAction("&Check files", self)
self.checkFileAction.triggered.connect(self.check_files)
self.exitAction = QAction("&Beenden", self)
self.exitAction.setShortcut("Alt+F4")
self.exitAction.triggered.connect(self.close)
def _create_menubar(self):
menu_bar = self.menuBar()
# File menu
file_menu = QMenu("&Datei")
menu_bar.addMenu(file_menu)
file_menu.addAction(self.exitAction)
# Kontor menu
kontor_menu = QMenu("&Kontor")
menu_bar.addMenu(kontor_menu)
kontor_menu.addAction(self.importAction)
kontor_menu.addAction(self.exportAction)
comic_menu = QMenu("&Comic")
comic_menu.addAction(self.showComicWindow)
tysc_menu = QMenu("&TradeYourSportCards")
media_file_menu = QMenu("&MediaFile")
media_file_menu.addAction(self.updateTitleAction)
media_file_menu.addAction(self.downloadAction)
media_file_menu.addAction(self.checkFileAction)
kontor_menu.addMenu(comic_menu)
kontor_menu.addMenu(tysc_menu)
kontor_menu.addMenu(media_file_menu)
window_menu = QMenu("&Window")
layouts_menu = QMenu("&Layouts")
window_menu.addMenu(layouts_menu)
window_menu.addAction(self.showComicWindow)
window_menu.addAction(self.showMediaWindow)
window_menu.addAction(self.showMetaDataWindow)
menu_bar.addMenu(window_menu)
# Help menu
help_menu = QMenu("&Hilfe")
menu_bar.addMenu(help_menu)
help_menu.addAction(self.aboutAction)
def _create_toolbars(self):
# Kontor toolbar
kontor_tool_bar = self.addToolBar("Kontor")
kontor_tool_bar.addAction(self.importAction)
kontor_tool_bar.addAction(self.exportAction)
kontor_tool_bar.addAction(self.refreshAction)
def _create_statusbar(self):
self.statusBar = self.statusBar()
self.statusBar.showMessage("Kontor ready", 6000)
self.status_label = QLabel("")
self.status_progress.setEnabled(False)
self.statusBar.addPermanentWidget(self.status_progress)
def about(self):
QMessageBox.about(self, "Über Kontor", f"Python: 3.11\nKontor: 0.1.0")
def show_comic_window(self):
if 'comic' not in self._subwindows:
comic = ComicWindow(self)
comic.closed.connect(self.sub_window_closed)
self._subwindows['comic'] = comic
self.mdi_area.addSubWindow(comic)
comic.show()
else:
comic = self._subwindows.pop('comic')
comic.close()
self.mdi_area.removeSubWindow(comic)
def show_media_window(self):
if 'media' not in self._subwindows:
media = MediaWindow(self)
media.closed.connect(self.sub_window_closed)
self._subwindows['media'] = media
self.mdi_area.addSubWindow(media)
media.show()
else:
media = self._subwindows.pop('media')
media.close()
self.mdi_area.removeSubWindow(media)
def show_meta_data_window(self):
if 'meta_data' not in self._subwindows:
meta_data = MetaDataWindow(self)
meta_data.closed.connect(self.sub_window_closed)
self._subwindows['meta_data'] = meta_data
self.mdi_area.addSubWindow(meta_data)
meta_data.show()
else:
meta_data = self._subwindows.pop('meta_data')
meta_data.close()
self.mdi_area.removeSubWindow(meta_data)
def remove_sub_window(self, name: str):
self.log.info("remove subwindow %s", name)
if name in self._subwindows:
window = self._subwindows.pop(name)
window.close()
self.mdi_area.removeSubWindow(window)
def sub_window_closed(self):
self.log.info("close subwindow")
def import_from_file(self):
import_dlg = ImportKontorDialog(self)
if import_dlg.exec():
print(f"import DB from file {import_dlg.file_name}")
self.kontor_db.import_db(import_dlg.file_name)
else:
print("do nothing for import")
def export_to_file(self):
export_dlg = ExportKontorDialog(self, self.kontor_db)
if export_dlg.exec():
self.log.info(export_dlg.get_tables_to_export())
self.log.info(f"export DB to {export_dlg.file_name}")
self.statusBar.showMessage(f"export DB to {export_dlg.file_name}", 3000)
self.kontor_db.export_db(export_dlg.current_export_type, export_dlg.file_name)
else:
self.statusBar.showMessage("Export cancelled", 3000)
def update_title(self):
self.log.info("update title for table MediaFile")
self.statusBar.showMessage("update title for table MediaFile", 3000)
self.status_progress.setEnabled(True)
self.kontor_db.update_titles()
self.status_progress.setEnabled(False)
self.refresh()
def start_download(self):
self.status_progress.setEnabled(True)
self.statusBar.showMessage("download videos for table MediaFile", 3000)
self.downloader = VideoDownloader(self.kontor_db, self.log)
self.downloader.setTotalProgress.connect(self.status_progress.setMaximum)
self.downloader.setCurrentProgress.connect(self.downloadProgress)
self.downloader.succeeded.connect(self.downloadSucceeded)
self.downloader.finished.connect(self.downloadFinished)
self.downloader.start()
def downloadProgress(self, value: int):
self.status_progress.setValue(value)
self.refresh()
def downloadSucceeded(self):
self.status_progress.setValue(self.status_progress.maximum())
self.statusBar.showMessage("Download succeeded", 3000)
def downloadFinished(self):
self.status_progress.setEnabled(False)
del self.downloader
def check_files(self):
self.log.info("check files")
self.statusBar.showMessage("check files for table MediaFile", 3000)
self.kontor_db.check_files()
def refresh(self):
self.log.info("refresh")
for (_, window) in self._subwindows.items():
window.refresh()
def update_status(self, message, timeout=3000):
self.statusBar.showMessage(message, timeout=timeout)
python/kontor-gui/gui/media_window.py
-78
View File
@@ -1,78 +0,0 @@
from PySide6.QtCore import Signal, QSortFilterProxyModel
from PySide6.QtWidgets import QMdiSubWindow, QWidget, QVBoxLayout, QTabWidget, QTableView, QHeaderView, QLabel, \
QHBoxLayout, QFormLayout, QLineEdit
from .model_config import KontorModelConfig
from .table_details import KontorTableDetailsView
from .table_model import KontorTableModel
class MediaWindow(QMdiSubWindow):
closed = Signal()
def __init__(self, main_window):
super().__init__()
self.data_views = list()
self._main_window = main_window
self.log = main_window.log
self._init_gui()
self.tick = main_window.tick
self.cross = main_window.cross
def _init_gui(self):
self.setWindowTitle("Media")
self.setWidget(QWidget())
layout = QVBoxLayout()
self.tabs = QTabWidget()
self.tabs.addTab(self.generate_data_tab("media_file"), "Media File")
self.tabs.addTab(self.generate_data_tab("media_video"), "Media Video")
self.tabs.addTab(self.generate_data_tab("media_article"), "Media Article")
self.tabs.addTab(self.generate_data_tab_with_details("media_actor"), "Media Actor")
self.tabs.currentChanged.connect(self._tab_changed)
layout.addWidget(self.tabs)
self.setLayout(layout)
self.setWidget(self.tabs)
def closeEvent(self, event):
self.closed.emit()
super().closeEvent(event)
self._main_window.remove_sub_window('media')
def refresh(self):
self.log.info("MediaWindow.refresh")
self.data_views[self.tabs.currentIndex()].refresh()
def _tab_changed(self, tab_index):
self.data_views[tab_index].refresh()
def update_status(self, message):
self._main_window.update_status(message)
def generate_data_tab(self, table_name):
data_tab = QWidget()
table_config = KontorModelConfig(self._main_window.kontor_db, self, table_name)
model = KontorTableModel(table_config)
layout = QVBoxLayout()
self.data_views.append(model)
data_tab.setLayout(layout)
table_view = QTableView()
proxy_model = QSortFilterProxyModel()
proxy_model.setSourceModel(model)
table_view.setSortingEnabled(True)
table_view.setModel(proxy_model)
layout.addLayout(table_config.get_filter_layout())
layout.addWidget(table_view)
model.refresh()
table_view.resizeColumnToContents(0)
return data_tab
def generate_data_tab_with_details(self, table_name):
table_config = KontorModelConfig(self._main_window.kontor_db, self, table_name)
model = KontorTableModel(table_config)
self.data_views.append(model)
details_view = KontorTableDetailsView(model)
return details_view.data_view
def cell_selected(self, item):
self.log.info(f"Cell {item.row()}:{item.column()} clicked")
python/kontor-gui/gui/meta_data_window.py
-67
View File
@@ -1,67 +0,0 @@
from PySide6.QtCore import Signal, QSortFilterProxyModel
from PySide6.QtWidgets import QMainWindow, QWidget, QVBoxLayout, QTabWidget, QMenu, QTableView, QMdiSubWindow, \
QHeaderView
from gui.model_config import KontorModelConfig
from gui.table_model import KontorTableModel
class MetaDataWindow(QMdiSubWindow):
closed = Signal()
def __init__(self, main_window):
super().__init__()
self.data_views = list()
self._main_window = main_window
self.log = main_window.log
self._init_gui()
self.tick = main_window.tick
self.cross = main_window.cross
def _init_gui(self):
self.setWindowTitle("Meta Data")
self.setWidget(QWidget())
layout = QVBoxLayout()
self.tabs = QTabWidget()
self.tabs.addTab(self.generate_data_tab("module_data"), "Module")
self.tabs.addTab(self.generate_data_tab("meta_data_table"), "Tables")
self.tabs.addTab(self.generate_data_tab("meta_data_column"), "Columns")
self.tabs.currentChanged.connect(self._tab_changed)
layout.addWidget(self.tabs)
self.setLayout(layout)
self.setWidget(self.tabs)
def closeEvent(self, event):
self.closed.emit()
super().closeEvent(event)
self._main_window.remove_sub_window('meta_data')
def refresh(self):
# self.log.info("refresh")
self.data_views[self.tabs.currentIndex()].refresh()
def _tab_changed(self, tab_index):
self.data_views[tab_index].refresh()
def update_status(self, message):
self._main_window.update_status(message)
def generate_data_tab(self, table_name):
data_tab = QWidget()
table_config = KontorModelConfig(self._main_window.kontor_db, self, table_name)
model = KontorTableModel(table_config)
layout = QVBoxLayout()
self.data_views.append(model)
data_tab.setLayout(layout)
table_view = QTableView()
# header = table_view.horizontalHeader()
# header.setSectionResizeMode(QHeaderView.ResizeMode.ResizeToContents)
proxy_model = QSortFilterProxyModel()
proxy_model.setSourceModel(model)
table_view.setSortingEnabled(True)
table_view.setModel(proxy_model)
layout.addLayout(table_config.get_filter_layout())
layout.addWidget(table_view)
model.refresh()
table_view.resizeColumnToContents(0)
return data_tab
python/kontor-gui/gui/model_config.py
-55
View File
@@ -1,55 +0,0 @@
from PySide6.QtWidgets import QHBoxLayout, QCheckBox, QMdiSubWindow
from kontor_schema import KontorDB, ColumnEntry
class KontorModelConfig:
def __init__(self, kontor_db: KontorDB, main_window, table_name: str):
self.header = {}
self.filter = {}
self.main_window = main_window
self.log = main_window.log
self._table_name = table_name
self.kontor_db = kontor_db
self.get_table_config()
def __str__(self):
return f"KontorModelConfig({self._table_name})"
def get_table_config(self):
# self.log.info("get_table_config %s", self)
self.header = self.kontor_db.get_column_meta_data(self._table_name)
self.filter = self.kontor_db.get_filters(self._table_name)
# self.log.info("headers: %s", self.header)
# self.log.info("%s filters: %s", self, self.filter)
def filters(self) -> dict:
# self.log.info("%s filters: %s", self, self.filter)
_filters = {}
# print(self.filter["download"].isChecked())
for column, filter_info in self.filter.items():
# print(column, filter_info)
if filter_info[ColumnEntry.COLUMN_WIDGET].isChecked():
_filters[column] = True
# print(f"{filter_rule=}")
# self.log.info("filters -> %s", _filters)
return _filters
def get_data(self) -> list:
# self.log.info("get_data")
data = self.kontor_db.data(self._table_name, self.header, self.filters())
# self.log.info("get_data: %d %s", len(data), data)
return data
def get_filter_layout(self) -> QHBoxLayout:
# self.log.info("get_filter_layout: %s", self.filter)
filter_layout = QHBoxLayout()
for column, filter_info in self.filter.items():
filter_checkbox = QCheckBox()
filter_checkbox.setText(filter_info[ColumnEntry.COLUMN_LABEL])
filter_checkbox.checkStateChanged.connect(self.main_window.refresh)
self.filter[column][ColumnEntry.COLUMN_WIDGET] = filter_checkbox
filter_layout.addWidget(filter_checkbox)
filter_layout.addStretch()
# self.log.info("get_filter_layout: %s", self.filter)
return filter_layout
python/kontor-gui/gui/progress.py
-18
View File
@@ -1,18 +0,0 @@
from PySide6.QtWidgets import QProgressBar
class ProgressUpdate:
def __init__(self, progress: QProgressBar):
self.start = 0
self.end = 0
self.current = 0
self.progress = progress
def start(self, start_value, end_value):
self.start = start_value
self.end = end_value
self.current = start_value
self.progress.update()
def update(self, current):
self.progress.update()
python/kontor-gui/gui/table_details.py
-60
View File
@@ -1,60 +0,0 @@
from PySide6.QtCore import QSortFilterProxyModel
from PySide6.QtWidgets import QHBoxLayout, QWidget, QTableView, QVBoxLayout, QFormLayout, QLineEdit, QLabel
from .table_model import KontorTableModel
class KontorTableDetailsView:
def __init__(self, table_model: KontorTableModel):
self._data_view: QWidget = QWidget()
self._model = table_model
self.log = table_model.log
self._table_view = QTableView()
self._label = QLabel()
self.init_gui()
def init_gui(self):
self.log.info("KontorTableDetailsView.init_gui()")
layout = QVBoxLayout()
self._data_view.setLayout(layout)
details_layout = QHBoxLayout()
table_with_details = QWidget()
table_with_details.setLayout(details_layout)
self._table_view.setSelectionBehavior(QTableView.SelectionBehavior.SelectRows)
proxy_model = QSortFilterProxyModel()
proxy_model.setSourceModel(self._model)
self._table_view.setSortingEnabled(True)
self._table_view.setModel(proxy_model)
self._table_view.clicked.connect(self.update_details)
self._table_view.activated.connect(self.refresh_details)
layout.addLayout(self._model.config.get_filter_layout())
details_layout.addWidget(self._table_view)
form = QWidget()
form_layout = QFormLayout(form)
form.setLayout(form_layout)
title = QLineEdit(form)
form_layout.addRow("ID", self._label)
form_layout.addRow("Title", title)
# layout.addWidget(table_view)
details_layout.addWidget(form)
layout.addWidget(table_with_details)
self._model.refresh()
self._table_view.resizeColumnToContents(0)
@property
def data_view(self):
return self._data_view
def update_details(self, item):
print(f"Cell {item.row()}-{item.column()} selected")
self.log.info(f"Cell {item.row()}-{item.column()} selected")
self._label.setText(self._model.raw_data()[item.row()][0])
def refresh_details(self):
indexes = self._table_view.selectedIndexes()
for index in indexes:
self.log.info(f"refresh_details: Cell {index.row()}-{index.column()} selected")
python/kontor-gui/gui/table_model.py
-128
View File
@@ -1,128 +0,0 @@
from datetime import datetime
from typing import Any
from PySide6.QtCore import QAbstractTableModel, QModelIndex
from PySide6.QtGui import Qt, QColor
from kontor_schema.database import ColumnEntry
from .model_config import KontorModelConfig
def get_display_value(value: Any, column_config: dict, window) -> str:
if isinstance(value, datetime):
return value.strftime("%Y-%m-%d %M:%M:%S")
if column_config[ColumnEntry.COLUMN_TYPE] == 'BOOLEAN':
if value == 1:
return window.tick
else:
return window.cross
if value is None:
return ""
# window.log.info(f"unknown type: {column_config[ColumnEntry.COLUMN_TYPE]} - {type(value)}")
return str(value)
def get_edit_value(value, column_config, window):
# window.log.info(f"edit value {value}")
return str(value)
def get_decoration_value(value: Any, column_config: dict, window):
if column_config[ColumnEntry.COLUMN_TYPE] == 'BOOLEAN':
if value == 1:
return window.tick
else:
return window.cross
def get_background_value(value: Any, column_config: dict, window):
if value is None:
return QColor('lightgrey')
class KontorTableModel(QAbstractTableModel):
def __init__(self, model_config: KontorModelConfig):
super().__init__()
self._main_window = model_config.main_window
self._config = model_config
self._data = []
self.log = model_config.log
def __str__(self):
return f"KontorTableModel({self._config})"
@property
def config(self):
return self._config
@property
def raw_data(self):
return self._data
def refresh(self):
# self.log.info("refresh")
data = self._config.get_data()
count = 0
# print(data)
if data is not None:
self.beginResetModel()
self._data.clear()
self._data = data
self.endResetModel()
count = len(data)
# print(data)
# print(self._data)
self.layoutChanged.emit()
self._main_window.update_status(f"{count} Einträge geladen")
def rowCount(self, parent=QModelIndex()):
# self.log.info("rowCount %s: %d", self, len(self._data))
# The length of the outer list.
if self._data is None:
return 0
return len(self._data)
def headerData(self, col, orientation, role=Qt.ItemDataRole.DisplayRole):
# self.log.info(f"{self._config.header[col]}")
if orientation == Qt.Orientation.Horizontal and role == Qt.ItemDataRole.DisplayRole:
return self._config.header[col][ColumnEntry.COLUMN_LABEL]
if orientation == Qt.Orientation.Vertical and role == Qt.ItemDataRole.DisplayRole:
return str(col + 1)
def data(self, index, role=Qt.ItemDataRole.DisplayRole):
if self._data is None:
return None
value = self._data[index.row()][index.column()]
# print('{}:: {}:: {}:: {}: {}'.format(index, role, self._config.header[index.column()][ColumnEntry.COLUMN_TYPE], value, type(value)))
match role:
case Qt.ItemDataRole.DisplayRole:
return get_display_value(value, self._config.header[index.column()], self._config.main_window)
case Qt.ItemDataRole.EditRole:
return get_edit_value(value, self._config.header[index.column()], self._config.main_window)
case Qt.ItemDataRole.DecorationRole:
return get_decoration_value(value, self._config.header[index.column()], self._config.main_window)
case Qt.ItemDataRole.BackgroundRole:
return get_background_value(value, self._config.header[index.column()], self._config.main_window)
def columnCount(self, index=QModelIndex()):
# self.log.info("rowCount %s: %d", self, len(self._config.header))
return len(self._config.header)
def setData(self, index, value, role=Qt.ItemDataRole.EditRole) -> bool:
# self._config.log.info(f"{index}: {role}")
if role == Qt.ItemDataRole.EditRole:
self._data[index.row()][index.column()] = value
# self._config.log.info(f"{index.row()}-{index.column()}: {self._data[index.row()][index.column()]}")
self.dataChanged.emit(index, index)
return True
if role == Qt.ItemDataRole.CheckStateRole:
print("role == Qt.ItemDataRole.CheckStateRole")
checked = value == Qt.CheckState.Checked
self._data[index.row()][index.column()] = checked
return False
def flags(self, index):
if self._config.header[index.column()][ColumnEntry.COLUMN_NAME] == 'id':
return Qt.ItemFlag.ItemIsEnabled
return Qt.ItemFlag.ItemIsSelectable | Qt.ItemFlag.ItemIsEnabled | Qt.ItemFlag.ItemIsEditable | Qt.ItemFlag.ItemIsUserTristate
python/kontor-gui/gui/worker.py
-28
View File
@@ -1,28 +0,0 @@
import sys
from PySide6.QtCore import QObject, Signal, QRunnable, Slot, QThread
class VideoDownloader(QThread):
# Signal for the window to establish the maximum value
# of the progress bar.
setTotalProgress = Signal(int)
# Signal to increase the progress.
setCurrentProgress = Signal(int)
# Signal to be emitted when the file has been downloaded successfully.
succeeded = Signal()
def __init__(self, kontor_db, log):
super().__init__()
self.kontor_db = kontor_db
self.log = log
def run(self):
self.log.info("download videos for table MediaFile")
download_entries = self.kontor_db.get_download_list()
self.setTotalProgress.emit(len(download_entries))
for index, entry in enumerate(download_entries):
self.kontor_db.download_file(entry)
self.setCurrentProgress.emit(index)
self.succeeded.emit()
python/kontor-gui/main.py
-42
View File
@@ -1,42 +0,0 @@
"""
PySide6 GUI for Kontor
"""
import sys
import logging.config
from pathlib import Path
from platformdirs import PlatformDirs
from PySide6.QtWidgets import QApplication
import yaml
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from kontor_schema.base import Base
from gui.main_window import MainWindow
if __name__ == '__main__':
app = QApplication(sys.argv)
dirs = PlatformDirs("kontor")
database_config = Path(dirs.user_config_dir, 'database-config.yaml')
with open(database_config, 'rt') as f:
db_config = yaml.safe_load(f.read())
connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format(
db_config['mariadb']['user'],
db_config['mariadb']['password'],
db_config['mariadb']['host'],
db_config['mariadb']['port'],
db_config['mariadb']['database']
))
logging_config = Path(dirs.user_config_dir, 'logging-config.yaml')
with open(logging_config, 'rt') as f:
config = yaml.safe_load(f.read())
logging.config.dictConfig(config)
logger = logging.getLogger('development')
# engine = create_engine(connect_string, echo=True)
engine = create_engine(connect_string)
Base.metadata.create_all(bind=engine, checkfirst=True)
__session__ = sessionmaker(bind=engine)
window = MainWindow(engine, logger)
window.show()
app.exec()
python/kontor-gui/pyvenv.cfg
-5
View File
@@ -1,5 +0,0 @@
home = /usr/bin
include-system-site-packages = false
version = 3.11.2
executable = /usr/bin/python3.11
command = /usr/bin/python -m venv /home/tpeetz/projects/kontor/python/kontor-gui
python/kontor-gui/requirements.txt
-6
View File
@@ -1,6 +0,0 @@
-e ../kontor-schema
platformdirs
pyyaml
PySide6
python/kontor-gui/res/application-export.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 513 B

python/kontor-gui/res/application-import.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 524 B

python/kontor-gui/res/arrow-circle-double.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 836 B

python/kontor-gui/res/cross.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 544 B

python/kontor-gui/res/tick.png
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 634 B

python/kontor-schema/README.md
-4
View File
@@ -1,4 +0,0 @@
# Schema for Kontor DB
This library contains the schema for the Kontor DB.
python/kontor-schema/kontor_schema/__init__.py
-10
View File
@@ -1,10 +0,0 @@
from enum import Enum, auto
from .admin import User, Token, Role, AuthorizationMatrix, ModuleData, MailAccount, Mail
from .bookshelf import Article, Book, Author, BookshelfPublisher, ArticleAuthor, BookAuthor
from .comic import Comic, Artist, Publisher, Issue, StoryArc, TradePaperback, Volume, ComicWork, WorkType
from .metadata import MetaDataTable, MetaDataColumn
from .tysc import Card, CardSet, Sport, Team, FieldPosition, Rooster, Player, Vendor
from .media import MediaFile, MediaArticle, MediaVideo
from .base import Base
from .database import KontorDB, ColumnEntry
python/kontor-schema/kontor_schema/admin.py
-78
View File
@@ -1,78 +0,0 @@
from datetime import datetime
from sqlalchemy import Column, DateTime, ForeignKey, Integer, String
from sqlalchemy.dialects.mysql import BIT
from sqlalchemy.orm import relationship, mapped_column, Mapped
from .base import Base, BaseMixin
class User(Base, BaseMixin):
__tablename__ = 'user'
first_name = Column(String(255))
last_name = Column(String(255))
user_name = Column(String(255), nullable=False)
email = Column(String(255))
password = Column(String(255))
enabled = Column(BIT(1))
matrix = relationship("AuthorizationMatrix")
tokens = relationship("Token")
def get_full_name(self) -> str:
full_name = ""
if self.first_name is not None:
full_name += self.first_name
if self.last_name is not None:
if len(full_name) > 0:
full_name += " "
full_name += self.last_name
return full_name
class Token(Base, BaseMixin):
__tablename__ = "token"
token = Column(String(255), nullable=False, unique=True)
name = Column(String(255))
last_used_date: Mapped[datetime] = mapped_column()
enabled = Column(BIT(1))
user_id = Column(String(255), ForeignKey("user.id"), nullable=False)
user = relationship("User", back_populates="tokens")
class Role(Base, BaseMixin):
__tablename__ = "role"
name = Column(String(255), nullable=False)
matrix = relationship("AuthorizationMatrix")
class AuthorizationMatrix(Base, BaseMixin):
__tablename__ = "authorization_matrix"
user_id = Column(String, ForeignKey("user.id"), nullable=False)
user = relationship("User", back_populates="matrix")
role_id = Column(String, ForeignKey("role.id"), nullable=False)
role = relationship("Role", back_populates="matrix")
class ModuleData(Base, BaseMixin):
__tablename__ = "module_data"
module_name = Column(String(255), nullable=False)
import_data = Column(BIT(1))
class MailAccount(Base, BaseMixin):
__tablename__ = "mail_account"
host = Column(String(255))
port = Column(Integer)
protocol = Column(String(255))
user_name = Column(String(255))
password = Column(String(255))
start_tls = Column(BIT(1))
class Mail(Base, BaseMixin):
__tablename__ = "mail"
folder: Mapped[str] = mapped_column()
subject: Mapped[str] = mapped_column()
body: Mapped[str] = mapped_column()
sent_date: Mapped[datetime] = mapped_column()
received_date: Mapped[datetime] = mapped_column()
python/kontor-schema/kontor_schema/base.py
-31
View File
@@ -1,31 +0,0 @@
import uuid
from datetime import datetime
from sqlalchemy import func, Column, String
from sqlalchemy.dialects.mysql import BIT
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
class Base(DeclarativeBase):
pass
class BaseMixin:
id = Column(String(255), primary_key=True, default=uuid.uuid4())
# id: Mapped[str] = mapped_column(primary_key=True, default=uuid.uuid4())
# created_date = Column(DateTime)
created_date: Mapped[datetime] = mapped_column(default=func.now())
# last_modified_date = Column(DateTime)
last_modified_date: Mapped[datetime] = mapped_column(default=func.now())
# version = Column(Integer)
version: Mapped[int] = mapped_column(default=0)
class BaseVideoMixin:
cloud_link = Column(String(255))
file_name = Column(String(255))
path = Column(String(255))
review = Column(BIT(1))
title = Column(String(255))
url = Column(String(255), unique=True)
should_download = Column(BIT(1))
python/kontor-schema/kontor_schema/bookshelf.py
-51
View File
@@ -1,51 +0,0 @@
from sqlalchemy import Column, DateTime, ForeignKey, Integer, String
from sqlalchemy.dialects.mysql import BIT
from sqlalchemy.orm import relationship
from .base import Base, BaseMixin
class Article(Base, BaseMixin):
__tablename__ = 'article'
title = Column(String(length=255), unique=True)
article_authors = relationship("ArticleAuthor")
class Author(Base, BaseMixin):
__tablename__ = 'author'
first_name = Column(String(255))
last_name = Column(String(255))
article_authors = relationship("ArticleAuthor")
book_authors = relationship("BookAuthor")
class BookshelfPublisher(Base, BaseMixin):
__tablename__ = 'bookshelf_publisher'
name = Column(String(length=255), unique=True)
books = relationship("Book")
class Book(Base, BaseMixin):
__tablename__ = 'book'
isbn = Column(String(255), unique=True)
title = Column(String(255))
year = Column(Integer, nullable=False)
publisher_id = Column(String, ForeignKey('bookshelf_publisher.id'), nullable=False)
publisher = relationship('BookshelfPublisher', back_populates="books")
book_authors = relationship("BookAuthor")
class ArticleAuthor(Base, BaseMixin):
__tablename__ = 'article_author'
article_id = Column(String, ForeignKey('article.id'), nullable=False)
article = relationship('Article', back_populates="article_authors")
author_id = Column(String, ForeignKey('author.id'), nullable=False)
author = relationship('Author', back_populates="article_authors")
class BookAuthor(Base, BaseMixin):
__tablename__ = 'book_author'
author_id = Column(String, ForeignKey('author.id'), nullable=False)
author = relationship('Author', back_populates="book_authors")
book_id = Column(String, ForeignKey('book.id'), nullable=False)
book = relationship('Book', back_populates="book_authors")
python/kontor-schema/kontor_schema/comic.py
-100
View File
@@ -1,100 +0,0 @@
from sqlalchemy import Column, DateTime, ForeignKey, Integer, String
from sqlalchemy.dialects.mysql import BIT
from sqlalchemy.orm import relationship
from .base import Base, BaseMixin
class Publisher(Base, BaseMixin):
__tablename__ = "publisher"
name = Column(String(length=255), unique=True)
comics = relationship("Comic")
def __repr__(self):
return f'Publisher({self.id} {self.name})'
def __str__(self):
return self.__repr__()
class Comic(Base, BaseMixin):
__tablename__ = 'comic'
title = Column(String(length=255), unique=True)
publisher_id = Column(String, ForeignKey('publisher.id'), nullable=False)
publisher = relationship("Publisher", back_populates="comics")
current_order = Column(BIT(1))
completed = Column(BIT(1))
issues = relationship("Issue")
story_arcs = relationship("StoryArc")
trade_paperbacks = relationship("TradePaperback")
volumes = relationship("Volume")
comic_works = relationship("ComicWork")
def __repr__(self):
return f'Comic({self.id} {self.version} {self.title} {self.publisher.name})'
def __str__(self):
return f'{self.title}({self.id})'
class Volume(Base, BaseMixin):
__tablename__ = "volume"
name = Column(String(length=255), nullable=False)
comic_id = Column(String, ForeignKey("comic.id"), nullable=False)
comic = relationship("Comic", back_populates="volumes")
issues = relationship("Issue")
class TradePaperback(Base, BaseMixin):
__tablename__ = "trade_paperback"
name = Column(String(length=255), nullable=False)
issue_start = Column(Integer)
issue_end = Column(Integer)
comic_id = Column(String, ForeignKey("comic.id"), nullable=False)
comic = relationship("Comic", back_populates="trade_paperbacks")
class StoryArc(Base, BaseMixin):
__tablename__ = "story_arc"
name = Column(String(length=255), nullable=False)
comic_id = Column(String, ForeignKey("comic.id"), nullable=False)
comic = relationship("Comic", back_populates="story_arcs")
class Issue(Base, BaseMixin):
__tablename__ = "issue"
issue_number = Column(String(255))
in_stock = Column(BIT(1))
is_read = Column(BIT(1))
comic_id = Column(String, ForeignKey("comic.id"), nullable=False)
comic = relationship("Comic", back_populates="issues")
volume_id = Column(String, ForeignKey("volume.id"), nullable=True)
volume = relationship("Volume", back_populates="issues")
class Artist(Base, BaseMixin):
__tablename__ = "artist"
name = Column(String(length=255), nullable=False)
comic_works = relationship("ComicWork")
class WorkType(Base, BaseMixin):
__tablename__ = "worktype"
name = Column(String(length=255), nullable=False, unique=True)
comic_works = relationship("ComicWork")
def __repr__(self):
return f'Worktype({self.id} {self.version} {self.name} {len(self.comic_works)})'
def __str__(self):
return f'{self.name}({self.id})'
class ComicWork(Base, BaseMixin):
__tablename__ = "comic_work"
comic_id = Column(String, ForeignKey("comic.id"), nullable=False)
comic = relationship("Comic", back_populates="comic_works")
artist_id = Column(String, ForeignKey("artist.id"), nullable=False)
artist = relationship("Artist", back_populates="comic_works")
work_type_id = Column(String, ForeignKey("worktype.id"), nullable=False)
work_type = relationship("WorkType", back_populates="comic_works")
python/kontor-schema/kontor_schema/database.py
-391
View File
@@ -1,391 +0,0 @@
import json
import uuid
from datetime import datetime
from enum import Enum, auto
from logging import Logger
from pathlib import Path
from sqlalchemy import Engine, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import sessionmaker
from .tysc import Card, CardSet, Rooster, Team, FieldPosition, Player, Vendor, Sport
from .comic import Issue, TradePaperback, StoryArc, Volume, ComicWork, Artist, Comic, Publisher, WorkType
from .bookshelf import ArticleAuthor, BookAuthor, BookshelfPublisher, Article, Book, Author
from .admin import Mail, MailAccount, ModuleData, Role, User, Token, AuthorizationMatrix
from .metadata import MetaDataTable, MetaDataColumn
from .media import MediaVideo, MediaArticle, MediaFile, MediaActor, MediaActorFile
class ColumnEntry(Enum):
COLUMN_NAME = 'column'
COLUMN_LABEL = 'label'
COLUMN_ORDER = 'order'
COLUMN_REF_COLUMN = 'ref_column'
COLUMN_TYPE = 'type'
COLUMN_WIDGET = 'widget'
class StatusType(Enum):
UNKNOWN = auto()
FILE_NAME = auto()
FILE_ID = auto()
DUPLICATE = auto()
CLOUD_LINK = auto()
CLOUD_LINK_ID = auto()
class KontorDB:
def __init__(self, db_engine: Engine, log: Logger):
self.engine = db_engine
self.registry = {}
self.init_registry()
self.log = log
def init_registry(self):
self.registry[Card.__tablename__] = Card
self.registry[CardSet.__tablename__] = CardSet
self.registry[Rooster.__tablename__] = Rooster
self.registry[Team.__tablename__] = Team
self.registry[FieldPosition.__tablename__] = FieldPosition
self.registry[Player.__tablename__] = Player
self.registry[Vendor.__tablename__] = Vendor
self.registry[Sport.__tablename__] = Sport
self.registry[Issue.__tablename__] = Issue
self.registry[TradePaperback.__tablename__] = TradePaperback
self.registry[StoryArc.__tablename__] = StoryArc
self.registry[Volume.__tablename__] = Volume
self.registry[ComicWork.__tablename__] = ComicWork
self.registry[Artist.__tablename__] = Artist
self.registry[Comic.__tablename__] = Comic
self.registry[Publisher.__tablename__] = Publisher
self.registry[WorkType.__tablename__] = WorkType
self.registry[ArticleAuthor.__tablename__] = ArticleAuthor
self.registry[BookAuthor.__tablename__] = BookAuthor
self.registry[BookshelfPublisher.__tablename__] = BookshelfPublisher
self.registry[Article.__tablename__] = Article
self.registry[Book.__tablename__] = Book
self.registry[Author.__tablename__] = Author
self.registry[MediaFile.__tablename__] = MediaFile
self.registry[MediaActor.__tablename__] = MediaActor
self.registry[MediaActorFile.__tablename__] = MediaActorFile
self.registry[MediaArticle.__tablename__] = MediaArticle
self.registry[MediaVideo.__tablename__] = MediaVideo
self.registry[MetaDataColumn.__tablename__] = MetaDataColumn
self.registry[MetaDataTable.__tablename__] = MetaDataTable
self.registry[AuthorizationMatrix.__tablename__] = AuthorizationMatrix
self.registry[Token.__tablename__] = Token
self.registry[User.__tablename__] = User
self.registry[Role.__tablename__] = Role
self.registry[ModuleData.__tablename__] = ModuleData
self.registry[MailAccount.__tablename__] = MailAccount
self.registry[Mail.__tablename__] = Mail
def get_table_names(self) -> list:
result = []
__session__ = sessionmaker(self.engine)
with __session__() as session:
tables = session.scalars(select(MetaDataTable)).all()
result = [table.table_name for table in tables]
return result
def get_table_by_name(self, table_name: str) -> dict:
result = {}
__session__ = sessionmaker(self.engine)
_filter = {'table_name': table_name}
with __session__() as session:
table = session.query(MetaDataTable).filter_by(**_filter).one()
result['id'] = table.id
result['table_name'] = table.table_name
return result
def get_column_meta_data(self, table_name: str, view_only=True) -> dict:
meta_data = {}
order = 0
__session__ = sessionmaker(self.engine)
columns = list()
table_info = self.get_table_by_name(table_name)
_filters = {'table_id': table_info['id']}
if view_only:
_filters['is_shown'] = True
with __session__() as session:
columns = session.query(MetaDataColumn).filter_by(**_filters).all()
for column in columns:
# self.log.info("get_column_meta_data: %s %s %d", column.column_name, column.column_label, column.column_order)
meta_data[order] = {
ColumnEntry.COLUMN_NAME: column.column_name,
ColumnEntry.COLUMN_LABEL: column.column_label,
ColumnEntry.COLUMN_ORDER: column.column_order,
ColumnEntry.COLUMN_REF_COLUMN: column.ref_column,
ColumnEntry.COLUMN_TYPE: column.column_type
}
order += 1
return meta_data
def get_columns(self, table_name: str) -> dict:
columns = {}
order = 0
__session__ = sessionmaker(self.engine)
table_info = self.get_table_by_name(table_name)
_filters = {'table_id': table_info['id']}
with __session__() as session:
for column in session.query(MetaDataColumn).filter_by(**_filters).all():
columns[column.column_name] = {
ColumnEntry.COLUMN_ORDER: column.column_order,
ColumnEntry.COLUMN_TYPE: column.column_type
}
return columns
def get_filters(self, table_name: str) -> dict:
_filter_map = {}
__session__ = sessionmaker(self.engine)
table_info = self.get_table_by_name(table_name)
_filters = {'table_id': table_info['id'], 'show_filter': True}
with __session__() as session:
for column in session.query(MetaDataColumn).filter_by(**_filters).all():
_filter_map[column.column_name] = {
ColumnEntry.COLUMN_LABEL: column.filter_label,
ColumnEntry.COLUMN_WIDGET: None
}
return _filter_map
def data(self, table_name: str, columns: dict, filters: dict) -> list:
data = []
__session__ = sessionmaker(self.engine)
table = self.registry[table_name]
with __session__() as session:
entries = []
if len(filters) == 0:
entries = session.scalars(select(table)).all()
else:
entries = session.scalars(select(table).filter_by(**filters)).all()
for entry in entries:
# self.log.info("data: %s", entry)
row = []
for order in columns.keys():
column_name = columns[order][ColumnEntry.COLUMN_NAME]
ref_column = columns[order][ColumnEntry.COLUMN_REF_COLUMN]
if str(column_name).endswith("_id"):
ref_table = column_name[:-3]
ref = getattr(entry, ref_table)
value = getattr(ref, ref_column)
row.append(value)
else:
row.append(getattr(entry, column_name))
data.append(row)
# self.log.info("data: %s", data)
return data
def export_db(self, export_type: str, export_file_name: str) -> dict:
results = {}
db = {}
export_table_list = self.get_table_names()
for table in export_table_list:
columns = self.get_column_meta_data(table, view_only=False)
if table in self.registry:
model = self.registry[table]
else:
self.log.info(f"table {table} is not registered")
continue
__session__ = sessionmaker(self.engine)
with __session__() as session:
rows = session.query(model).all()
entries = []
for row in rows:
# print(row)
entry = {}
for order in columns:
# print(columns[order])
column_name = columns[order][ColumnEntry.COLUMN_NAME]
# print(f"get value {column_name} from {row} of table {table}")
try:
value = getattr(row, column_name)
if isinstance(value, datetime):
entry[column_name] = str(value)
else:
entry[column_name] = value
except AttributeError:
pass
entries.append(entry)
db[table] = entries
results[table] = len(entries)
match export_type:
case "JSON":
json_dump = json.dumps(db, indent=4)
with open(export_file_name, "w") as dump_file:
dump_file.write(json_dump)
case "YAML":
export_file = Path(export_file_name)
case "SQLite":
export_file = Path(export_file_name)
self.log.info(f"{len(results)} tables exported")
return results
def import_db(self, import_file_name: str) -> dict:
result = {}
import_file = Path(import_file_name)
if not import_file.exists():
self.log.info(f"File {import_file_name} does not exist. Do nothing.")
return result
match import_file.suffix:
case '.json':
print("read json file")
with open(import_file_name, 'r') as json_file:
json_load = json.load(json_file)
for table in json_load:
self.log.info(f"{table}: {len(json_load[table])}")
result[table] = self.import_table(table, json_load[table])
case '.yml':
print("read yaml file")
case '.yaml':
print("read yaml file")
case '.db':
print("read sqlite file")
return result
def import_table(self, table_name: str, items:list) -> dict:
result = {}
updated = []
added = []
remaining = []
existing_ids = self.get_ids(table_name)
self.log.info(f"found {len(existing_ids)} existing ids for table {table_name}")
for item in items:
current_id = item['id']
# print(f"import item: {item}")
found_item = None
__session__ = sessionmaker(self.engine)
with __session__() as session:
found_item = session.get(self.registry[table_name], current_id)
# print(f"found item: {found_item}")
if found_item is not None:
changed = self.update_entry(table_name, current_id, item)
updated.append(item)
if changed:
self.log.info(f"{current_id} has changed")
updated.append(item)
existing_ids.remove(current_id)
else:
try:
self.add_entry(table_name, item)
added.append(item)
except IntegrityError as error:
self.log.info(f"Could not add item, due to: {error.detail}")
if len(existing_ids) > 0:
print(f"remaining items: {existing_ids}")
remaining.extend(existing_ids)
result['updated'] = updated
result['added'] = added
result['remaining'] = remaining
return result
def get_ids(self, table_name: str) -> list:
existing_ids = []
__session__ = sessionmaker(self.engine)
with __session__() as session:
items = session.query(self.registry[table_name]).all()
for item in items:
existing_ids.append(getattr(item, 'id'))
return existing_ids
def add_entry(self, table_name: str, update_item: dict):
self.log.debug(f"add entry to table {table_name} with {update_item}")
__session__ = sessionmaker(self.engine)
with __session__() as session:
add_item = self.registry[table_name]()
for key in update_item.keys():
update_value = update_item[key]
setattr(add_item, key, update_value)
session.add(add_item)
session.commit()
def update_entry(self, table_name, current_id, update_item: dict) -> bool:
# self.log.info("update entry to table %s", table_name)
__session__ = sessionmaker(self.engine)
with __session__() as session:
existing_item = session.query(self.registry[table_name]).get(current_id)
changed = False
for key in update_item.keys():
update_value = update_item[key]
existing_value = getattr(existing_item, key)
if type(existing_value) is not type(update_value):
existing_value = str(existing_value)
if existing_value != update_value:
self.log.info(f"{key} has changed: {existing_value} != {update_value}")
setattr(existing_item, key, update_value)
session.commit()
changed = True
self.log.info(f"update {key} with {update_value}")
return changed
def add_link(self, link: str) -> dict:
result = {}
__session__ = sessionmaker(self.engine)
with __session__() as session:
media_file = MediaFile()
media_file.id = str(uuid.uuid4())
media_file.created_date = datetime.now()
media_file.last_modified_date = datetime.now()
media_file.version = 0
media_file.url = link
media_file.review = 1
media_file.should_download = 1
try:
session.add(media_file)
session.commit()
result['added'] = {'url': media_file.url, 'title': media_file.title, 'review': media_file.review, 'download': media_file.should_download}
except IntegrityError as error:
session.rollback()
result['error'] = error.orig
return result
def update_titles(self) -> dict:
update_list = {}
__session__ = sessionmaker(self.engine)
_filter = { 'review': True}
with __session__() as session:
links = session.query(MediaFile).filter_by(**_filter).all()
for link in links:
url = link.url
if url is None:
continue
link.update_title()
session.commit()
update_list[link.id] = link.title
return update_list
def get_download_list(self) -> list:
download_list = []
__session__ = sessionmaker(self.engine)
_filter = { 'should_download': True}
with __session__() as session:
links = session.query(MediaFile).filter_by(**_filter).all()
for link in links:
url = link.url
if url is None:
continue
download_list.append(link.id)
return download_list
def download_file(self, entry_id: str, download_dir = "/data/media", dl_tool = "yt-dlp") -> str:
__session__ = sessionmaker(self.engine)
with __session__() as session:
link = session.query(MediaFile).get(entry_id)
link.download_file(download_dir, dl_tool)
session.commit()
file_name = link.file_name
return file_name
def delete_entries(self):
for (table_name, table) in self.registry.items():
# self.log.info("delete entries from table %s", table_name)
__session__ = sessionmaker(self.engine)
with __session__() as session:
items = session.query(table).all()
for item in items:
session.delete(item)
session.commit()
def check_files(self):
pass
python/kontor-schema/kontor_schema/media.py
-100
View File
@@ -1,100 +0,0 @@
import re
import subprocess
from datetime import datetime
from pathlib import Path
import requests
from bs4 import BeautifulSoup
from sqlalchemy import Column, DateTime, Integer, String, ForeignKey
from sqlalchemy.dialects.mysql import BIT
from sqlalchemy.orm import relationship
from .base import Base, BaseMixin, BaseVideoMixin
class MediaFile(Base, BaseMixin, BaseVideoMixin):
__tablename__ = 'media_file'
media_actor_files = relationship("MediaActorFile")
def __repr__(self):
return f'MediaFile({self.id} {self.title} {self.title})'
def __str__(self):
return f'{self.title}({self.id})'
def update_title(self) -> None:
print(f"update title for {self.url}")
try:
r = requests.get(self.url)
soup = BeautifulSoup(r.content, "html.parser")
title = soup.title.string
self.title = title
self.review = 0
except:
self.title = None
self.review = 1
self.last_modified_date = datetime.now()
def download_file(self, download_dir: str, dl_tool: str):
print(f"download file for {self.url} to {download_dir}")
result = subprocess.run([dl_tool, self.url], cwd=download_dir, capture_output=True, text=True)
if result.returncode == 0:
output = result.stdout
output = re.sub(' +', ' ', output)
lines_list = output.splitlines()
file_name = self.__parse_output__(lines_list)
if file_name is None:
self.review = 1
self.should_download = 1
self.file_name = None
else:
download_file = Path(file_name)
self.should_download = 0
self.file_name = download_file.name
self.cloud_link = str(download_file.absolute())
self.last_modified_date = datetime.now()
def __parse_output__(self, lines_list):
self.file_name = None
for line in lines_list:
if 'has already been downloaded' in line:
end_len = len(' has already been downloaded')
self.file_name = line[11:-end_len]
if 'Destination' in line:
line_len = len(line)
start_len = len('[download] Destination: ')
file_len = line_len - start_len
self.file_name = line[-file_len:]
return self.file_name
class MediaActor(Base, BaseMixin):
__tablename__ = 'media_actor'
name = Column(String(255))
media_actor_files = relationship("MediaActorFile")
class MediaActorFile(Base, BaseMixin):
__tablename__ = 'media_actor_file'
media_actor_id = Column(String(255), ForeignKey("media_actor.id"), nullable=False)
media_actor = relationship("MediaActor", back_populates="media_actor_files")
media_file_id = Column(String(255), ForeignKey("media_file.id"), nullable=False)
media_file = relationship("MediaFile", back_populates="media_actor_files")
class MediaArticle(Base, BaseMixin):
__tablename__ = 'media_article'
review = Column(BIT(1))
title = Column(String(255))
url = Column(String(255), unique=True)
class MediaVideo(Base, BaseMixin):
__tablename__ = 'media_video'
cloud_link = Column(String(255))
file_name = Column(String(255))
path = Column(String(255))
review = Column(BIT(1))
title = Column(String(255))
url = Column(String(255), unique=True)
should_download = Column(BIT(1))
python/kontor-schema/kontor_schema/metadata.py
-42
View File
@@ -1,42 +0,0 @@
from sqlalchemy import Column, String, ForeignKey, DateTime, Integer, Boolean
from sqlalchemy.dialects.mysql import BIT
from sqlalchemy.orm import relationship
from .base import Base, BaseMixin
class MetaDataTable(Base, BaseMixin):
__tablename__ = 'meta_data_table'
table_name = Column(String(255), unique=True)
table_columns = relationship("MetaDataColumn")
def __repr__(self):
return f'MetaDataTable({self.id} {self.table_name})'
def __str__(self):
return f'{self.table_name}({self.id})'
class MetaDataColumn(Base, BaseMixin):
__tablename__ = 'meta_data_column'
column_name = Column(String(255), nullable=False)
column_sync_name = Column(String(255))
column_type = Column(String(255))
column_modifier = Column(String(255), nullable=True)
column_order = Column(Integer)
table_id = Column(String, ForeignKey('meta_data_table.id'))
table = relationship("MetaDataTable", back_populates="table_columns")
column_label = Column(String(255))
filter_label = Column(String(255))
is_shown = Column(BIT(1))
show_filter = Column(BIT(1))
ref_column = Column(String, nullable=True)
def __repr__(self):
if self.column_name is None:
return f'MetaDataColumn({self.id} {self.table.table_name}.__)'
else:
return f'MetaDataColumn({self.id} {self.table.table_name}.{self.column_name})'
def __str__(self):
return f'{self.column_name}({self.id})'
python/kontor-schema/kontor_schema/tysc.py
-100
View File
@@ -1,100 +0,0 @@
from sqlalchemy import Column, DateTime, Integer, String, ForeignKey, UniqueConstraint
from sqlalchemy.dialects.mysql import BIT
from sqlalchemy.orm import relationship
from .base import Base, BaseMixin
class Sport(Base, BaseMixin):
__tablename__ = "sport"
__table_args__ = (
UniqueConstraint("name"),
)
name = Column(String(255), nullable=False, index=True, unique=True)
teams = relationship("Team")
positions = relationship("FieldPosition")
class Team(Base, BaseMixin):
__tablename__ = "team"
name = Column(String(255), nullable=False, index=True, unique=True)
short_name = Column(String(255), nullable=False, )
sport_id = Column(String, ForeignKey("sport.id"), nullable=False)
sport = relationship("Sport", back_populates="teams")
roosters = relationship("Rooster")
class FieldPosition(Base, BaseMixin):
__tablename__ = "field_position"
__table_args__ = (
UniqueConstraint("name", "sport_id"),
UniqueConstraint("short_name", "sport_id"),
)
name = Column(String(255), nullable=False, index=True)
short_name = Column(String(255), nullable=False)
sport_id = Column(String, ForeignKey("sport.id"), nullable=False, index=True)
sport = relationship("Sport", back_populates="positions")
roosters = relationship("Rooster")
class Player(Base, BaseMixin):
__tablename__ = "player"
__table_args__ = (
UniqueConstraint("first_name", "last_name"),
)
first_name = Column(String(255), nullable=False, index=True)
last_name = Column(String(255), nullable=False, index=True)
roosters = relationship("Rooster")
def get_full_name(self) -> str:
return f"{self.last_name}, {self.first_name}"
class Rooster(Base, BaseMixin):
__tablename__ = "rooster"
__table_args__ = (
UniqueConstraint("year", "team_id", "player_id", "position_id"),
)
year = Column(Integer)
team_id = Column(String, ForeignKey("team.id"), nullable=False, index=True)
team = relationship("Team", back_populates="roosters")
player_id = Column(String, ForeignKey("player.id"), nullable=False, index=True)
player = relationship("Player", back_populates="roosters")
position_id = Column(String, ForeignKey("field_position.id"), nullable=False, index=True)
position = relationship("FieldPosition", back_populates="roosters")
cards = relationship("Card")
class Vendor(Base, BaseMixin):
__tablename__ = "vendor"
name = Column(String(255), nullable=False, unique=True, index=True)
card_sets = relationship("CardSet")
cards = relationship("Card")
class CardSet(Base, BaseMixin):
__tablename__ = "card_set"
__table_args__ = (
UniqueConstraint("name", "vendor_id"),
)
name = Column(String(255), index=True)
parallel_set = Column(BIT(1))
insert_set = Column(BIT(1))
vendor_id = Column(String, ForeignKey("vendor.id"), nullable=False, index=True)
vendor = relationship("Vendor", back_populates="card_sets")
cards = relationship("Card")
class Card(Base, BaseMixin):
__tablename__ = "card"
__table_args__ = (
UniqueConstraint("card_number", "year", "vendor_id", "card_set_id"),
)
card_number = Column(Integer, index=True)
year = Column(Integer, index=True)
card_set_id = Column(String, ForeignKey("card_set.id"), nullable=False)
card_set = relationship("CardSet", back_populates="cards")
rooster_id = Column(String, ForeignKey("rooster.id"), nullable=False)
rooster = relationship("Rooster", back_populates="cards")
vendor_id = Column(String, ForeignKey("vendor.id"), nullable=False)
vendor = relationship("Vendor", back_populates="cards")
python/kontor-schema/pyvenv.cfg
-5
View File
@@ -1,5 +0,0 @@
home = /usr/bin
include-system-site-packages = false
version = 3.11.2
executable = /usr/bin/python3.11
command = /usr/bin/python -m venv /home/tpeetz/projects/kontor/python/kontor-schema
python/kontor-schema/requirements.txt
-4
View File
@@ -1,4 +0,0 @@
mariadb
sqlalchemy
beautifulsoup4
requests
python/kontor-schema/setup.py
-23
View File
@@ -1,23 +0,0 @@
from setuptools import setup, find_packages
import pathlib
here = pathlib.Path(__file__).parent.resolve()
long_description = ( here / "README.md").read_text(encoding="utf-8")
setup(
name='kontor_schema',
version='0.1.0',
description='Schema for Kontor DB',
long_description=long_description,
long_description_content_type="text/markdown",
author='Thomas Peetz',
classifiers=[
"Development Status :: 3 - Alpha",
"Intended Audience :: Developers",
"License :: OSI Approved :: MIT License",
"Programming Language :: Python :: 3.11",
],
install_requires=["sqlalchemy", "mariadb", "requests", "beautifulsoup4"],
packages=find_packages(),
)