diff --git a/python/kontor-cli/kontor/controllers/database.py b/python/kontor-cli/kontor/controllers/database.py index 73732f4..a289c99 100644 --- a/python/kontor-cli/kontor/controllers/database.py +++ b/python/kontor-cli/kontor/controllers/database.py @@ -1,9 +1,9 @@ +import mariadb from cement import Controller, ex from kontor_schema import KontorDB class Database(Controller): - class Meta: label = 'database' stacked_type = 'nested' @@ -49,3 +49,43 @@ class Database(Controller): kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log) self.app.render(data, 'import.jinja2') kontor_db.import_db(data['db_file'], self.app.pargs.dry_run) + + @ex( + help='check the db schema against MetaDataTable and MetaDataColumn' + ) + def check(self): + mariadb_conn = mariadb.connect( + host=self.app.config['mariadb']['host'], + port=int(self.app.config['mariadb']['port']), + user=self.app.config['mariadb']['user'], + password=self.app.config['mariadb']['password'], + database=self.app.config['mariadb']['database'] + ) + table_list = [] + cursor = mariadb_conn.cursor() + cursor.execute("SHOW TABLES") + for (tablename,) in cursor.fetchall(): + table_list.append(tablename) + kontor_db = KontorDB(self.app.engine, self.app.log) + table_names = kontor_db.get_table_names() + for table in table_list: + if table not in table_names: + self.app.log.info(f"{table} is not stored in MetaDataTable") + continue + meta_data = kontor_db.get_columns(table) + field_info = self.get_table_field_info(cursor, table) + for column in field_info: + if column not in meta_data: + self.app.log.info(f"column {column} of table {table} is not in MetaDataColumn") + mariadb_conn.close() + + def get_table_field_info(self, cursor, table) -> dict: + info = {} + cursor.execute(f"SELECT * FROM {table} LIMIT 1") + field_info = mariadb.fieldinfo() + for column in cursor.description: + column_name = column[0] + column_type = field_info.type(column) + column_flags = field_info.flag(column) + info[column_name] = {"type": column_type, "flags": column_flags} + return info diff --git a/python/kontor-cli/kontor/main.py b/python/kontor-cli/kontor/main.py index 8884b87..2d71a3e 100644 --- a/python/kontor-cli/kontor/main.py +++ b/python/kontor-cli/kontor/main.py @@ -12,7 +12,6 @@ from .controllers.media import Media # configuration defaults CONFIG = init_defaults('kontor', 'mariadb', 'media') -CONFIG['kontor']['foo'] = 'bar' CONFIG['mariadb']['user'] = 'kontor' CONFIG['mariadb']['password'] = 'kontor' CONFIG['mariadb']['host'] = '127.0.0.1' @@ -21,8 +20,9 @@ CONFIG['mariadb']['database'] = 'kontor' CONFIG['media']['yt-dlp'] = '/home/tpeetz/bin/yt-dlp' CONFIG['media']['dir'] = '/data/media' + def extend_sqlalchemy(app): - app.log.info('extending kontor application with sqlalchemy') + app.log.debug('extending kontor application with sqlalchemy') connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format( app.config.get('mariadb', 'user'), app.config.get('mariadb', 'password'), diff --git a/python/kontor-cli/requirements.txt b/python/kontor-cli/requirements.txt index 92890d6..65cbadd 100644 --- a/python/kontor-cli/requirements.txt +++ b/python/kontor-cli/requirements.txt @@ -1,5 +1,5 @@ --e /home/tpeetz/projects/kontor/python/kontor-schema --e /home/tpeetz/projects/kontor/python/kontor-video +-e ../kontor-schema +-e ../kontor-video cement==3.0.12 cement[jinja2] diff --git a/python/kontor-gui/.gitignore b/python/kontor-gui/.gitignore index 8d5ef37..9c0554f 100644 --- a/python/kontor-gui/.gitignore +++ b/python/kontor-gui/.gitignore @@ -1,4 +1,5 @@ deployment/ +venv/ kontor.bin bin/ include/ diff --git a/python/kontor-gui/gui/main_window.py b/python/kontor-gui/gui/main_window.py index 4f4ee91..739212e 100644 --- a/python/kontor-gui/gui/main_window.py +++ b/python/kontor-gui/gui/main_window.py @@ -1,4 +1,4 @@ -from PySide6.QtGui import QAction, QIcon +from PySide6.QtGui import QAction, QIcon, QGuiApplication from PySide6.QtWidgets import QWidget, QVBoxLayout, QMenu, QMessageBox, QTabWidget, QTableView, QProgressBar from PySide6.QtWidgets import QLabel, QMainWindow from sqlalchemy import Engine @@ -45,6 +45,8 @@ class MainWindow(QMainWindow): parent_layout.addWidget(self.tabs) self.setCentralWidget(self.central_widget) + centerPoint = QGuiApplication.screens()[0].geometry().center() + self.move(centerPoint - self.frameGeometry().center()) def _create_actions(self): self.newAction = QAction("&New", self) diff --git a/python/kontor-gui/requirements.txt b/python/kontor-gui/requirements.txt index 90b0945..057d486 100644 --- a/python/kontor-gui/requirements.txt +++ b/python/kontor-gui/requirements.txt @@ -1,5 +1,5 @@ --e /home/tpeetz/projects/kontor/python/kontor-schema --e /home/tpeetz/projects/kontor/python/kontor-video +-e ../kontor-schema +-e ../kontor-video platformdirs pyyaml diff --git a/python/kontor-schema/include/site/python3.11/greenlet/greenlet.h b/python/kontor-schema/include/site/python3.11/greenlet/greenlet.h deleted file mode 100644 index d02a16e..0000000 --- a/python/kontor-schema/include/site/python3.11/greenlet/greenlet.h +++ /dev/null @@ -1,164 +0,0 @@ -/* -*- indent-tabs-mode: nil; tab-width: 4; -*- */ - -/* Greenlet object interface */ - -#ifndef Py_GREENLETOBJECT_H -#define Py_GREENLETOBJECT_H - - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/* This is deprecated and undocumented. It does not change. */ -#define GREENLET_VERSION "1.0.0" - -#ifndef GREENLET_MODULE -#define implementation_ptr_t void* -#endif - -typedef struct _greenlet { - PyObject_HEAD - PyObject* weakreflist; - PyObject* dict; - implementation_ptr_t pimpl; -} PyGreenlet; - -#define PyGreenlet_Check(op) (op && PyObject_TypeCheck(op, &PyGreenlet_Type)) - - -/* C API functions */ - -/* Total number of symbols that are exported */ -#define PyGreenlet_API_pointers 12 - -#define PyGreenlet_Type_NUM 0 -#define PyExc_GreenletError_NUM 1 -#define PyExc_GreenletExit_NUM 2 - -#define PyGreenlet_New_NUM 3 -#define PyGreenlet_GetCurrent_NUM 4 -#define PyGreenlet_Throw_NUM 5 -#define PyGreenlet_Switch_NUM 6 -#define PyGreenlet_SetParent_NUM 7 - -#define PyGreenlet_MAIN_NUM 8 -#define PyGreenlet_STARTED_NUM 9 -#define PyGreenlet_ACTIVE_NUM 10 -#define PyGreenlet_GET_PARENT_NUM 11 - -#ifndef GREENLET_MODULE -/* This section is used by modules that uses the greenlet C API */ -static void** _PyGreenlet_API = NULL; - -# define PyGreenlet_Type \ - (*(PyTypeObject*)_PyGreenlet_API[PyGreenlet_Type_NUM]) - -# define PyExc_GreenletError \ - ((PyObject*)_PyGreenlet_API[PyExc_GreenletError_NUM]) - -# define PyExc_GreenletExit \ - ((PyObject*)_PyGreenlet_API[PyExc_GreenletExit_NUM]) - -/* - * PyGreenlet_New(PyObject *args) - * - * greenlet.greenlet(run, parent=None) - */ -# define PyGreenlet_New \ - (*(PyGreenlet * (*)(PyObject * run, PyGreenlet * parent)) \ - _PyGreenlet_API[PyGreenlet_New_NUM]) - -/* - * PyGreenlet_GetCurrent(void) - * - * greenlet.getcurrent() - */ -# define PyGreenlet_GetCurrent \ - (*(PyGreenlet * (*)(void)) _PyGreenlet_API[PyGreenlet_GetCurrent_NUM]) - -/* - * PyGreenlet_Throw( - * PyGreenlet *greenlet, - * PyObject *typ, - * PyObject *val, - * PyObject *tb) - * - * g.throw(...) - */ -# define PyGreenlet_Throw \ - (*(PyObject * (*)(PyGreenlet * self, \ - PyObject * typ, \ - PyObject * val, \ - PyObject * tb)) \ - _PyGreenlet_API[PyGreenlet_Throw_NUM]) - -/* - * PyGreenlet_Switch(PyGreenlet *greenlet, PyObject *args) - * - * g.switch(*args, **kwargs) - */ -# define PyGreenlet_Switch \ - (*(PyObject * \ - (*)(PyGreenlet * greenlet, PyObject * args, PyObject * kwargs)) \ - _PyGreenlet_API[PyGreenlet_Switch_NUM]) - -/* - * PyGreenlet_SetParent(PyObject *greenlet, PyObject *new_parent) - * - * g.parent = new_parent - */ -# define PyGreenlet_SetParent \ - (*(int (*)(PyGreenlet * greenlet, PyGreenlet * nparent)) \ - _PyGreenlet_API[PyGreenlet_SetParent_NUM]) - -/* - * PyGreenlet_GetParent(PyObject* greenlet) - * - * return greenlet.parent; - * - * This could return NULL even if there is no exception active. - * If it does not return NULL, you are responsible for decrementing the - * reference count. - */ -# define PyGreenlet_GetParent \ - (*(PyGreenlet* (*)(PyGreenlet*)) \ - _PyGreenlet_API[PyGreenlet_GET_PARENT_NUM]) - -/* - * deprecated, undocumented alias. - */ -# define PyGreenlet_GET_PARENT PyGreenlet_GetParent - -# define PyGreenlet_MAIN \ - (*(int (*)(PyGreenlet*)) \ - _PyGreenlet_API[PyGreenlet_MAIN_NUM]) - -# define PyGreenlet_STARTED \ - (*(int (*)(PyGreenlet*)) \ - _PyGreenlet_API[PyGreenlet_STARTED_NUM]) - -# define PyGreenlet_ACTIVE \ - (*(int (*)(PyGreenlet*)) \ - _PyGreenlet_API[PyGreenlet_ACTIVE_NUM]) - - - - -/* Macro that imports greenlet and initializes C API */ -/* NOTE: This has actually moved to ``greenlet._greenlet._C_API``, but we - keep the older definition to be sure older code that might have a copy of - the header still works. */ -# define PyGreenlet_Import() \ - { \ - _PyGreenlet_API = (void**)PyCapsule_Import("greenlet._C_API", 0); \ - } - -#endif /* GREENLET_MODULE */ - -#ifdef __cplusplus -} -#endif -#endif /* !Py_GREENLETOBJECT_H */ diff --git a/python/kontor-schema/kontor_schema/__init__.py b/python/kontor-schema/kontor_schema/__init__.py index 3549955..a70aa6d 100644 --- a/python/kontor-schema/kontor_schema/__init__.py +++ b/python/kontor-schema/kontor_schema/__init__.py @@ -5,10 +5,12 @@ import uuid from datetime import datetime from pathlib import Path +import requests from sqlalchemy import Engine from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import sessionmaker +from .admin import User, Token, Role, AuthorizationMatrix, ModuleData, MailAccount, Mail from .bookshelf import Article, Book, Author, BookshelfPublisher, ArticleAuthor, BookAuthor from .comic import Comic, Artist, Publisher, Issue, StoryArc, TradePaperback, Volume, ComicWork, WorkType from .metadata import MetaDataTable, MetaDataColumn @@ -51,8 +53,15 @@ class KontorDB: self.registry['media_file'] = MediaFile self.registry['media_article'] = MediaArticle self.registry['media_video'] = MediaVideo - self.registry['meta_data_table'] = MetaDataTable + self.registry[MetaDataTable.__tablename__] = MetaDataTable self.registry[MetaDataColumn.__tablename__] = MetaDataColumn + self.registry[User.__tablename__] = User + self.registry[Token.__tablename__] = Token + self.registry[Role.__tablename__] = Role + self.registry[AuthorizationMatrix.__tablename__] = AuthorizationMatrix + self.registry[ModuleData.__tablename__] = ModuleData + self.registry[MailAccount.__tablename__] = MailAccount + self.registry[Mail.__tablename__] = Mail def get_table_names(self) -> list: result = [] @@ -87,6 +96,17 @@ class KontorDB: order += 1 return meta_data + def get_columns(self, table_name: str) -> dict: + columns = {} + order = 0 + __session__ = sessionmaker(self.engine) + with __session__() as session: + for (_, column) in (session.query(MetaDataTable, MetaDataColumn). + filter(MetaDataTable.id == MetaDataColumn.table_id). + filter(MetaDataTable.table_name == table_name).all()): + columns[column.column_name] = {"order": column.column_order, "type": column.column_type} + return columns + def get_filters(self, table_name): _filter_map = {} __session__ = sessionmaker(self.engine) @@ -300,6 +320,21 @@ class KontorDB: link.review = 0 session.commit() + def get_update_list(self) -> list[str]: + self.log.debug("get links marked as review") + update_list = [] + __session__ = sessionmaker(self.engine) + with __session__() as session: + links = session.query(MediaFile).filter(MediaFile.review == 1).all() + for link in links: + url = link.url + if url is None: + self.log.info(f"url has not been set for {link.id}") + continue + update_list.append(url) + self.log.debug(f"found {len(update_list)} urls for updates") + return update_list + def get_download_list(self) -> list[str]: self.log.debug("get links marked as should_download") download_list = [] diff --git a/python/kontor-schema/kontor_schema/admin.py b/python/kontor-schema/kontor_schema/admin.py new file mode 100644 index 0000000..b846d97 --- /dev/null +++ b/python/kontor-schema/kontor_schema/admin.py @@ -0,0 +1,78 @@ +from datetime import datetime + +from sqlalchemy import Column, DateTime, ForeignKey, Integer, String +from sqlalchemy.dialects.mysql import BIT +from sqlalchemy.orm import relationship, mapped_column, Mapped + +from .base import Base, BaseMixin + + +class User(Base, BaseMixin): + __tablename__ = 'user' + first_name = Column(String(255)) + last_name = Column(String(255)) + user_name = Column(String(255), nullable=False) + email = Column(String(255)) + password = Column(String(255)) + enabled = Column(BIT(1)) + matrix = relationship("AuthorizationMatrix") + tokens = relationship("Token") + + def get_full_name(self) -> str: + full_name = "" + if self.first_name is not None: + full_name += self.first_name + if self.last_name is not None: + if len(full_name) > 0: + full_name += " " + full_name += self.last_name + return full_name + + +class Token(Base, BaseMixin): + __tablename__ = "token" + token = Column(String(255), nullable=False, unique=True) + name = Column(String(255)) + last_used_date: Mapped[datetime] = mapped_column() + enabled = Column(BIT(1)) + user_id = Column(String, ForeignKey("user.id"), nullable=False) + user = relationship("User", back_populates="tokens") + + +class Role(Base, BaseMixin): + __tablename__ = "role" + name = Column(String(255), nullable=False) + matrix = relationship("AuthorizationMatrix") + + +class AuthorizationMatrix(Base, BaseMixin): + __tablename__ = "authorization_matrix" + user_id = Column(String, ForeignKey("user.id"), nullable=False) + user = relationship("User", back_populates="matrix") + role_id = Column(String, ForeignKey("role.id"), nullable=False) + role = relationship("Role", back_populates="matrix") + + +class ModuleData(Base, BaseMixin): + __tablename__ = "module_data" + module_name = Column(String(255), nullable=False) + import_data = Column(BIT(1)) + + +class MailAccount(Base, BaseMixin): + __tablename__ = "mail_account" + host = Column(String(255)) + port = Column(Integer) + protocol = Column(String(255)) + user_name = Column(String(255)) + password = Column(String(255)) + start_tls = Column(BIT(1)) + + +class Mail(Base, BaseMixin): + __tablename__ = "mail" + folder: Mapped[str] = mapped_column() + subject: Mapped[str] = mapped_column() + body: Mapped[str] = mapped_column() + sent_date: Mapped[datetime] = mapped_column() + received_date: Mapped[datetime] = mapped_column() diff --git a/python/kontor-schema/kontor_schema/base.py b/python/kontor-schema/kontor_schema/base.py index c976167..21186d4 100644 --- a/python/kontor-schema/kontor_schema/base.py +++ b/python/kontor-schema/kontor_schema/base.py @@ -1,7 +1,7 @@ import uuid from datetime import datetime -from sqlalchemy import Integer, func, String +from sqlalchemy import func from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column diff --git a/python/kontor-schema/kontor_schema/metadata.py b/python/kontor-schema/kontor_schema/metadata.py index 21d4d49..f9538bb 100644 --- a/python/kontor-schema/kontor_schema/metadata.py +++ b/python/kontor-schema/kontor_schema/metadata.py @@ -19,11 +19,11 @@ class MetaDataTable(Base, BaseMixin): class MetaDataColumn(Base, BaseMixin): __tablename__ = 'meta_data_column' - column_modifier = Column(String(255), nullable=True) - column_name = Column(String(255)) - column_order = Column(Integer) + column_name = Column(String(255), nullable=False) column_sync_name = Column(String(255)) column_type = Column(String(255)) + column_modifier = Column(String(255), nullable=True) + column_order = Column(Integer) table_id = Column(String, ForeignKey('meta_data_table.id')) table = relationship("MetaDataTable", back_populates="table_columns") column_label = Column(String(255)) diff --git a/springboot/src/main/java/de/thpeetz/kontor/admin/SetupModuleAdmin.java b/springboot/src/main/java/de/thpeetz/kontor/admin/SetupModuleAdmin.java index b54f74e..3f12472 100644 --- a/springboot/src/main/java/de/thpeetz/kontor/admin/SetupModuleAdmin.java +++ b/springboot/src/main/java/de/thpeetz/kontor/admin/SetupModuleAdmin.java @@ -332,5 +332,89 @@ public class SetupModuleAdmin implements ApplicationListener