tpeetz/kontor
1
0
Fork 0
Code Issues 6 Pull Requests 2 Actions Packages Projects Releases 2 Wiki Activity

complete MetaDataTable and MetaDataColumn

Browse Source
This commit is contained in:
Thomas Peetz
2025-01-20 15:40:45 +01:00
parent 60fba0d3e9
commit 65288a53a1
16 changed files with 257 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/kontor-cli/kontor/controllers/database.py
+41 -1
View File
@@ -1,9 +1,9 @@
import mariadb
from cement import Controller, ex
from kontor_schema import KontorDB
class Database(Controller):
class Meta:
label = 'database'
stacked_type = 'nested'
@@ -49,3 +49,43 @@ class Database(Controller):
kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log)
self.app.render(data, 'import.jinja2')
kontor_db.import_db(data['db_file'], self.app.pargs.dry_run)
@ex(
help='check the db schema against MetaDataTable and MetaDataColumn'
)
def check(self):
mariadb_conn = mariadb.connect(
host=self.app.config['mariadb']['host'],
port=int(self.app.config['mariadb']['port']),
user=self.app.config['mariadb']['user'],
password=self.app.config['mariadb']['password'],
database=self.app.config['mariadb']['database']
)
table_list = []
cursor = mariadb_conn.cursor()
cursor.execute("SHOW TABLES")
for (tablename,) in cursor.fetchall():
table_list.append(tablename)
kontor_db = KontorDB(self.app.engine, self.app.log)
table_names = kontor_db.get_table_names()
for table in table_list:
if table not in table_names:
self.app.log.info(f"{table} is not stored in MetaDataTable")
continue
meta_data = kontor_db.get_columns(table)
field_info = self.get_table_field_info(cursor, table)
for column in field_info:
if column not in meta_data:
self.app.log.info(f"column {column} of table {table} is not in MetaDataColumn")
mariadb_conn.close()
def get_table_field_info(self, cursor, table) -> dict:
info = {}
cursor.execute(f"SELECT * FROM {table} LIMIT 1")
field_info = mariadb.fieldinfo()
for column in cursor.description:
column_name = column[0]
column_type = field_info.type(column)
column_flags = field_info.flag(column)
info[column_name] = {"type": column_type, "flags": column_flags}
return info
python/kontor-cli/kontor/main.py
+2 -2
View File
@@ -12,7 +12,6 @@ from .controllers.media import Media
# configuration defaults
CONFIG = init_defaults('kontor', 'mariadb', 'media')
CONFIG['kontor']['foo'] = 'bar'
CONFIG['mariadb']['user'] = 'kontor'
CONFIG['mariadb']['password'] = 'kontor'
CONFIG['mariadb']['host'] = '127.0.0.1'
@@ -21,8 +20,9 @@ CONFIG['mariadb']['database'] = 'kontor'
CONFIG['media']['yt-dlp'] = '/home/tpeetz/bin/yt-dlp'
CONFIG['media']['dir'] = '/data/media'
def extend_sqlalchemy(app):
app.log.info('extending kontor application with sqlalchemy')
app.log.debug('extending kontor application with sqlalchemy')
connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format(
app.config.get('mariadb', 'user'),
app.config.get('mariadb', 'password'),
python/kontor-cli/requirements.txt
+2 -2
View File
@@ -1,5 +1,5 @@
-e /home/tpeetz/projects/kontor/python/kontor-schema
-e /home/tpeetz/projects/kontor/python/kontor-video
-e ../kontor-schema
-e ../kontor-video
cement==3.0.12
cement[jinja2]
python/kontor-gui/.gitignore
+1
View File
@@ -1,4 +1,5 @@
deployment/
venv/
kontor.bin
bin/
include/
python/kontor-gui/gui/main_window.py
+3 -1
View File
@@ -1,4 +1,4 @@
from PySide6.QtGui import QAction, QIcon
from PySide6.QtGui import QAction, QIcon, QGuiApplication
from PySide6.QtWidgets import QWidget, QVBoxLayout, QMenu, QMessageBox, QTabWidget, QTableView, QProgressBar
from PySide6.QtWidgets import QLabel, QMainWindow
from sqlalchemy import Engine
@@ -45,6 +45,8 @@ class MainWindow(QMainWindow):
parent_layout.addWidget(self.tabs)
self.setCentralWidget(self.central_widget)
centerPoint = QGuiApplication.screens()[0].geometry().center()
self.move(centerPoint - self.frameGeometry().center())
def _create_actions(self):
self.newAction = QAction("&New", self)
python/kontor-gui/requirements.txt
+2 -2
View File
@@ -1,5 +1,5 @@
-e /home/tpeetz/projects/kontor/python/kontor-schema
-e /home/tpeetz/projects/kontor/python/kontor-video
-e ../kontor-schema
-e ../kontor-video
platformdirs
pyyaml
python/kontor-schema/include/site/python3.11/greenlet/greenlet.h
-164
View File
@@ -1,164 +0,0 @@
/* -*- indent-tabs-mode: nil; tab-width: 4; -*- */
/* Greenlet object interface */
#ifndef Py_GREENLETOBJECT_H
#define Py_GREENLETOBJECT_H
#include <Python.h>
#ifdef __cplusplus
extern "C" {
#endif
/* This is deprecated and undocumented. It does not change. */
#define GREENLET_VERSION "1.0.0"
#ifndef GREENLET_MODULE
#define implementation_ptr_t void*
#endif
typedef struct _greenlet {
PyObject_HEAD
PyObject* weakreflist;
PyObject* dict;
implementation_ptr_t pimpl;
} PyGreenlet;
#define PyGreenlet_Check(op) (op && PyObject_TypeCheck(op, &PyGreenlet_Type))
/* C API functions */
/* Total number of symbols that are exported */
#define PyGreenlet_API_pointers 12
#define PyGreenlet_Type_NUM 0
#define PyExc_GreenletError_NUM 1
#define PyExc_GreenletExit_NUM 2
#define PyGreenlet_New_NUM 3
#define PyGreenlet_GetCurrent_NUM 4
#define PyGreenlet_Throw_NUM 5
#define PyGreenlet_Switch_NUM 6
#define PyGreenlet_SetParent_NUM 7
#define PyGreenlet_MAIN_NUM 8
#define PyGreenlet_STARTED_NUM 9
#define PyGreenlet_ACTIVE_NUM 10
#define PyGreenlet_GET_PARENT_NUM 11
#ifndef GREENLET_MODULE
/* This section is used by modules that uses the greenlet C API */
static void** _PyGreenlet_API = NULL;
# define PyGreenlet_Type \
(*(PyTypeObject*)_PyGreenlet_API[PyGreenlet_Type_NUM])
# define PyExc_GreenletError \
((PyObject*)_PyGreenlet_API[PyExc_GreenletError_NUM])
# define PyExc_GreenletExit \
((PyObject*)_PyGreenlet_API[PyExc_GreenletExit_NUM])
/*
* PyGreenlet_New(PyObject *args)
*
* greenlet.greenlet(run, parent=None)
*/
# define PyGreenlet_New \
(*(PyGreenlet * (*)(PyObject * run, PyGreenlet * parent)) \
_PyGreenlet_API[PyGreenlet_New_NUM])
/*
* PyGreenlet_GetCurrent(void)
*
* greenlet.getcurrent()
*/
# define PyGreenlet_GetCurrent \
(*(PyGreenlet * (*)(void)) _PyGreenlet_API[PyGreenlet_GetCurrent_NUM])
/*
* PyGreenlet_Throw(
* PyGreenlet *greenlet,
* PyObject *typ,
* PyObject *val,
* PyObject *tb)
*
* g.throw(...)
*/
# define PyGreenlet_Throw \
(*(PyObject * (*)(PyGreenlet * self, \
PyObject * typ, \
PyObject * val, \
PyObject * tb)) \
_PyGreenlet_API[PyGreenlet_Throw_NUM])
/*
* PyGreenlet_Switch(PyGreenlet *greenlet, PyObject *args)
*
* g.switch(*args, **kwargs)
*/
# define PyGreenlet_Switch \
(*(PyObject * \
(*)(PyGreenlet * greenlet, PyObject * args, PyObject * kwargs)) \
_PyGreenlet_API[PyGreenlet_Switch_NUM])
/*
* PyGreenlet_SetParent(PyObject *greenlet, PyObject *new_parent)
*
* g.parent = new_parent
*/
# define PyGreenlet_SetParent \
(*(int (*)(PyGreenlet * greenlet, PyGreenlet * nparent)) \
_PyGreenlet_API[PyGreenlet_SetParent_NUM])
/*
* PyGreenlet_GetParent(PyObject* greenlet)
*
* return greenlet.parent;
*
* This could return NULL even if there is no exception active.
* If it does not return NULL, you are responsible for decrementing the
* reference count.
*/
# define PyGreenlet_GetParent \
(*(PyGreenlet* (*)(PyGreenlet*)) \
_PyGreenlet_API[PyGreenlet_GET_PARENT_NUM])
/*
* deprecated, undocumented alias.
*/
# define PyGreenlet_GET_PARENT PyGreenlet_GetParent
# define PyGreenlet_MAIN \
(*(int (*)(PyGreenlet*)) \
_PyGreenlet_API[PyGreenlet_MAIN_NUM])
# define PyGreenlet_STARTED \
(*(int (*)(PyGreenlet*)) \
_PyGreenlet_API[PyGreenlet_STARTED_NUM])
# define PyGreenlet_ACTIVE \
(*(int (*)(PyGreenlet*)) \
_PyGreenlet_API[PyGreenlet_ACTIVE_NUM])
/* Macro that imports greenlet and initializes C API */
/* NOTE: This has actually moved to ``greenlet._greenlet._C_API``, but we
keep the older definition to be sure older code that might have a copy of
the header still works. */
# define PyGreenlet_Import() \
{ \
_PyGreenlet_API = (void**)PyCapsule_Import("greenlet._C_API", 0); \
}
#endif /* GREENLET_MODULE */
#ifdef __cplusplus
}
#endif
#endif /* !Py_GREENLETOBJECT_H */
python/kontor-schema/kontor_schema/__init__.py
+36 -1
View File
@@ -5,10 +5,12 @@ import uuid
from datetime import datetime
from pathlib import Path
import requests
from sqlalchemy import Engine
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import sessionmaker
from .admin import User, Token, Role, AuthorizationMatrix, ModuleData, MailAccount, Mail
from .bookshelf import Article, Book, Author, BookshelfPublisher, ArticleAuthor, BookAuthor
from .comic import Comic, Artist, Publisher, Issue, StoryArc, TradePaperback, Volume, ComicWork, WorkType
from .metadata import MetaDataTable, MetaDataColumn
@@ -51,8 +53,15 @@ class KontorDB:
self.registry['media_file'] = MediaFile
self.registry['media_article'] = MediaArticle
self.registry['media_video'] = MediaVideo
self.registry['meta_data_table'] = MetaDataTable
self.registry[MetaDataTable.__tablename__] = MetaDataTable
self.registry[MetaDataColumn.__tablename__] = MetaDataColumn
self.registry[User.__tablename__] = User
self.registry[Token.__tablename__] = Token
self.registry[Role.__tablename__] = Role
self.registry[AuthorizationMatrix.__tablename__] = AuthorizationMatrix
self.registry[ModuleData.__tablename__] = ModuleData
self.registry[MailAccount.__tablename__] = MailAccount
self.registry[Mail.__tablename__] = Mail
def get_table_names(self) -> list:
result = []
@@ -87,6 +96,17 @@ class KontorDB:
order += 1
return meta_data
def get_columns(self, table_name: str) -> dict:
columns = {}
order = 0
__session__ = sessionmaker(self.engine)
with __session__() as session:
for (_, column) in (session.query(MetaDataTable, MetaDataColumn).
filter(MetaDataTable.id == MetaDataColumn.table_id).
filter(MetaDataTable.table_name == table_name).all()):
columns[column.column_name] = {"order": column.column_order, "type": column.column_type}
return columns
def get_filters(self, table_name):
_filter_map = {}
__session__ = sessionmaker(self.engine)
@@ -300,6 +320,21 @@ class KontorDB:
link.review = 0
session.commit()
def get_update_list(self) -> list[str]:
self.log.debug("get links marked as review")
update_list = []
__session__ = sessionmaker(self.engine)
with __session__() as session:
links = session.query(MediaFile).filter(MediaFile.review == 1).all()
for link in links:
url = link.url
if url is None:
self.log.info(f"url has not been set for {link.id}")
continue
update_list.append(url)
self.log.debug(f"found {len(update_list)} urls for updates")
return update_list
def get_download_list(self) -> list[str]:
self.log.debug("get links marked as should_download")
download_list = []
python/kontor-schema/kontor_schema/admin.py
+78
View File
@@ -0,0 +1,78 @@
from datetime import datetime
from sqlalchemy import Column, DateTime, ForeignKey, Integer, String
from sqlalchemy.dialects.mysql import BIT
from sqlalchemy.orm import relationship, mapped_column, Mapped
from .base import Base, BaseMixin
class User(Base, BaseMixin):
__tablename__ = 'user'
first_name = Column(String(255))
last_name = Column(String(255))
user_name = Column(String(255), nullable=False)
email = Column(String(255))
password = Column(String(255))
enabled = Column(BIT(1))
matrix = relationship("AuthorizationMatrix")
tokens = relationship("Token")
def get_full_name(self) -> str:
full_name = ""
if self.first_name is not None:
full_name += self.first_name
if self.last_name is not None:
if len(full_name) > 0:
full_name += " "
full_name += self.last_name
return full_name
class Token(Base, BaseMixin):
__tablename__ = "token"
token = Column(String(255), nullable=False, unique=True)
name = Column(String(255))
last_used_date: Mapped[datetime] = mapped_column()
enabled = Column(BIT(1))
user_id = Column(String, ForeignKey("user.id"), nullable=False)
user = relationship("User", back_populates="tokens")
class Role(Base, BaseMixin):
__tablename__ = "role"
name = Column(String(255), nullable=False)
matrix = relationship("AuthorizationMatrix")
class AuthorizationMatrix(Base, BaseMixin):
__tablename__ = "authorization_matrix"
user_id = Column(String, ForeignKey("user.id"), nullable=False)
user = relationship("User", back_populates="matrix")
role_id = Column(String, ForeignKey("role.id"), nullable=False)
role = relationship("Role", back_populates="matrix")
class ModuleData(Base, BaseMixin):
__tablename__ = "module_data"
module_name = Column(String(255), nullable=False)
import_data = Column(BIT(1))
class MailAccount(Base, BaseMixin):
__tablename__ = "mail_account"
host = Column(String(255))
port = Column(Integer)
protocol = Column(String(255))
user_name = Column(String(255))
password = Column(String(255))
start_tls = Column(BIT(1))
class Mail(Base, BaseMixin):
__tablename__ = "mail"
folder: Mapped[str] = mapped_column()
subject: Mapped[str] = mapped_column()
body: Mapped[str] = mapped_column()
sent_date: Mapped[datetime] = mapped_column()
received_date: Mapped[datetime] = mapped_column()
python/kontor-schema/kontor_schema/base.py
+1 -1
View File
@@ -1,7 +1,7 @@
import uuid
from datetime import datetime
from sqlalchemy import Integer, func, String
from sqlalchemy import func
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
python/kontor-schema/kontor_schema/metadata.py
+3 -3
View File
@@ -19,11 +19,11 @@ class MetaDataTable(Base, BaseMixin):
class MetaDataColumn(Base, BaseMixin):
__tablename__ = 'meta_data_column'
column_modifier = Column(String(255), nullable=True)
column_name = Column(String(255))
column_order = Column(Integer)
column_name = Column(String(255), nullable=False)
column_sync_name = Column(String(255))
column_type = Column(String(255))
column_modifier = Column(String(255), nullable=True)
column_order = Column(Integer)
table_id = Column(String, ForeignKey('meta_data_table.id'))
table = relationship("MetaDataTable", back_populates="table_columns")
column_label = Column(String(255))