From 1ecf64a2285ecc14dc8c18e431b30db7fc6f8697 Mon Sep 17 00:00:00 2001 From: Thomas Peetz Date: Wed, 14 May 2025 19:36:15 +0200 Subject: [PATCH] update model for import --- kontor-scripts/db/models/comic.py | 6 ++++ kontor-scripts/import.py | 51 ++++++++++++++++-------------- kontor-scripts/json_to_postgres.py | 10 +++--- kontor-scripts/pyproject.toml | 1 + 4 files changed, 40 insertions(+), 28 deletions(-) diff --git a/kontor-scripts/db/models/comic.py b/kontor-scripts/db/models/comic.py index c0678ba..40bb9b7 100644 --- a/kontor-scripts/db/models/comic.py +++ b/kontor-scripts/db/models/comic.py @@ -52,6 +52,7 @@ class Volume(Base, BaseMixin): name = Column(String, nullable=False) comic_id = Column(String, ForeignKey("comic.id"), nullable=False) comic = relationship("Comic", back_populates="volumes") + story_arcs = relationship("StoryArc") issues = relationship("Issue") @@ -69,6 +70,9 @@ class StoryArc(Base, BaseMixin): name = Column(String, nullable=False) comic_id = Column(String, ForeignKey("comic.id"), nullable=False) comic = relationship("Comic", back_populates="story_arcs") + volume_id = Column(String, ForeignKey("volume.id"), nullable=True) + volume = relationship("Volume", back_populates="story_arcs") + issues = relationship("Issue") class Issue(Base, BaseMixin): @@ -80,6 +84,8 @@ class Issue(Base, BaseMixin): comic = relationship("Comic", back_populates="issues") volume_id = Column(String, ForeignKey("volume.id"), nullable=True) volume = relationship("Volume", back_populates="issues") + story_arc_id = Column(String, ForeignKey("story_arc.id"), nullable=True) + story_arc = relationship("StoryArc", back_populates="issues") class Artist(Base, BaseMixin): diff --git a/kontor-scripts/import.py b/kontor-scripts/import.py index f50d82d..dc1764a 100644 --- a/kontor-scripts/import.py +++ b/kontor-scripts/import.py @@ -3,40 +3,45 @@ import data from json file to MariaDB """ from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -import yaml +from typing import Generator +import os from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from platformdirs import PlatformDirs -from pathlib import Path - -from schema import Base, KontorDB +from sqlalchemy.orm import sessionmaker, Session +from db.models.base import Base +from db.models import registry from config import get_logger parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument('--verbose', '-v', action='count', default=0) -parser.add_argument('--config', '-c', default='kontor-docker') parser.add_argument('--recreate-db', action='store_true') parser.add_argument('--file', '-f', default='~/data.json') args = parser.parse_args() +DB_USER: str = os.getenv("DB_USER", "kontor") +DB_PASSWORD: str = os.getenv("DB_PASSWORD", "kontor") +DB_SERVER: str = os.getenv("DB_SERVER", "127.0.0.1") +DB_PORT: int = os.getenv("DB_PORT", 5432) +DB_DBNAME: str = os.getenv("DB_DBNAME", "kontor") +DATABASE_URL: str = f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_SERVER}:{DB_PORT}/{DB_DBNAME}" + + +def cleanup_database(db: Session, log): + # get tables from registry + for table in registry: + log.info(f"{table}") + model = registry[table] + entries = db.query(model).all() + for entry in entries: + db.delete(entry) + db.commit() if __name__ == '__main__': - logger = get_logger(args.verbose, args.config) + logger = get_logger(args.verbose, "kontor") logger.info('kontor.import started') - dirs = PlatformDirs(args.config) - database_config = Path(dirs.user_config_dir, 'database-config.yaml') - with open(database_config, 'rt') as f: - db_config = yaml.safe_load(f.read()) - connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format( - db_config['mariadb']['user'], - db_config['mariadb']['password'], - db_config['mariadb']['host'], - db_config['mariadb']['port'], - db_config['mariadb']['database'] - )) - engine = create_engine(connect_string) + engine = create_engine(DATABASE_URL) Base.metadata.create_all(bind=engine, checkfirst=True) - __session__ = sessionmaker(bind=engine) - kontor_db = KontorDB(engine, logger) - kontor_db.import_db(args.file) + SessionLocal = sessionmaker(bind=engine) + with SessionLocal() as db: + cleanup_database(db, logger) logger.info('kontor.import finished') + diff --git a/kontor-scripts/json_to_postgres.py b/kontor-scripts/json_to_postgres.py index 2616313..546597c 100644 --- a/kontor-scripts/json_to_postgres.py +++ b/kontor-scripts/json_to_postgres.py @@ -69,15 +69,15 @@ def insert_data(postgres_conn, data: dict, log): postgres_cursor = postgres_conn.cursor() log.info("insert data") table_list = [] - #table_list = ['worktype', 'artist', 'publisher', 'volume', 'comic', 'issue', 'story_arc', 'trade_paperback', 'comic_work'] - #table_list.extend(['sport', 'team', 'field_position', 'vendor', 'player', 'rooster', 'card_set', 'card']) + table_list = ['worktype', 'artist', 'publisher', 'volume', 'comic', 'issue', 'story_arc', 'trade_paperback', 'comic_work'] + table_list.extend(['sport', 'team', 'field_position', 'vendor', 'player', 'rooster', 'card_set', 'card']) #table_list.extend(['card']) - #table_list.extend(['media_file', 'media_video', 'media_actor', 'media_article', 'media_actor_file']) + table_list.extend(['media_file', 'media_video', 'media_actor', 'media_article', 'media_actor_file']) #table_list.extend(['media_actor_file']) table_list.extend(['profile', 'permission', 'token', 'assignment']) - #table_list.extend(['mail', 'mail_account', 'module_data', 'meta_data_table', 'meta_data_column']) + table_list.extend(['mail', 'mail_account', 'module_data', 'meta_data_table', 'meta_data_column']) table_list.extend(['meta_data_table', 'meta_data_column']) - #table_list.extend(['book', 'author', 'article', 'bookshelf_publisher', 'book_author', 'article_author']) + table_list.extend(['book', 'author', 'article', 'bookshelf_publisher', 'book_author', 'article_author']) #if len(table_list) != 37: # log.info(f"number of tables incorrect: {len(table_list)}") # return diff --git a/kontor-scripts/pyproject.toml b/kontor-scripts/pyproject.toml index 99a547e..589b312 100644 --- a/kontor-scripts/pyproject.toml +++ b/kontor-scripts/pyproject.toml @@ -1,4 +1,5 @@ [project] +requires-python = ">=3.13" name = "kontor-scripts" version = "0.1.0" readme = "README.md"