tpeetz/kontor
1
0
Fork 0
Code Issues 6 Pull Requests 2 Actions Packages Projects Releases 2 Wiki Activity

update models to use string type for id fields

Browse Source
This commit is contained in:
Thomas Peetz
2025-05-02 11:21:57 +02:00
parent 7ff2bf912d
commit c77adb0e04
31 changed files with 397 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kontor-scripts/json_to_postgres.py
+100 -6
View File
@@ -3,6 +3,8 @@ copy data from JSON to Postgres
"""
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from pathlib import Path
from typing import Dict, List
from config import get_logger, get_database_cursors
import json
import psycopg2
@@ -29,7 +31,10 @@ def copy_data(postgres_conn, data_file: Path, log):
# result[table] = import_table(table, json_load[table])
truncate_statement = 'TRUNCATE {} CASCADE'.format(table)
#log.info(f"truncate: {truncate_statement}")
postgres_cursor.execute(truncate_statement)
try:
postgres_cursor.execute(truncate_statement)
except:
log.info(f"statement: {insert_statement} FAILED")
items = json_load[table]
for item in items:
#log.info(f"item: {item}")
@@ -39,21 +44,110 @@ def copy_data(postgres_conn, data_file: Path, log):
columns.append(key)
values.append(value)
row = tuple(values)
log.info(f"values: {row}")
#log.info(f"values: {row}")
insert_statement = 'INSERT INTO {}({}) VALUES({})'.format(table, ', '.join(columns), ', '.join(['%s']*len(columns)))
#log.info(f"statement: {insert_statement}")
postgres_cursor.execute(SQL(insert_statement), row)
try:
postgres_cursor.execute(SQL(insert_statement), row)
postgres_conn.commit()
except psycopg2.Error as error:
log.info('insert failed with %s', error)
except:
log.info(f'insert failed with {insert_statement}')
postgres_cursor.execute("SET session_replication_role='origin'")
def load_json(data_file, log) -> dict:
import_file = Path(data_file)
if not import_file.exists():
log.info(f"File {data_file} does not exist. Do nothing.")
return
log.info("read json file")
with open(data_file, 'r') as json_file:
json_load = json.load(json_file)
return json_load
def insert_data(postgres_conn, data: dict, log):
postgres_cursor = postgres_conn.cursor()
log.info("insert data")
table_list = []
#table_list = ['worktype', 'artist', 'publisher', 'volume', 'comic', 'issue', 'story_arc', 'trade_paperback', 'comic_work']
#table_list.extend(['sport', 'team', 'field_position', 'vendor', 'player', 'rooster', 'card_set', 'card'])
#table_list.extend(['card'])
#table_list.extend(['media_file', 'media_video', 'media_actor', 'media_actor_file', 'media_article'])
#table_list.extend(['media_actor_file'])
#table_list.extend(['profile', 'permission', 'token', 'assignment'])
#table_list.extend(['mail', 'mail_account', 'module_data', 'meta_data_table', 'meta_data_column'])
table_list.extend(['meta_data_column'])
#table_list.extend(['book', 'author', 'article', 'bookshelf_publisher', 'book_author', 'article_author'])
#if len(table_list) != 37:
# log.info(f"number of tables incorrect: {len(table_list)}")
# return
for table in table_list:
log.info(f"{table}: {len(data[table])}")
truncate_statement = 'DELETE FROM {}'.format(table)
log.info(f"truncate: {truncate_statement}")
try:
postgres_cursor.execute(truncate_statement)
postgres_conn.commit()
except:
log.info(f"statement: {truncate_statement} FAILED")
items = data[table]
for item in items:
# log.info(f"item: {item}")
values = []
columns = []
for (key, value) in item.items():
columns.append(key)
values.append(value)
row = tuple(values)
# log.info(f"values: {row}")
insert_statement = 'INSERT INTO {}({}) VALUES({})'.format(table, ', '.join(columns),
', '.join(['%s'] * len(columns)))
# log.info(f"statement: {insert_statement}")
try:
postgres_cursor.execute(SQL(insert_statement), row)
postgres_conn.commit()
except:
log.info(f'insert failed with {insert_statement}')
def parse_table_order(data: dict, log):
log.info("parse_table_order")
table_refs: Dict[str, List[str]] = {}
for table in data:
log.info(f"{table}: {len(data[table])}")
items = data[table]
table_refs[table] = []
if len(items) == 0:
continue
item = items[0]
for key, _ in item.items():
if key.endswith('_id'):
ref = key[0:-3]
log.info(f"table {table} has reference to {ref}")
if table in table_refs:
table_refs[table].append(ref)
else:
table_refs[table] = [ref]
log.info(f"parsed refs: {table_refs}")
table_order = []
for table in table_refs:
if len(table_refs[table]) == 0:
log.info(f"insert {table} at beginning")
table_order.insert(0, table)
else:
log.info(f"insert {table} at end")
table_order.append(table)
log.info(f"table_list: {len(table_order)}: {table_order}")
if __name__ == '__main__':
logger = get_logger(args.verbose, args.config)
logger.info('kontor.json_to_postgres started')
_, _, p_conn = get_database_cursors(logger, args.config)
copy_data(p_conn, args.file, logger)
data = load_json(args.file, logger)
#parse_table_order(data, logger)
insert_data(p_conn, data, logger)
#copy_data(p_conn, args.file, logger)
p_conn.close()
logger.info('kontor.json_to_postgres finished')
kontor-scripts/schema/admin.py
+6 -6
View File
@@ -40,7 +40,7 @@ class Token(Base, BaseMixin):
class Permission(Base, BaseMixin):
__tablename__ = "permission"
name = Column(String(255), nullable=False)
name = Column(String, nullable=False)
assignments = relationship("Assignment")
@@ -54,17 +54,17 @@ class Assignment(Base, BaseMixin):
class ModuleData(Base, BaseMixin):
__tablename__ = "module_data"
module_name = Column(String(255), nullable=False)
module_name = Column(String, nullable=False)
import_data = Column(Boolean)
class MailAccount(Base, BaseMixin):
__tablename__ = "mail_account"
host = Column(String(255))
host = Column(String)
port = Column(Integer)
protocol = Column(String(255))
user_name = Column(String(255))
password = Column(String(255))
protocol = Column(String)
user_name = Column(String)
password = Column(String)
start_tls = Column(Boolean)
kontor-scripts/schema/base.py
+7 -7
View File
@@ -10,8 +10,8 @@ class Base(DeclarativeBase):
class BaseMixin:
id = Column(String(255), primary_key=True, default=uuid.uuid4())
# id: Mapped[str] = mapped_column(primary_key=True, default=uuid.uuid4())
#id = Column(String, primary_key=True, default=uuid.uuid4)
id: Mapped[str] = mapped_column(primary_key=True, default=uuid.uuid4)
# created_date = Column(DateTime)
created_date: Mapped[datetime] = mapped_column(default=func.now())
# last_modified_date = Column(DateTime)
@@ -21,10 +21,10 @@ class BaseMixin:
class BaseVideoMixin:
cloud_link = Column(String(255))
file_name = Column(String(255))
path = Column(String(255))
cloud_link = Column(String)
file_name = Column(String)
path = Column(String)
review = Column(Boolean)
title = Column(String(255))
url = Column(String(255), unique=True)
title = Column(String)
url = Column(String, unique=True)
should_download = Column(Boolean)
kontor-scripts/schema/bookshelf.py
+6 -6
View File
@@ -6,28 +6,28 @@ from .base import Base, BaseMixin
class Article(Base, BaseMixin):
__tablename__ = 'article'
title = Column(String(length=255), unique=True)
title = Column(String, unique=True)
article_authors = relationship("ArticleAuthor")
class Author(Base, BaseMixin):
__tablename__ = 'author'
first_name = Column(String(255))
last_name = Column(String(255))
first_name = Column(String)
last_name = Column(String)
article_authors = relationship("ArticleAuthor")
book_authors = relationship("BookAuthor")
class BookshelfPublisher(Base, BaseMixin):
__tablename__ = 'bookshelf_publisher'
name = Column(String(length=255), unique=True)
name = Column(String, unique=True)
books = relationship("Book")
class Book(Base, BaseMixin):
__tablename__ = 'book'
isbn = Column(String(255), unique=True)
title = Column(String(255))
isbn = Column(String, unique=True)
title = Column(String)
year = Column(Integer, nullable=False)
publisher_id = Column(String, ForeignKey('bookshelf_publisher.id'), nullable=False)
publisher = relationship('BookshelfPublisher', back_populates="books")
kontor-scripts/schema/comic.py
+8 -8
View File
@@ -6,7 +6,7 @@ from .base import Base, BaseMixin
class Publisher(Base, BaseMixin):
__tablename__ = "publisher"
name = Column(String(length=255), unique=True)
name = Column(String, unique=True)
comics = relationship("Comic")
def __repr__(self):
@@ -18,7 +18,7 @@ class Publisher(Base, BaseMixin):
class Comic(Base, BaseMixin):
__tablename__ = 'comic'
title = Column(String(length=255), unique=True)
title = Column(String, unique=True)
publisher_id = Column(String, ForeignKey('publisher.id'), nullable=False)
publisher = relationship("Publisher", back_populates="comics")
current_order = Column(Boolean)
@@ -38,7 +38,7 @@ class Comic(Base, BaseMixin):
class Volume(Base, BaseMixin):
__tablename__ = "volume"
name = Column(String(length=255), nullable=False)
name = Column(String, nullable=False)
comic_id = Column(String, ForeignKey("comic.id"), nullable=False)
comic = relationship("Comic", back_populates="volumes")
issues = relationship("Issue")
@@ -46,7 +46,7 @@ class Volume(Base, BaseMixin):
class TradePaperback(Base, BaseMixin):
__tablename__ = "trade_paperback"
name = Column(String(length=255), nullable=False)
name = Column(String, nullable=False)
issue_start = Column(Integer)
issue_end = Column(Integer)
comic_id = Column(String, ForeignKey("comic.id"), nullable=False)
@@ -55,14 +55,14 @@ class TradePaperback(Base, BaseMixin):
class StoryArc(Base, BaseMixin):
__tablename__ = "story_arc"
name = Column(String(length=255), nullable=False)
name = Column(String, nullable=False)
comic_id = Column(String, ForeignKey("comic.id"), nullable=False)
comic = relationship("Comic", back_populates="story_arcs")
class Issue(Base, BaseMixin):
__tablename__ = "issue"
issue_number = Column(String(255))
issue_number = Column(String)
in_stock = Column(Boolean)
is_read = Column(Boolean)
comic_id = Column(String, ForeignKey("comic.id"), nullable=False)
@@ -73,13 +73,13 @@ class Issue(Base, BaseMixin):
class Artist(Base, BaseMixin):
__tablename__ = "artist"
name = Column(String(length=255), nullable=False)
name = Column(String, nullable=False)
comic_works = relationship("ComicWork")
class WorkType(Base, BaseMixin):
__tablename__ = "worktype"
name = Column(String(length=255), nullable=False, unique=True)
name = Column(String, nullable=False, unique=True)
comic_works = relationship("ComicWork")
def __repr__(self):
kontor-scripts/schema/media.py
+10 -10
View File
@@ -69,31 +69,31 @@ class MediaFile(Base, BaseMixin, BaseVideoMixin):
class MediaActor(Base, BaseMixin):
__tablename__ = 'media_actor'
name = Column(String(255))
name = Column(String)
media_actor_files = relationship("MediaActorFile")
class MediaActorFile(Base, BaseMixin):
__tablename__ = 'media_actor_file'
media_actor_id = Column(String(255), ForeignKey("media_actor.id"), nullable=False)
media_actor_id = Column(String, ForeignKey("media_actor.id"), nullable=False)
media_actor = relationship("MediaActor", back_populates="media_actor_files")
media_file_id = Column(String(255), ForeignKey("media_file.id"), nullable=True)
media_file_id = Column(String, ForeignKey("media_file.id"), nullable=True)
media_file = relationship("MediaFile", back_populates="media_actor_files")
class MediaArticle(Base, BaseMixin):
__tablename__ = 'media_article'
review = Column(Boolean)
title = Column(String(255))
url = Column(String(255), unique=True)
title = Column(String)
url = Column(String, unique=True)
class MediaVideo(Base, BaseMixin):
__tablename__ = 'media_video'
cloud_link = Column(String(255))
file_name = Column(String(255))
path = Column(String(255))
cloud_link = Column(String)
file_name = Column(String)
path = Column(String)
review = Column(Boolean)
title = Column(String(255))
url = Column(String(255), unique=True)
title = Column(String)
url = Column(String, unique=True)
should_download = Column(Boolean)
kontor-scripts/schema/metadata.py
+7 -7
View File
@@ -6,7 +6,7 @@ from .base import Base, BaseMixin
class MetaDataTable(Base, BaseMixin):
__tablename__ = 'meta_data_table'
table_name = Column(String(255), unique=True)
table_name = Column(String, unique=True)
table_columns = relationship("MetaDataColumn")
def __repr__(self):
@@ -18,15 +18,15 @@ class MetaDataTable(Base, BaseMixin):
class MetaDataColumn(Base, BaseMixin):
__tablename__ = 'meta_data_column'
column_name = Column(String(255), nullable=False)
column_sync_name = Column(String(255))
column_type = Column(String(255))
column_modifier = Column(String(255), nullable=True)
column_name = Column(String, nullable=False)
column_sync_name = Column(String)
column_type = Column(String)
column_modifier = Column(String, nullable=True)
column_order = Column(Integer)
table_id = Column(String, ForeignKey('meta_data_table.id'))
table = relationship("MetaDataTable", back_populates="table_columns")
column_label = Column(String(255))
filter_label = Column(String(255))
column_label = Column(String)
filter_label = Column(String)
is_shown = Column(Boolean)
show_filter = Column(Boolean)
ref_column = Column(String, nullable=True)
kontor-scripts/schema/tysc.py
+9 -9
View File
@@ -9,15 +9,15 @@ class Sport(Base, BaseMixin):
__table_args__ = (
UniqueConstraint("name"),
)
name = Column(String(255), nullable=False, index=True, unique=True)
name = Column(String, nullable=False, index=True, unique=True)
teams = relationship("Team")
positions = relationship("FieldPosition")
class Team(Base, BaseMixin):
__tablename__ = "team"
name = Column(String(255), nullable=False, index=True, unique=True)
short_name = Column(String(255), nullable=False, )
name = Column(String, nullable=False, index=True, unique=True)
short_name = Column(String, nullable=False, )
sport_id = Column(String, ForeignKey("sport.id"), nullable=False)
sport = relationship("Sport", back_populates="teams")
roosters = relationship("Rooster")
@@ -29,8 +29,8 @@ class FieldPosition(Base, BaseMixin):
UniqueConstraint("name", "sport_id"),
UniqueConstraint("short_name", "sport_id"),
)
name = Column(String(255), nullable=False, index=True)
short_name = Column(String(255), nullable=False)
name = Column(String, nullable=False, index=True)
short_name = Column(String, nullable=False)
sport_id = Column(String, ForeignKey("sport.id"), nullable=False, index=True)
sport = relationship("Sport", back_populates="positions")
roosters = relationship("Rooster")
@@ -41,8 +41,8 @@ class Player(Base, BaseMixin):
__table_args__ = (
UniqueConstraint("first_name", "last_name"),
)
first_name = Column(String(255), nullable=False, index=True)
last_name = Column(String(255), nullable=False, index=True)
first_name = Column(String, nullable=False, index=True)
last_name = Column(String, nullable=False, index=True)
roosters = relationship("Rooster")
def get_full_name(self) -> str:
@@ -66,7 +66,7 @@ class Rooster(Base, BaseMixin):
class Vendor(Base, BaseMixin):
__tablename__ = "vendor"
name = Column(String(255), nullable=False, unique=True, index=True)
name = Column(String, nullable=False, unique=True, index=True)
card_sets = relationship("CardSet")
cards = relationship("Card")
@@ -76,7 +76,7 @@ class CardSet(Base, BaseMixin):
__table_args__ = (
UniqueConstraint("name", "vendor_id"),
)
name = Column(String(255), index=True)
name = Column(String, index=True)
parallel_set = Column(Boolean)
insert_set = Column(Boolean)
vendor_id = Column(String, ForeignKey("vendor.id"), nullable=False, index=True)