update scripts to use Postgres

.gitignore

@@ -15,3 +15,6 @@ kontor-gui/dist
 fastapi/.coverage
 kontor-api/.coverage
 db-password.txt
+kontor-api/tests/test_main.py
+kontor-api/tests/test_db.db
+kontor-api/test_db.db

docker-compose.yml

@@ -43,7 +43,7 @@ services:
     volumes:
       - mariadb-storage:/var/lib/mysql:rw
   kontor:
-    image: kontor:0.1.0
+    image: kontor:0.2.0-SNAPSHOT
     restart: unless-stopped
     networks:
       - database
@@ -54,7 +54,7 @@ services:
       postgres:
         condition: service_healthy
   kontor-api:
-    image: kontor-api:0.1.0
+    image: kontor-api:0.2.0-SNAPSHOT
     restart: unless-stopped
     networks:
       - database

kontor-api/tests/conftest.py

@@ -0,0 +1,75 @@
+
+import os
+import sys
+from typing import Any
+from typing import Generator
+
+import pytest
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from src.apis.base import api_router
+from src.db.models.base import Base
+from src.db.session import get_db
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+# this is to include backend dir in sys.path so that we can import from db,main.py
+
+
+
+def start_application():
+    app = FastAPI()
+    app.include_router(api_router)
+    return app
+
+
+SQLALCHEMY_DATABASE_URL = "sqlite:///./test_db.db"
+engine = create_engine(
+    SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False}
+)
+# Use connect_args parameter only with sqlite
+SessionTesting = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+
+@pytest.fixture(scope="module")
+def app() -> Generator[FastAPI, Any, None]:
+    """
+    Create a fresh database on each test case.
+    """
+    Base.metadata.create_all(engine)  # Create the tables.
+    _app = start_application()
+    yield _app
+    Base.metadata.drop_all(engine)
+
+
+@pytest.fixture(scope="module")
+def db_session(app: FastAPI) -> Generator[SessionTesting, Any, None]:
+    connection = engine.connect()
+    transaction = connection.begin()
+    session = SessionTesting(bind=connection)
+    yield session  # use the session in tests.
+    session.close()
+    transaction.rollback()
+    connection.close()
+
+
+@pytest.fixture(scope="module")
+def client(
+    app: FastAPI, db_session: SessionTesting
+) -> Generator[TestClient, Any, None]:
+    """
+    Create a new FastAPI TestClient that uses the `db_session` fixture to override
+    the `get_db` dependency that is injected into routes.
+    """
+
+    def _get_test_db():
+        try:
+            yield db_session
+        finally:
+            pass
+
+    app.dependency_overrides[get_db] = _get_test_db
+    with TestClient(app) as client:
+        yield client

kontor-api/tests/test_main.py

@@ -1,15 +1,7 @@
 from fastapi.testclient import TestClient
-import pytest
-from src.main import kontor
-
-
-@pytest.fixture(name="client")
-def client_fixture():
-    client = TestClient(kontor)
-    yield client
 
 
 def test_get_artists(client: TestClient):
     response = client.get("/api/comic/artists")
     assert response.status_code == 200
-    assert len(response.json()) == 5
+    assert len(response.json()) == 0

kontor-scripts/config.py

@@ -25,7 +25,7 @@ def get_database_cursors(log, config: str):
         password=db_config['mariadb']['password'],
         database=db_config['mariadb']['database']
     )
-    postgres_conn = psycopg2.connect(f"host={db_config['postgres']['host']} port={db_config['postgres']['port']} user={db_config['postgres']['user']} password={db_config['postgres']['password']} dbname={db_config['postgres']['']}")
+    postgres_conn = psycopg2.connect(f"host={db_config['postgres']['host']} port={db_config['postgres']['port']} user={db_config['postgres']['user']} password={db_config['postgres']['password']} dbname={db_config['postgres']['database']}")
     return sqlite_conn, mariadb_conn, postgres_conn
 
 

kontor-scripts/export.py

@@ -28,12 +28,12 @@ if __name__ == '__main__':
     database_config = Path(dirs.user_config_dir, 'database-config.yaml')
     with open(database_config, 'rt') as f:
         db_config = yaml.safe_load(f.read())
-    connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format(
+    connect_string = ('postgresql://{}:{}@{}:{}/{}'.format(
-        db_config['mariadb']['user'],
+        db_config['postgres']['user'],
-        db_config['mariadb']['password'],
+        db_config['postgres']['password'],
-        db_config['mariadb']['host'],
+        db_config['postgres']['host'],
-        db_config['mariadb']['port'],
+        db_config['postgres']['port'],
-        db_config['mariadb']['database']
+        db_config['postgres']['database']
     ))
     engine = create_engine(connect_string)
     Base.metadata.create_all(bind=engine, checkfirst=True)

kontor-scripts/json_to_postgres.py

@@ -23,12 +23,12 @@ def copy_data(postgres_conn, data_file: Path, log):
     log.info("read json file")
     with open(data_file, 'r') as json_file:
         json_load = json.load(json_file)
+        postgres_cursor.execute("SET session_replication_role='replica'")
         for table in json_load:
             log.info(f"{table}: {len(json_load[table])}")
             # result[table] = import_table(table, json_load[table])
-            truncate_statement = 'TRUNCATE {}'.format(table)
+            truncate_statement = 'TRUNCATE {} CASCADE'.format(table)
             #log.info(f"truncate: {truncate_statement}")
-            postgres_cursor.execute("SET FOREIGN_KEY_CHECKS = 0")
             postgres_cursor.execute(truncate_statement)
             items = json_load[table]
             for item in items:
@@ -47,7 +47,7 @@ def copy_data(postgres_conn, data_file: Path, log):
                     postgres_conn.commit()
                 except psycopg2.Error as error:
                     log.info('insert failed with %s', error)
-
+        postgres_cursor.execute("SET session_replication_role='origin'")
 
 if __name__ == '__main__':
     logger = get_logger(args.verbose, args.config)

kontor-scripts/schema/admin.py

@@ -1,7 +1,6 @@
 from datetime import datetime
 
-from sqlalchemy import Column, ForeignKey, Integer, String
+from sqlalchemy import Column, ForeignKey, Integer, String, Boolean
-from sqlalchemy.dialects.mysql import BIT
 from sqlalchemy.orm import relationship, mapped_column, Mapped
 
 from .base import Base, BaseMixin
@@ -14,7 +13,7 @@ class Profile(Base, BaseMixin):
     user_name = Column(String(255), nullable=False)
     email = Column(String(255))
     password = Column(String(255))
-    enabled = Column(BIT(1))
+    enabled = Column(Boolean)
     assignments = relationship("Assignment")
     tokens = relationship("Token")
 
@@ -34,7 +33,7 @@ class Token(Base, BaseMixin):
     token = Column(String(255), nullable=False, unique=True)
     name = Column(String(255))
     last_used_date: Mapped[datetime] = mapped_column()
-    enabled = Column(BIT(1))
+    enabled = Column(Boolean)
     profile_id = Column(String(255), ForeignKey("profile.id"), nullable=False)
     profile = relationship("Profile", back_populates="tokens")
 
@@ -56,7 +55,7 @@ class Assignment(Base, BaseMixin):
 class ModuleData(Base, BaseMixin):
     __tablename__ = "module_data"
     module_name = Column(String(255), nullable=False)
-    import_data = Column(BIT(1))
+    import_data = Column(Boolean)
 
 
 class MailAccount(Base, BaseMixin):
@@ -66,7 +65,7 @@ class MailAccount(Base, BaseMixin):
     protocol = Column(String(255))
     user_name = Column(String(255))
     password = Column(String(255))
-    start_tls = Column(BIT(1))
+    start_tls = Column(Boolean)
 
 
 class Mail(Base, BaseMixin):

kontor-scripts/schema/base.py

@@ -1,8 +1,7 @@
 import uuid
 from datetime import datetime
 
-from sqlalchemy import func, Column, String
+from sqlalchemy import func, Column, String, Boolean
-from sqlalchemy.dialects.mysql import BIT
 from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
 
 
@@ -25,7 +24,7 @@ class BaseVideoMixin:
     cloud_link = Column(String(255))
     file_name = Column(String(255))
     path = Column(String(255))
-    review = Column(BIT(1))
+    review = Column(Boolean)
     title = Column(String(255))
     url = Column(String(255), unique=True)
-    should_download = Column(BIT(1))
+    should_download = Column(Boolean)

kontor-scripts/schema/comic.py

@@ -1,5 +1,4 @@
-from sqlalchemy import Column, ForeignKey, Integer, String
+from sqlalchemy import Column, ForeignKey, Integer, String, Boolean
-from sqlalchemy.dialects.mysql import BIT
 from sqlalchemy.orm import relationship
 
 from .base import Base, BaseMixin
@@ -22,8 +21,8 @@ class Comic(Base, BaseMixin):
     title = Column(String(length=255), unique=True)
     publisher_id = Column(String, ForeignKey('publisher.id'), nullable=False)
     publisher = relationship("Publisher", back_populates="comics")
-    current_order = Column(BIT(1))
+    current_order = Column(Boolean)
-    completed = Column(BIT(1))
+    completed = Column(Boolean)
     issues = relationship("Issue")
     story_arcs = relationship("StoryArc")
     trade_paperbacks = relationship("TradePaperback")
@@ -64,8 +63,8 @@ class StoryArc(Base, BaseMixin):
 class Issue(Base, BaseMixin):
     __tablename__ = "issue"
     issue_number = Column(String(255))
-    in_stock = Column(BIT(1))
+    in_stock = Column(Boolean)
-    is_read = Column(BIT(1))
+    is_read = Column(Boolean)
     comic_id = Column(String, ForeignKey("comic.id"), nullable=False)
     comic = relationship("Comic", back_populates="issues")
     volume_id = Column(String, ForeignKey("volume.id"), nullable=True)

kontor-scripts/schema/metadata.py

@@ -1,5 +1,4 @@
-from sqlalchemy import Column, String, ForeignKey, Integer
+from sqlalchemy import Column, String, ForeignKey, Integer, Boolean
-from sqlalchemy.dialects.mysql import BIT
 from sqlalchemy.orm import relationship
 
 from .base import Base, BaseMixin
@@ -28,8 +27,8 @@ class MetaDataColumn(Base, BaseMixin):
     table = relationship("MetaDataTable", back_populates="table_columns")
     column_label = Column(String(255))
     filter_label = Column(String(255))
-    is_shown = Column(BIT(1))
+    is_shown = Column(Boolean)
-    show_filter = Column(BIT(1))
+    show_filter = Column(Boolean)
     ref_column = Column(String, nullable=True)
 
     def __repr__(self):

kontor-scripts/schema/tysc.py

@@ -1,5 +1,4 @@
-from sqlalchemy import Column, Integer, String, ForeignKey, UniqueConstraint
+from sqlalchemy import Column, Integer, String, ForeignKey, UniqueConstraint, Boolean
-from sqlalchemy.dialects.mysql import BIT
 from sqlalchemy.orm import relationship
 
 from .base import Base, BaseMixin
@@ -78,8 +77,8 @@ class CardSet(Base, BaseMixin):
         UniqueConstraint("name", "vendor_id"),
     )
     name = Column(String(255), index=True)
-    parallel_set = Column(BIT(1))
+    parallel_set = Column(Boolean)
-    insert_set = Column(BIT(1))
+    insert_set = Column(Boolean)
     vendor_id = Column(String, ForeignKey("vendor.id"), nullable=False, index=True)
     vendor = relationship("Vendor", back_populates="card_sets")
     cards = relationship("Card")