diff --git a/fastapi/app/models/media.py b/fastapi/app/models/media.py deleted file mode 100644 index 5a0d30b..0000000 --- a/fastapi/app/models/media.py +++ /dev/null @@ -1,16 +0,0 @@ -from uuid import UUID - -from pydantic import BaseModel - - -class MediaFileResponse(BaseModel): - id: UUID - title: str | None = None - file_name: str | None = None - cloud_link: str | None = None - url: str - review: bool = False - should_download: bool = False - -class Link(BaseModel): - url: str diff --git a/fastapi/app/models/media/file.py b/fastapi/app/models/media/file.py new file mode 100644 index 0000000..3871a79 --- /dev/null +++ b/fastapi/app/models/media/file.py @@ -0,0 +1,28 @@ +from uuid import UUID + +from app.schema.media import MediaFile +from pydantic import BaseModel + + +class MediaFileResponse(BaseModel): + id: UUID + title: str | None = None + file_name: str | None = None + cloud_link: str | None = None + url: str + review: bool = False + should_download: bool = False + +class Link(BaseModel): + url: str + +def get_file_details(mediafile: MediaFile) -> MediaFileResponse | None: + response = MediaFileResponse(id=mediafile.id, + title=mediafile.title, + file_name=mediafile.file_name, + cloud_link=mediafile.cloud_link, + url=str(mediafile.url), + review=(mediafile.review == 1), + should_download=(mediafile.should_download == 1)) + return response + diff --git a/fastapi/app/routers/media.py b/fastapi/app/routers/media.py index ec3a450..185f8de 100644 --- a/fastapi/app/routers/media.py +++ b/fastapi/app/routers/media.py @@ -5,7 +5,7 @@ from uuid import uuid4, UUID from fastapi import APIRouter, status, HTTPException from sqlalchemy import select, Sequence -from app.models.media import MediaFileResponse, Link +from app.models.media.file import MediaFileResponse, Link, get_file_details from app.schema import MediaFile, SessionDep router = APIRouter( @@ -33,7 +33,7 @@ def update_titles(db: SessionDep) -> list[MediaFileResponse]: @router.get("/files", response_model=List[MediaFileResponse]) -def get_files(db: SessionDep, review: bool = False, download: bool = False) -> List[MediaFileResponse]: +def get_all_files(db: SessionDep, review: bool = False, download: bool = False) -> List[MediaFileResponse]: results: list[MediaFileResponse] = [] files: Sequence[MediaFile] if review: @@ -53,6 +53,14 @@ def get_files(db: SessionDep, review: bool = False, download: bool = False) -> L results.append(response) return results +@router.get("/files/{file_id}", response_model=MediaFileResponse) +def get_file(file_id: UUID, db: SessionDep) -> MediaFileResponse: + mediafile = db.get(MediaFile, file_id) + if not mediafile: + raise HTTPException(status_code=404, detail="MediaFile could not be found") + response = get_file_details(mediafile) + return response + @router.put("/files/{file_id}", response_model=MediaFileResponse) def update_file(file_id: UUID, db: SessionDep, info: MediaFileResponse) -> MediaFileResponse: mediaFile = db.get(MediaFile, file_id) diff --git a/scripts/check_kontor.py b/scripts/check_kontor.py index 72b26ae..ae512c4 100644 --- a/scripts/check_kontor.py +++ b/scripts/check_kontor.py @@ -1,19 +1,18 @@ """ Checks the database kontor """ -from dataclasses import dataclass from enum import Enum, auto - +import mariadb +import requests from pathlib import Path from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -import requests from config import get_logger, get_database_cursors parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument('--verbose', '-v', action='count', default=0) -parser.add_argument('--config', '-c', default='kontor-docker') -parser.add_argument('--dir', '-d', default='/media/tpeetz/Media') +parser.add_argument('--config', '-c', default='kontor') +parser.add_argument('--dir', '-d', default='/data/media') parser.add_argument('--dry-run', '-m', action='store_true') parser.add_argument('--reset-cloud-link', '-r', action='store_true') args = parser.parse_args() @@ -26,93 +25,116 @@ class StatusType(Enum): CLOUD_LINK = auto() CLOUD_LINK_ID = auto() -@dataclass class FileStatus: - id: str - status_type: StatusType + id: str | None = None + status_type: StatusType = StatusType.UNKNOWN + + def get_response(self, response: dict): + self.status_type = StatusType.FILE_NAME + self.id = response['id'] -def get_status_of_file(found_file: Path, log) -> FileStatus: +def get_status_of_file(found_file: Path, cursor, log) -> FileStatus: status = FileStatus() - response = requests.post("http://127.0.0.1:8800/media/search") - log.info(f"Status: {response.status_code}") - data = response.json() - status.import(data) - if len(data) == 1: - status = StatusType.FILE_NAME - status.id = data['id'] + try: + cursor.execute(f'SELECT id, cloud_link FROM media_file WHERE file_name="{found_file.name}"') + rows = cursor.fetchall() + if len(rows) == 1: + status.status_type = StatusType.FILE_NAME + status.id = rows[0][0] + except mariadb.Error as error: + log.debug(f'select failed with {error}') + try: + cursor.execute(f'SELECT id FROM media_file WHERE id="{found_file.stem}"') + rows = cursor.fetchall() + if len(rows) == 1: + status.status_type = StatusType.FILE_ID + status.id = rows[0][0] + if len(rows) > 1: + status.status_type = StatusType.DUPLICATE + for row in rows: + log.info(f"found {row[0]} with {found_file}") + except mariadb.Error as error: + log.debug(f'select failed with {error}') + try: + cursor.execute(f'SELECT id FROM media_file WHERE cloud_link LIKE "%{found_file.stem}%"') + rows = cursor.fetchall() + if len(rows) == 1: + status.id = rows[0][0] + if rows[0][0] == found_file.stem: + status.status_type = StatusType.CLOUD_LINK_ID + else: + status.status_type = StatusType.CLOUD_LINK + except mariadb.Error as error: + log.debug(f'select failed with {error}') response = requests.get(f"http://127.0.0.1:8800/media/files/{found_file.stem}") - log.info(f"Status: {response.status_code}") - data = response.json() - if len(data) == 1: - status = StatusType.FILE_ID - file_id = data['id'] - response = requests.get(f"http://127.0.0.1:8800/media/files?cloud_link=true") - log.info(f"Status: {response.status_code}") - data = response.json() - if len(data) == 1: - status = StatusType.CLOUD_LINK_ID - file_id = data['id'] + log.debug(f"Status: {response.status_code}") + if response.status_code == 200: + status.status_type = StatusType.FILE_ID + status.id = response.json()['id'] return status -def rename_files_to_id(media_dir, dry_run, log): +def rename_files_to_id(media_dir, dry_run, conn, log): media_path = Path(media_dir) + cursor = conn.cursor() for file in media_path.iterdir(): log.debug('found file: {}'.format(file.name)) - status = get_status_of_file(file, log) - new_file_path = file.with_name(f"{status.id}{file.suffix}") + status: FileStatus = get_status_of_file(file, cursor, log) file_id = status.id + if not file_id: + log.info(f"ID of file {file.name} is unknown") + continue + new_file_path = file.with_name(f"{file_id}{file.suffix}") match status.status_type: case StatusType.FILE_NAME: log.info(f'status of {file.name} is file_name') - rename_file(file, new_file_path, dry_run) - update_cloud_link(file_id, new_file_path, dry_run) + rename_file(file, new_file_path, dry_run, log) + update_cloud_link(file_id, new_file_path, conn, dry_run, log) case StatusType.FILE_ID: log.info(f'status of {file.name} is file_id') - update_cloud_link(file_id, new_file_path, dry_run) + update_cloud_link(file_id, new_file_path, conn, dry_run, log) case StatusType.CLOUD_LINK: log.info(f'status of {file.name} is cloud_link') - rename_file(file, new_file_path, dry_run) - update_cloud_link(file_id, new_file_path, dry_run) + rename_file(file, new_file_path, dry_run, log) + update_cloud_link(file_id, new_file_path, conn, dry_run, log) case StatusType.CLOUD_LINK_ID: log.debug(f'status of {file.name} is cloud_link_id') - update_cloud_link(file_id, new_file_path, dry_run) + update_cloud_link(file_id, new_file_path, conn, dry_run, log) case StatusType.DUPLICATE: log.info(f'status of {file.name} is duplicate') case StatusType.UNKNOWN: log.info(f'status of {file.name} is unknown') - case _: - log.info(f'status of {file.name} is not defined') -def rename_file(current_file, new_file_path, dry_run): +def rename_file(current_file, new_file_path, dry_run, log): if dry_run: - logger.info('rename file {} to {}'.format(current_file.name, new_file_path.name)) + log.info('rename file {} to {}'.format(current_file.name, new_file_path.name)) else: current_file.rename(Path(new_file_path)) -def update_cloud_link(file_id, file_path, conn, dry_run): +def update_cloud_link(file_id, file_path, conn, dry_run, log): cursor = conn.cursor() - logger.debug(f'update entry {file_id} with {file_path.absolute()}') + log.debug(f'update entry {file_id} with {file_path.absolute()}') if dry_run: - logger.info(f'UPDATE media_file: cloud_link={file_path.absolute()}') + log.debug(f'UPDATE media_file: cloud_link={file_path.absolute()}') else: cursor.execute('UPDATE media_file SET cloud_link="{}" WHERE id="{}"'.format(file_path.absolute(), file_id)) conn.commit() -def reset_cloud_link(conn, dry_run): +def reset_cloud_link(conn, dry_run, log): cursor = conn.cursor() if dry_run: - logger.info('UPDATE media_file SET cloud_link=""') + log.info('UPDATE media_file SET cloud_link=""') else: cursor.execute('UPDATE media_file SET cloud_link="" WHERE id is NOT NULL') conn.commit() if __name__ == '__main__': - logger = get_logger(args.verbose, args.config) - logger.info("kontor.check_kontor started") - logger.info("kontor.check_kontor.rename_files_to_id") - rename_files_to_id(args.dir, args.dry_run, logger) + log = get_logger(args.verbose, args.config) + log.info("kontor.check_kontor started") + _, m_conn = get_database_cursors(log, args.config) + log.info("kontor.check_kontor.rename_files_to_id") + rename_files_to_id(args.dir, args.dry_run, m_conn, log) #logger.info("kontor.check_kontor.update_cloud_link_with_found_files") #update_cloud_link_with_found_files(data_dir, mariadb_conn, args.dry_run) #logger.info("kontor.check_kontor.get_ids_from_column_cloud_link") @@ -120,5 +142,5 @@ if __name__ == '__main__': #logger.info('found {} ids in column cloud_link'.format(len(link_list))) #logger.info("kontor.check_kontor.checking_ids_from_cloud_link") #checking_ids_from_cloud_link(link_list, mariadb_cursor) - logger.info("kontor.check_kontor finished") + log.info("kontor.check_kontor finished") diff --git a/scripts/config.py b/scripts/config.py index 3e738e9..bf65cd5 100644 --- a/scripts/config.py +++ b/scripts/config.py @@ -44,17 +44,19 @@ def get_logger(level, config: str): dirs = PlatformDirs(config) logging_config = Path(dirs.user_config_dir, 'logging-config.yaml') with open(logging_config, 'rt') as f: - config = yaml.safe_load(f.read()) - logging.config.dictConfig(config) + log_config = yaml.safe_load(f.read()) + logging.config.dictConfig(log_config) logger = logging.getLogger('development') if level is not None: match level: case 0: - logger.setLevel(logging.INFO) + logger.setLevel(logging.CRITICAL) case 1: + logger.setLevel(logging.INFO) + case 2: logger.setLevel(logging.DEBUG) case _: - logger.setLevel(logging.CRITICAL) + logger.setLevel(logging.INFO) return logger diff --git a/scripts/export.py b/scripts/export.py new file mode 100644 index 0000000..0b730bd --- /dev/null +++ b/scripts/export.py @@ -0,0 +1,42 @@ +""" +import data from json file to MariaDB +""" +from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter + +import yaml +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from platformdirs import PlatformDirs +from pathlib import Path + +from schema import Base, KontorDB +from config import get_logger + +parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) +parser.add_argument('--verbose', '-v', action='count', default=0) +parser.add_argument('--config', '-c', default='kontor-docker') +parser.add_argument('--recreate-db', action='store_true') +parser.add_argument('--file', '-f', default='~/data.json') +args = parser.parse_args() + + +if __name__ == '__main__': + logger = get_logger(args.verbose, args.config) + logger.info('kontor.import started') + dirs = PlatformDirs(args.config) + database_config = Path(dirs.user_config_dir, 'database-config.yaml') + with open(database_config, 'rt') as f: + db_config = yaml.safe_load(f.read()) + connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format( + db_config['mariadb']['user'], + db_config['mariadb']['password'], + db_config['mariadb']['host'], + db_config['mariadb']['port'], + db_config['mariadb']['database'] + )) + engine = create_engine(connect_string) + Base.metadata.create_all(bind=engine, checkfirst=True) + __session__ = sessionmaker(bind=engine) + kontor_db = KontorDB(engine, logger) + kontor_db.export_db("JSON", args.file) + logger.info('kontor.import finished') diff --git a/scripts/import.py b/scripts/import.py index ac30a03..1de2fe4 100644 --- a/scripts/import.py +++ b/scripts/import.py @@ -13,16 +13,17 @@ from schema import Base, KontorDB from config import get_logger parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) -parser.add_argument('--recreate-db', action='store_true') parser.add_argument('--verbose', '-v', action='count', default=0) +parser.add_argument('--config', '-c', default='kontor-docker') +parser.add_argument('--recreate-db', action='store_true') parser.add_argument('--file', '-f', default='~/data.json') args = parser.parse_args() if __name__ == '__main__': - logger = get_logger(args.verbose) + logger = get_logger(args.verbose, args.config) logger.info('kontor.import started') - dirs = PlatformDirs("kontor") + dirs = PlatformDirs(args.config) database_config = Path(dirs.user_config_dir, 'database-config.yaml') with open(database_config, 'rt') as f: db_config = yaml.safe_load(f.read()) @@ -37,7 +38,5 @@ if __name__ == '__main__': Base.metadata.create_all(bind=engine, checkfirst=True) __session__ = sessionmaker(bind=engine) kontor_db = KontorDB(engine, logger) - if args.recreate_db: - kontor_db.delete_entries() - kontor_db.import_db(args.file) + kontor_db.export_db(args.file) logger.info('kontor.import finished') diff --git a/scripts/schema/database.py b/scripts/schema/database.py index 1018cd5..fd166d6 100644 --- a/scripts/schema/database.py +++ b/scripts/schema/database.py @@ -35,6 +35,11 @@ class StatusType(Enum): CLOUD_LINK = auto() CLOUD_LINK_ID = auto() +class ExportType(Enum): + JSON = "JSON" + YAML = "YAML" + SQLITE = "SQLite" + class KontorDB: