tpeetz/kontor
1
0
Fork 0
Code Issues 6 Pull Requests 2 Actions Packages Projects Releases 2 Wiki Activity

update check_kontor.py to call Kontor API

Browse Source
This commit is contained in:
Thomas Peetz
2025-04-18 00:09:30 +02:00
parent a6eeea6c1f
commit 116bb77e4c
8 changed files with 168 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fastapi/app/models/media.py
-16
View File
@@ -1,16 +0,0 @@
from uuid import UUID
from pydantic import BaseModel
class MediaFileResponse(BaseModel):
id: UUID
title: str | None = None
file_name: str | None = None
cloud_link: str | None = None
url: str
review: bool = False
should_download: bool = False
class Link(BaseModel):
url: str
fastapi/app/models/media/file.py
+28
View File
@@ -0,0 +1,28 @@
from uuid import UUID
from app.schema.media import MediaFile
from pydantic import BaseModel
class MediaFileResponse(BaseModel):
id: UUID
title: str | None = None
file_name: str | None = None
cloud_link: str | None = None
url: str
review: bool = False
should_download: bool = False
class Link(BaseModel):
url: str
def get_file_details(mediafile: MediaFile) -> MediaFileResponse | None:
response = MediaFileResponse(id=mediafile.id,
title=mediafile.title,
file_name=mediafile.file_name,
cloud_link=mediafile.cloud_link,
url=str(mediafile.url),
review=(mediafile.review == 1),
should_download=(mediafile.should_download == 1))
return response
fastapi/app/routers/media.py
+10 -2
View File
@@ -5,7 +5,7 @@ from uuid import uuid4, UUID
from fastapi import APIRouter, status, HTTPException
from sqlalchemy import select, Sequence
from app.models.media import MediaFileResponse, Link
from app.models.media.file import MediaFileResponse, Link, get_file_details
from app.schema import MediaFile, SessionDep
router = APIRouter(
@@ -33,7 +33,7 @@ def update_titles(db: SessionDep) -> list[MediaFileResponse]:
@router.get("/files", response_model=List[MediaFileResponse])
def get_files(db: SessionDep, review: bool = False, download: bool = False) -> List[MediaFileResponse]:
def get_all_files(db: SessionDep, review: bool = False, download: bool = False) -> List[MediaFileResponse]:
results: list[MediaFileResponse] = []
files: Sequence[MediaFile]
if review:
@@ -53,6 +53,14 @@ def get_files(db: SessionDep, review: bool = False, download: bool = False) -> L
results.append(response)
return results
@router.get("/files/{file_id}", response_model=MediaFileResponse)
def get_file(file_id: UUID, db: SessionDep) -> MediaFileResponse:
mediafile = db.get(MediaFile, file_id)
if not mediafile:
raise HTTPException(status_code=404, detail="MediaFile could not be found")
response = get_file_details(mediafile)
return response
@router.put("/files/{file_id}", response_model=MediaFileResponse)
def update_file(file_id: UUID, db: SessionDep, info: MediaFileResponse) -> MediaFileResponse:
mediaFile = db.get(MediaFile, file_id)
scripts/check_kontor.py
+72 -50
View File
@@ -1,19 +1,18 @@
"""
Checks the database kontor
"""
from dataclasses import dataclass
from enum import Enum, auto
import mariadb
import requests
from pathlib import Path
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
import requests
from config import get_logger, get_database_cursors
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--verbose', '-v', action='count', default=0)
parser.add_argument('--config', '-c', default='kontor-docker')
parser.add_argument('--dir', '-d', default='/media/tpeetz/Media')
parser.add_argument('--config', '-c', default='kontor')
parser.add_argument('--dir', '-d', default='/data/media')
parser.add_argument('--dry-run', '-m', action='store_true')
parser.add_argument('--reset-cloud-link', '-r', action='store_true')
args = parser.parse_args()
@@ -26,93 +25,116 @@ class StatusType(Enum):
CLOUD_LINK = auto()
CLOUD_LINK_ID = auto()
@dataclass
class FileStatus:
id: str
status_type: StatusType
id: str | None = None
status_type: StatusType = StatusType.UNKNOWN
def get_response(self, response: dict):
self.status_type = StatusType.FILE_NAME
self.id = response['id']
def get_status_of_file(found_file: Path, log) -> FileStatus:
def get_status_of_file(found_file: Path, cursor, log) -> FileStatus:
status = FileStatus()
response = requests.post("http://127.0.0.1:8800/media/search")
log.info(f"Status: {response.status_code}")
data = response.json()
status.import(data)
if len(data) == 1:
status = StatusType.FILE_NAME
status.id = data['id']
try:
cursor.execute(f'SELECT id, cloud_link FROM media_file WHERE file_name="{found_file.name}"')
rows = cursor.fetchall()
if len(rows) == 1:
status.status_type = StatusType.FILE_NAME
status.id = rows[0][0]
except mariadb.Error as error:
log.debug(f'select failed with {error}')
try:
cursor.execute(f'SELECT id FROM media_file WHERE id="{found_file.stem}"')
rows = cursor.fetchall()
if len(rows) == 1:
status.status_type = StatusType.FILE_ID
status.id = rows[0][0]
if len(rows) > 1:
status.status_type = StatusType.DUPLICATE
for row in rows:
log.info(f"found {row[0]} with {found_file}")
except mariadb.Error as error:
log.debug(f'select failed with {error}')
try:
cursor.execute(f'SELECT id FROM media_file WHERE cloud_link LIKE "%{found_file.stem}%"')
rows = cursor.fetchall()
if len(rows) == 1:
status.id = rows[0][0]
if rows[0][0] == found_file.stem:
status.status_type = StatusType.CLOUD_LINK_ID
else:
status.status_type = StatusType.CLOUD_LINK
except mariadb.Error as error:
log.debug(f'select failed with {error}')
response = requests.get(f"http://127.0.0.1:8800/media/files/{found_file.stem}")
log.info(f"Status: {response.status_code}")
data = response.json()
if len(data) == 1:
status = StatusType.FILE_ID
file_id = data['id']
response = requests.get(f"http://127.0.0.1:8800/media/files?cloud_link=true")
log.info(f"Status: {response.status_code}")
data = response.json()
if len(data) == 1:
status = StatusType.CLOUD_LINK_ID
file_id = data['id']
log.debug(f"Status: {response.status_code}")
if response.status_code == 200:
status.status_type = StatusType.FILE_ID
status.id = response.json()['id']
return status
def rename_files_to_id(media_dir, dry_run, log):
def rename_files_to_id(media_dir, dry_run, conn, log):
media_path = Path(media_dir)
cursor = conn.cursor()
for file in media_path.iterdir():
log.debug('found file: {}'.format(file.name))
status = get_status_of_file(file, log)
new_file_path = file.with_name(f"{status.id}{file.suffix}")
status: FileStatus = get_status_of_file(file, cursor, log)
file_id = status.id
if not file_id:
log.info(f"ID of file {file.name} is unknown")
continue
new_file_path = file.with_name(f"{file_id}{file.suffix}")
match status.status_type:
case StatusType.FILE_NAME:
log.info(f'status of {file.name} is file_name')
rename_file(file, new_file_path, dry_run)
update_cloud_link(file_id, new_file_path, dry_run)
rename_file(file, new_file_path, dry_run, log)
update_cloud_link(file_id, new_file_path, conn, dry_run, log)
case StatusType.FILE_ID:
log.info(f'status of {file.name} is file_id')
update_cloud_link(file_id, new_file_path, dry_run)
update_cloud_link(file_id, new_file_path, conn, dry_run, log)
case StatusType.CLOUD_LINK:
log.info(f'status of {file.name} is cloud_link')
rename_file(file, new_file_path, dry_run)
update_cloud_link(file_id, new_file_path, dry_run)
rename_file(file, new_file_path, dry_run, log)
update_cloud_link(file_id, new_file_path, conn, dry_run, log)
case StatusType.CLOUD_LINK_ID:
log.debug(f'status of {file.name} is cloud_link_id')
update_cloud_link(file_id, new_file_path, dry_run)
update_cloud_link(file_id, new_file_path, conn, dry_run, log)
case StatusType.DUPLICATE:
log.info(f'status of {file.name} is duplicate')
case StatusType.UNKNOWN:
log.info(f'status of {file.name} is unknown')
case _:
log.info(f'status of {file.name} is not defined')
def rename_file(current_file, new_file_path, dry_run):
def rename_file(current_file, new_file_path, dry_run, log):
if dry_run:
logger.info('rename file {} to {}'.format(current_file.name, new_file_path.name))
log.info('rename file {} to {}'.format(current_file.name, new_file_path.name))
else:
current_file.rename(Path(new_file_path))
def update_cloud_link(file_id, file_path, conn, dry_run):
def update_cloud_link(file_id, file_path, conn, dry_run, log):
cursor = conn.cursor()
logger.debug(f'update entry {file_id} with {file_path.absolute()}')
log.debug(f'update entry {file_id} with {file_path.absolute()}')
if dry_run:
logger.info(f'UPDATE media_file: cloud_link={file_path.absolute()}')
log.debug(f'UPDATE media_file: cloud_link={file_path.absolute()}')
else:
cursor.execute('UPDATE media_file SET cloud_link="{}" WHERE id="{}"'.format(file_path.absolute(), file_id))
conn.commit()
def reset_cloud_link(conn, dry_run):
def reset_cloud_link(conn, dry_run, log):
cursor = conn.cursor()
if dry_run:
logger.info('UPDATE media_file SET cloud_link=""')
log.info('UPDATE media_file SET cloud_link=""')
else:
cursor.execute('UPDATE media_file SET cloud_link="" WHERE id is NOT NULL')
conn.commit()
if __name__ == '__main__':
logger = get_logger(args.verbose, args.config)
logger.info("kontor.check_kontor started")
logger.info("kontor.check_kontor.rename_files_to_id")
rename_files_to_id(args.dir, args.dry_run, logger)
log = get_logger(args.verbose, args.config)
log.info("kontor.check_kontor started")
_, m_conn = get_database_cursors(log, args.config)
log.info("kontor.check_kontor.rename_files_to_id")
rename_files_to_id(args.dir, args.dry_run, m_conn, log)
#logger.info("kontor.check_kontor.update_cloud_link_with_found_files")
#update_cloud_link_with_found_files(data_dir, mariadb_conn, args.dry_run)
#logger.info("kontor.check_kontor.get_ids_from_column_cloud_link")
@@ -120,5 +142,5 @@ if __name__ == '__main__':
#logger.info('found {} ids in column cloud_link'.format(len(link_list)))
#logger.info("kontor.check_kontor.checking_ids_from_cloud_link")
#checking_ids_from_cloud_link(link_list, mariadb_cursor)
logger.info("kontor.check_kontor finished")
log.info("kontor.check_kontor finished")
scripts/config.py
+6 -4
View File
@@ -44,17 +44,19 @@ def get_logger(level, config: str):
dirs = PlatformDirs(config)
logging_config = Path(dirs.user_config_dir, 'logging-config.yaml')
with open(logging_config, 'rt') as f:
config = yaml.safe_load(f.read())
logging.config.dictConfig(config)
log_config = yaml.safe_load(f.read())
logging.config.dictConfig(log_config)
logger = logging.getLogger('development')
if level is not None:
match level:
case 0:
logger.setLevel(logging.INFO)
logger.setLevel(logging.CRITICAL)
case 1:
logger.setLevel(logging.INFO)
case 2:
logger.setLevel(logging.DEBUG)
case _:
logger.setLevel(logging.CRITICAL)
logger.setLevel(logging.INFO)
return logger
scripts/export.py
+42
View File
@@ -0,0 +1,42 @@
"""
import data from json file to MariaDB
"""
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
import yaml
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from platformdirs import PlatformDirs
from pathlib import Path
from schema import Base, KontorDB
from config import get_logger
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--verbose', '-v', action='count', default=0)
parser.add_argument('--config', '-c', default='kontor-docker')
parser.add_argument('--recreate-db', action='store_true')
parser.add_argument('--file', '-f', default='~/data.json')
args = parser.parse_args()
if __name__ == '__main__':
logger = get_logger(args.verbose, args.config)
logger.info('kontor.import started')
dirs = PlatformDirs(args.config)
database_config = Path(dirs.user_config_dir, 'database-config.yaml')
with open(database_config, 'rt') as f:
db_config = yaml.safe_load(f.read())
connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format(
db_config['mariadb']['user'],
db_config['mariadb']['password'],
db_config['mariadb']['host'],
db_config['mariadb']['port'],
db_config['mariadb']['database']
))
engine = create_engine(connect_string)
Base.metadata.create_all(bind=engine, checkfirst=True)
__session__ = sessionmaker(bind=engine)
kontor_db = KontorDB(engine, logger)
kontor_db.export_db("JSON", args.file)
logger.info('kontor.import finished')
scripts/import.py
+5 -6
View File
@@ -13,16 +13,17 @@ from schema import Base, KontorDB
from config import get_logger
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--recreate-db', action='store_true')
parser.add_argument('--verbose', '-v', action='count', default=0)
parser.add_argument('--config', '-c', default='kontor-docker')
parser.add_argument('--recreate-db', action='store_true')
parser.add_argument('--file', '-f', default='~/data.json')
args = parser.parse_args()
if __name__ == '__main__':
logger = get_logger(args.verbose)
logger = get_logger(args.verbose, args.config)
logger.info('kontor.import started')
dirs = PlatformDirs("kontor")
dirs = PlatformDirs(args.config)
database_config = Path(dirs.user_config_dir, 'database-config.yaml')
with open(database_config, 'rt') as f:
db_config = yaml.safe_load(f.read())
@@ -37,7 +38,5 @@ if __name__ == '__main__':
Base.metadata.create_all(bind=engine, checkfirst=True)
__session__ = sessionmaker(bind=engine)
kontor_db = KontorDB(engine, logger)
if args.recreate_db:
kontor_db.delete_entries()
kontor_db.import_db(args.file)
kontor_db.export_db(args.file)
logger.info('kontor.import finished')
scripts/schema/database.py
+5
View File
@@ -35,6 +35,11 @@ class StatusType(Enum):
CLOUD_LINK = auto()
CLOUD_LINK_ID = auto()
class ExportType(Enum):
JSON = "JSON"
YAML = "YAML"
SQLITE = "SQLite"
class KontorDB: