tpeetz/kontor
1
0
Fork 0
Code Issues 6 Pull Requests 2 Actions Packages Projects Releases 2 Wiki Activity

reorganize python projects

Browse Source
This commit is contained in:
Thomas Peetz
2025-04-15 01:29:08 +02:00
parent a169f6a6c1
commit 98e3d91edd
35 changed files with 136 additions and 458 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fastapi/app/routers/media.py
+20 -1
View File
@@ -13,6 +13,25 @@ router = APIRouter(
tags=["media"]
)
@router.get("/update-titles")
def update_titles() -> list[MediaFileResponse]:
results: list[MediaFileResponse] = []
with __session__() as session:
files = session.query(MediaFile).filter(MediaFile.review == 1).all()
for mediafile in files:
mediafile.update_title()
session.add(mediafile)
response = MediaFileResponse(id=mediafile.id,
title=mediafile.title,
file_name=mediafile.file_name,
cloud_link= mediafile.cloud_link,
url=str(mediafile.url),
review=(mediafile.review == 1),
should_download=(mediafile.should_download == 1))
results.append(response)
session.commit()
return results
@router.get("/files")
def get_files() -> list[MediaFileResponse]:
results: list[MediaFileResponse] = []
@@ -25,7 +44,7 @@ def get_files() -> list[MediaFileResponse]:
cloud_link= mediafile.cloud_link,
url=str(mediafile.url),
review=(mediafile.review == 1),
shoud_download=(mediafile.should_download==1))
should_download=(mediafile.should_download == 1))
results.append(response)
return results
scripts/sqlalchemy/Makefile → scripts/Makefile
View File
scripts/sqlalchemy/setup.py → scripts/__init__.py
View File
scripts/check_kontor.py
+5 -3
View File
@@ -6,10 +6,12 @@ from enum import Enum, auto
import mariadb
from pathlib import Path
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from setup import get_database_cursors, get_logger
from config import get_logger, get_database_cursors
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--verbose', '-v', action='count', default=0)
parser.add_argument('--config', '-c', default='kontor-docker')
parser.add_argument('--dir', '-d', default='/media/tpeetz/Media')
parser.add_argument('--dry-run', '-m', action='store_true')
parser.add_argument('--reset-cloud-link', '-r', action='store_true')
@@ -114,9 +116,9 @@ def reset_cloud_link(conn, dry_run):
if __name__ == '__main__':
logger = get_logger(args.verbose)
logger = get_logger(args.verbose, args.config)
logger.info("kontor.check_kontor started")
_, mariadb_conn = get_database_cursors(logger)
_, mariadb_conn = get_database_cursors(logger, args.config)
mariadb_cursor = mariadb_conn.cursor()
if args.reset_cloud_link:
reset_cloud_link(mariadb_conn, args.dry_run)
scripts/sqlalchemy/config.py → scripts/config.py
+4 -4
View File
@@ -9,8 +9,8 @@ from pathlib import Path
import yaml
def get_database_cursors(log):
dirs = PlatformDirs("kontor")
def get_database_cursors(log, config: str):
dirs = PlatformDirs(config)
database_config = Path(dirs.user_config_dir, 'database-config.yaml')
with open(database_config, 'rt') as f:
db_config = yaml.safe_load(f.read())
@@ -40,8 +40,8 @@ def create_tables(sqlite_conn, logger, recreate_db, scripts):
cursor.execute(create_statement)
def get_logger(level):
dirs = PlatformDirs("kontor")
def get_logger(level, config: str):
dirs = PlatformDirs(config)
logging_config = Path(dirs.user_config_dir, 'logging-config.yaml')
with open(logging_config, 'rt') as f:
config = yaml.safe_load(f.read())
scripts/copy_to_mariadb.py
+2 -1
View File
@@ -4,7 +4,8 @@ copy data from SQLite to MariaDB
import sqlite3
import mariadb
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from setup import get_database_cursors, create_tables, get_logger, get_scripts, get_meta_data
from config import get_logger, get_database_cursors, get_meta_data, get_scripts
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--recreate-db', action='store_true')
scripts/copy_to_sqlite.py
+1 -1
View File
@@ -4,7 +4,7 @@ copy data from MariaDB to SQLite
import sqlite3
import mariadb
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from setup import get_database_cursors, create_tables, get_logger, get_meta_data, get_scripts
from config import get_database_cursors, create_tables, get_logger, get_meta_data, get_scripts
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--recreate-db', action='store_true')
scripts/db_structure.py
+1 -1
View File
@@ -3,7 +3,7 @@ Prints the database kontor structure
"""
import mariadb
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from setup import get_database_cursors, get_logger
from config import get_database_cursors, get_logger
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--verbose', '-v', action='count', default=0)
scripts/download.py
+44 -59
View File
@@ -1,73 +1,58 @@
"""
download files with URLs from DB
"""
import re
import subprocess
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
import mariadb
from setup import get_database_cursors, create_tables, get_logger
from platformdirs import PlatformDirs
from pathlib import Path
import yaml
from sqlalchemy import create_engine, select
from sqlalchemy.orm import sessionmaker
from schema import Base, KontorDB, MediaFile
from config import get_logger
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--verbose', '-v', action='count', default=0)
parser.add_argument('--config', '-c', default='kontor-docker')
parser.add_argument('--dir', '-d', default='/data/media')
parser.add_argument('--tool', '-t', default='yt-dlp')
parser.add_argument('--dry-run', '-m', action='store_true')
parser.add_argument('--rename', '-r', action='store_true')
args = parser.parse_args()
def parse_output(lines_list, log):
file_name = ""
for line in lines_list:
if 'has already been downloaded' in line:
end_len = len(' has already been downloaded')
file_name = line[11:-end_len]
log.info('found file: "%s"', file_name)
if 'Destination' in line:
line_len = len(line)
start_len = len('[download] Destination: ')
file_len = line_len-start_len
file_name = line[-file_len:]
log.info('new file: "%s"', file_name)
return file_name
def download_url(video_url, log):
result = subprocess.run(["/home/tpeetz/bin/yt-dlp", video_url], cwd=args.dir, capture_output=True, text=True)
if result.returncode == 0:
output = result.stdout
output = re.sub(' +', ' ', output)
lines_list = output.splitlines()
return parse_output(lines_list, log)
else:
return None
def download_and_update(link, entry_id, conn):
m_cursor = conn.cursor()
filename = download_url(link, logger)
if filename is None:
update_statement = 'UPDATE media_file set review = true WHERE id = ?'
logger.debug(f'entry {entry_id} could not downloaded, set to Review')
m_cursor.execute(update_statement, (entry_id,))
else:
update_statement = 'UPDATE media_file set file_name = ?, should_download = false, review = false WHERE id = ?'
logger.debug(f'entry {entry_id} successfully downloaded, set review and should_download to false')
m_cursor.execute(update_statement, (filename, entry_id))
conn.commit()
if __name__ == '__main__':
logger = get_logger(args.verbose)
logger.info('kontor.download started')
s_conn, m_conn = get_database_cursors(logger)
cursor = m_conn.cursor()
cursor.execute('SELECT id, url FROM media_file where should_download is true')
for (link_id, url) in cursor.fetchall():
if url is None:
logger.info('There is no url for id {}'.format(link_id))
else:
if args.dry_run:
logger.info(f'download {url} for {link_id}')
else:
download_and_update(url, link_id, m_conn)
logger.info('kontor.download finished')
log = get_logger(args.verbose, args.config)
log.info('kontor.download started')
dirs = PlatformDirs(args.config)
database_config = Path(dirs.user_config_dir, 'database-config.yaml')
with open(database_config, 'rt') as f:
db_config = yaml.safe_load(f.read())
print(db_config)
connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format(
db_config['mariadb']['user'],
db_config['mariadb']['password'],
db_config['mariadb']['host'],
db_config['mariadb']['port'],
db_config['mariadb']['database']
))
engine = create_engine(connect_string)
Base.metadata.create_all(bind=engine, checkfirst=True)
__session__ = sessionmaker(bind=engine)
_filter = {'should_download': 1}
with __session__() as session:
files = session.query(MediaFile).filter_by(**_filter).all()
log.info("found %d entries", len(files))
files2 = session.query(MediaFile).filter(MediaFile.should_download == 1).all()
log.info("found %d entries", len(files2))
for mediafile in files2:
mediafile.download_file(download_dir=args.dir, dl_tool="yt-dlp")
log.info("Datei {} erfolgreich heruntergeladen".format(mediafile.file_name))
if args.rename:
current_file = Path(mediafile.file_name)
new_file_path = current_file.with_name(f"{mediafile.id}{current_file.suffix}")
current_file.rename(Path(new_file_path))
mediafile.cloud_link = new_file_path
session.add(mediafile)
session.commit()
log.info('kontor.download finished')
scripts/sqlalchemy/import.py → scripts/import.py
+1 -1
View File
@@ -10,7 +10,7 @@ from platformdirs import PlatformDirs
from pathlib import Path
from schema import Base, KontorDB
from config import get_database_cursors, create_tables, get_logger, get_scripts, get_meta_data
from config import get_logger
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--recreate-db', action='store_true')
scripts/sqlalchemy/kontor.py → scripts/kontor.py
View File
scripts/sqlalchemy/pyproject.toml → scripts/pyproject.toml
View File
scripts/sqlalchemy/requirements-dev.txt → scripts/requirements-dev.txt
View File
scripts/sqlalchemy/requirements.txt → scripts/requirements.txt
View File
scripts/sqlalchemy/schema/__init__.py → scripts/schema/__init__.py
+1
View File
@@ -6,3 +6,4 @@ from .tysc import Card, CardSet, Sport, Team, FieldPosition, Rooster, Player, Ve
from .media import MediaFile, MediaArticle, MediaVideo
from .base import Base
from .database import KontorDB, ColumnEntry
scripts/sqlalchemy/schema/admin.py → scripts/schema/admin.py
View File
scripts/sqlalchemy/schema/base.py → scripts/schema/base.py
View File
scripts/sqlalchemy/schema/bookshelf.py → scripts/schema/bookshelf.py
View File
scripts/sqlalchemy/schema/comic.py → scripts/schema/comic.py
View File
scripts/sqlalchemy/schema/database.py → scripts/schema/database.py
+3 -2
View File
@@ -6,7 +6,7 @@ from logging import Logger
from pathlib import Path
from typing import Any
from sqlalchemy import select
from sqlalchemy import UUID, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import sessionmaker
@@ -347,6 +347,7 @@ class KontorDB:
_filter = { 'review': True}
with __session__() as session:
links = session.query(MediaFile).filter_by(**_filter).all()
self.log.info("%d entries found for updating titles", len(links))
for link in links:
url = link.url
if url is None:
@@ -356,7 +357,7 @@ class KontorDB:
update_list[link.id] = link.title
return update_list
def get_download_list(self) -> list:
def get_download_list(self) -> list[UUID]:
download_list = []
__session__ = sessionmaker(self.engine)
_filter = { 'should_download': True}
scripts/sqlalchemy/schema/media.py → scripts/schema/media.py
View File
scripts/sqlalchemy/schema/metadata.py → scripts/schema/metadata.py
View File
scripts/sqlalchemy/schema/tysc.py → scripts/schema/tysc.py
View File
scripts/setup.py
-116
View File
@@ -1,116 +0,0 @@
"""
Setup database connections
"""
import sqlite3
import mariadb
import logging.config
from platformdirs import PlatformDirs
from pathlib import Path
import yaml
def get_database_cursors(log):
dirs = PlatformDirs("kontor")
database_config = Path(dirs.user_config_dir, 'database-config.yaml')
with open(database_config, 'rt') as f:
db_config = yaml.safe_load(f.read())
sqlite_db = db_config["sqlite"]["file"]
log.info('using SQLite3 database {}'.format(sqlite_db))
sqlite_conn = sqlite3.connect(sqlite_db, detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES)
mariadb_conn = mariadb.connect(
host=db_config['mariadb']['host'],
port=db_config['mariadb']['port'],
user=db_config['mariadb']['user'],
password=db_config['mariadb']['password'],
database=db_config['mariadb']['database']
)
return sqlite_conn, mariadb_conn
def create_tables(sqlite_conn, logger, recreate_db, scripts):
logger.info('create_tables')
for table_id in scripts:
create_statement = scripts[table_id]['create']
drop_statement = scripts[table_id]['drop']
logger.debug(create_statement)
cursor = sqlite_conn.cursor()
if recreate_db:
logger.debug(drop_statement)
cursor.execute(drop_statement)
cursor.execute(create_statement)
def get_logger(level):
dirs = PlatformDirs("kontor")
logging_config = Path(dirs.user_config_dir, 'logging-config.yaml')
with open(logging_config, 'rt') as f:
config = yaml.safe_load(f.read())
logging.config.dictConfig(config)
logger = logging.getLogger('development')
if level is not None:
match level:
case 0:
logger.setLevel(logging.INFO)
case 1:
logger.setLevel(logging.DEBUG)
case _:
logger.setLevel(logging.CRITICAL)
return logger
def get_meta_data(mariadb_conn):
mariadb_cursor = mariadb_conn.cursor()
select_statement = "SELECT id, table_name FROM meta_data_table"
mariadb_cursor.execute(select_statement)
rows = mariadb_cursor.fetchall()
meta_data = {}
for (identifier, table_name) in rows:
table_data = {"name": table_name}
mariadb_cursor.execute("SELECT column_name, column_sync_name, column_type, column_modifier, column_order FROM meta_data_column WHERE table_id=?", (identifier, ))
column_rows = mariadb_cursor.fetchall()
column_list = []
for (column_name, column_sync_name, column_type, column_modifier, column_order) in column_rows:
column_data = {"column_name": column_name, "column_sync_name": column_sync_name, "column_type": column_type,
"column_modifier": column_modifier, "column_order": column_order}
column_list.append(column_data)
# logger.info(column_list)
table_data["columns"] = column_list
meta_data[identifier] = table_data
return meta_data
def get_scripts(meta_data, logger):
scripts_map = {}
for table_id in meta_data:
table_scripts = {}
m_columns = []
s_columns = []
columns = []
for column_data in meta_data[table_id]["columns"]:
column_line = "{} {}".format(column_data["column_sync_name"], column_data["column_type"])
if column_data["column_modifier"]:
column_line += " " + column_data["column_modifier"]
columns.append(column_line)
m_columns.append(column_data['column_name'])
s_columns.append(column_data['column_sync_name'])
table_name = meta_data[table_id]["name"]
create_statement = "CREATE TABLE IF NOT EXISTS {} ({});".format(table_name, ", ".join(columns))
drop_statement = 'DROP TABLE IF EXISTS {}'.format(table_name)
select_mariadb_statement = 'SELECT {} FROM {}'.format(', '.join(m_columns), table_name)
select_sqlite_statement = 'SELECT {} FROM {}'.format(', '.join(s_columns), table_name)
insert_sqlite_statement = 'INSERT INTO {}({}) VALUES({})'.format(table_name, ', '.join(s_columns), ', '.join(['?']*len(s_columns)))
insert_mariadb_statement = 'INSERT INTO {}({}) VALUES({})'.format(table_name, ', '.join(m_columns), ', '.join(['?']*len(m_columns)))
truncate_mariadb_statement = 'TRUNCATE {}'.format(table_name)
#logger.debug(create_statement)
#logger.debug(select_mariadb_statement)
table_scripts["create"] = create_statement
table_scripts["drop"] = drop_statement
table_scripts["select_mariadb"] = select_mariadb_statement
table_scripts["select_sqlite"] = select_sqlite_statement
table_scripts["insert_sqlite"] = insert_sqlite_statement
table_scripts["insert_mariadb"] = insert_mariadb_statement
table_scripts["truncate_mariadb"] = truncate_mariadb_statement
table_scripts["count"] = "SELECT COUNT(*) FROM {}".format(table_name)
table_scripts["name"] = table_name
scripts_map[table_id] = table_scripts
return scripts_map
scripts/sqlmodel/Makefile
-31
View File
@@ -1,31 +0,0 @@
.PHONY: clean virtualenv test docker dist dist-upload
clean:
find . -name '*.py[co]' -delete
virtualenv:
virtualenv --prompt '|> kontor <| ' env
env/bin/pip install -r requirements-dev.txt
env/bin/python setup.py develop
@echo
@echo "VirtualENV Setup Complete. Now run: source env/bin/activate"
@echo
test:
python -m pytest \
-v \
--cov=kontor \
--cov-report=term \
--cov-report=html:coverage-report \
tests/
docker: clean
docker build -t kontor-api:latest .
dist: clean
rm -rf dist/*
python setup.py sdist
python setup.py bdist_wheel
dist-upload:
twine upload dist/*
scripts/sqlmodel/kontor.py
-59
View File
@@ -1,59 +0,0 @@
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from pathlib import Path
import logging.config
import yaml
from platformdirs import PlatformDirs
from sqlmodel import SQLModel, create_engine, Session, select
from model import Comic, Publisher
from model.media import MediaFile
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--recreate-db', action='store_true')
parser.add_argument('--verbose', '-v', action='count', default=0)
parser.add_argument('--file', '-f', default='~/data.json')
parser.add_argument('--config', '-c', default='kontor')
args = parser.parse_args()
def get_logger(level: int, config: str):
dirs = PlatformDirs(config)
logging_config = Path(dirs.user_config_dir, 'logging-config.yaml')
with open(logging_config, 'rt') as f:
configDict = yaml.safe_load(f.read())
logging.config.dictConfig(configDict)
logger = logging.getLogger('development')
if level is not None:
match level:
case 0:
logger.setLevel(logging.INFO)
case 1:
logger.setLevel(logging.DEBUG)
case _:
logger.setLevel(logging.CRITICAL)
return logger
if __name__ == '__main__':
log = get_logger(args.verbose, args.config)
log.info('kontor started')
dirs = PlatformDirs(args.config)
database_config = Path(dirs.user_config_dir, 'database-config.yaml')
with open(database_config, 'rt') as f:
db_config = yaml.safe_load(f.read())
print(db_config)
connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format(
db_config['mariadb']['user'],
db_config['mariadb']['password'],
db_config['mariadb']['host'],
db_config['mariadb']['port'],
db_config['mariadb']['database']
))
engine = create_engine(connect_string, echo=True)
SQLModel.metadata.create_all(engine)
with Session(engine) as session:
files = session.exec(select(MediaFile)).all()
for file in files:
print("{} {}".format(file, file.actors))
log.info('kontor finished')
scripts/sqlmodel/model/__init__.py
-1
View File
@@ -1 +0,0 @@
from .comic import Comic, Publisher, ComicWork, Artist, Worktype
scripts/sqlmodel/model/base.py
-11
View File
@@ -1,11 +0,0 @@
import uuid
from datetime import datetime
from sqlmodel import SQLModel, Field
class AbstractEntity(SQLModel, table=False):
id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True)
created_date: datetime = Field(default_factory=datetime.now, nullable=False)
last_modified_date: datetime = Field(default_factory=datetime.now, nullable=False)
version: int = Field(default=0)
scripts/sqlmodel/model/comic.py
-58
View File
@@ -1,58 +0,0 @@
import uuid
from datetime import datetime
from sqlmodel import Field, Relationship, SQLModel
from model.base import AbstractEntity
class Publisher(AbstractEntity, table=True):
name: str = Field(index=True, unique=True)
comics: list["Comic"] = Relationship(back_populates="publisher")
def __repr__(self):
return f'Publisher({self.id} {self.name})'
def __str__(self):
return self.__repr__()
class Comic(AbstractEntity, table=True):
title: str = Field(index=True, unique=True)
publisher_id: uuid.UUID | None = Field(default=None, foreign_key="publisher.id")
publisher: Publisher | None = Relationship(back_populates="comics")
current_order: int = Field(default=False)
completed: int = Field(nullable=False)
#issues: list["Issue"] = Relationship(back_populates="comic")
#story_arcs: list["StoryArc"] = Relationship(back_populates="comic")
#trade_paperbacks: list["TradePaperback"] = Relationship(back_populates="comic")
#volumes: list["Volume"] = Relationship(back_populates="comic")
#comic_works: list["ComicWork"] = Relationship(back_populates="comic")
class Artist(AbstractEntity, table=True):
name: str = Field(nullable=False)
comic_works: list["ComicWork"] = Relationship(back_populates="artist")
class Worktype(AbstractEntity, table=True):
name: str = Field(nullable=False, unique=True)
#comic_works: list["ComicWork"] = Relationship(back_populates="worktype")
def __repr__(self):
return f'Worktype({self.id} {self.version} {self.name} {len(self.comic_works)})'
def __str__(self):
return f'{self.name}({self.id})'
class ComicWork(AbstractEntity, table=True):
__tablename__ = "comic_work"
comic_id: uuid.UUID | None = Field(nullable=False, foreign_key="comic.id")
#comic: Comic = Relationship(back_populates="comic_works")
artist_id: uuid.UUID | None = Field(nullable=False, foreign_key="artist.id")
artist: Artist = Relationship(back_populates="comic_works")
work_type_id: uuid.UUID | None = Field(nullable=False, foreign_key="worktype.id")
#worktype = Relationship(back_populates="comic_works")
scripts/sqlmodel/model/media.py
-28
View File
@@ -1,28 +0,0 @@
from sqlmodel import Field, Relationship, table
from uuid import UUID
from .base import AbstractEntity
class MediaActorFile(AbstractEntity, table=True):
__tablename__ = "media_actor_file"
media_actor_id: UUID = Field(nullable=False, foreign_key="media_actor.id")
media_file_id: UUID = Field(nullable=False, foreign_key="media_file.id")
class MediaFile(AbstractEntity, table=True):
__tablename__ = "media_file"
cloud_link: str = Field(nullable=True, max_length=255)
file_name: str = Field(nullable=True, max_length=255)
path : str = Field(nullable=True, max_length=255)
title: str = Field(nullable=True, max_length=255)
url: str = Field(nullable=True, max_length=255)
actors : list["MediaActor"] = Relationship(back_populates="videos", link_model=MediaActorFile)
class MediaActor(AbstractEntity, table=True):
__tablename__ = "media_actor"
name: str = Field(nullable=True, max_length=255)
videos : list["MediaFile"] = Relationship(back_populates="actors", link_model=MediaActorFile)
scripts/sqlmodel/pyproject.toml
-29
View File
@@ -1,29 +0,0 @@
[build-system]
requires = ["setuptools"]
[project]
name = "kontor-cli"
version = "0.1.0"
dependencies = [
"mariadb",
"sqlmodel",
"pathlib",
"platformdirs",
"pyyaml",
"beautifulsoup4",
]
requires-python = ">=3.10"
authors = [
{name = "Thomas Peetz", email = "thomas.peetz@thpeetz.de"}
]
maintainers = [
{name = "Thomas Peetz", email = "thomas.peetz@thpeetz.de"}
]
description = "CLI for Kontor application"
readme = "README.md"
classifiers = [
"Development Status :: 4 - Beta",
"Programming Language :: Python"
]
[project-scripts]
kontor = "kontor::main"
scripts/sqlmodel/requirements-dev.txt
-8
View File
@@ -1,8 +0,0 @@
-r requirements.txt
pytest
pytest-cov
coverage
twine>=1.11.0
setuptools>=38.6.0
wheel>=0.31.0
scripts/sqlmodel/requirements.txt
-7
View File
@@ -1,7 +0,0 @@
mariadb
sqlalchemy
pathlib
platformdirs
pyyaml
beautifulsoup4
sqlmodel
scripts/sqlmodel/setup.py
View File
scripts/update_title.py
+54 -37
View File
@@ -1,49 +1,66 @@
"""
download files with URLs from DB
"""
import re
import subprocess
import datetime
import logging
import mariadb
import requests
from bs4 import BeautifulSoup
import logging.config
import yaml
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
import mariadb
from setup import get_database_cursors, get_logger
from pathlib import Path
from platformdirs import PlatformDirs
from sqlalchemy import create_engine, select
from sqlalchemy.orm import sessionmaker
from schema import MediaFile, Base
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--verbose', '-v', action='count', default=0)
parser.add_argument('--config', '-c', default='kontor-docker')
args = parser.parse_args()
def get_logger(level: int, config: str):
dirs = PlatformDirs(config)
logging_config = Path(dirs.user_config_dir, 'logging-config.yaml')
with open(logging_config, 'rt') as f:
configDict = yaml.safe_load(f.read())
logging.config.dictConfig(configDict)
logger = logging.getLogger('development')
if level is not None:
match level:
case 0:
logger.setLevel(logging.INFO)
case 1:
logger.setLevel(logging.DEBUG)
case _:
logger.setLevel(logging.CRITICAL)
return logger
if __name__ == '__main__':
logger = get_logger(args.verbose)
logger.info('kontor.download started')
s_conn, m_conn = get_database_cursors(logger)
cursor = m_conn.cursor()
cursor.execute('SELECT id, url FROM media_file where review is true')
for (link_id, url) in cursor.fetchall():
if url is None:
logger.info('There is no url for id {}'.format(link_id))
else:
logger.info('get title for url {}'.format(url))
try:
r = requests.get(url)
soup = BeautifulSoup(r.content, "html.parser")
title = soup.title.string
except:
logger.info("Sorry, could not retrieve title")
update_statement = 'UPDATE media_file set review = true WHERE id = ?'
cursor.execute(update_statement, (link_id, ))
logger.info('ID {} has title {}'.format(link_id, title))
update = 'UPDATE media_file SET title = ?, review= False where id= ?'
try:
cursor.execute(update, (title, link_id))
logger.info('entry {} updated'.format(link_id))
except mariadb.Error as error:
logger.info(error)
m_conn.commit()
logger.info('kontor.download finished')
log = get_logger(args.verbose, args.config)
log.info('kontor.update_titles started')
dirs = PlatformDirs(args.config)
database_config = Path(dirs.user_config_dir, 'database-config.yaml')
with open(database_config, 'rt') as f:
db_config = yaml.safe_load(f.read())
print(db_config)
connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format(
db_config['mariadb']['user'],
db_config['mariadb']['password'],
db_config['mariadb']['host'],
db_config['mariadb']['port'],
db_config['mariadb']['database']
))
engine = create_engine(connect_string)
Base.metadata.create_all(bind=engine, checkfirst=True)
__session__ = sessionmaker(engine)
_filter = {'review': 1}
with __session__() as session:
files = session.query(MediaFile).filter_by(**_filter).all()
log.info("found %d entries", len(files))
files2 = session.query(MediaFile).filter(MediaFile.review ==1).all
log.info("found %d entries", len(files2))
for mediafile in files:
mediafile.update_title()
session.add(mediafile)
session.commit()
log.info("found %d entries", len(files))
log.info('kontor.update_titles finished')