diff --git a/python/kontor-cli.backup/.gitignore b/python/kontor-cli.backup/.gitignore deleted file mode 100644 index a74b246..0000000 --- a/python/kontor-cli.backup/.gitignore +++ /dev/null @@ -1,105 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -coverage-report/ -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ diff --git a/python/kontor-cli.backup/CHANGELOG.md b/python/kontor-cli.backup/CHANGELOG.md deleted file mode 100644 index 10e1f8a..0000000 --- a/python/kontor-cli.backup/CHANGELOG.md +++ /dev/null @@ -1,5 +0,0 @@ -# Kontor CLI Change History - -## 0.0.1 - -Initial release. diff --git a/python/kontor-cli.backup/Dockerfile b/python/kontor-cli.backup/Dockerfile deleted file mode 100644 index d32cf5c..0000000 --- a/python/kontor-cli.backup/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM python:3.9-alpine -LABEL MAINTAINER="Thomas Peetz " -ENV PS1="\[\e[0;33m\]|> kontor <| \[\e[1;35m\]\W\[\e[0m\] \[\e[0m\]# " - -WORKDIR /src -COPY . /src -RUN pip install --no-cache-dir -r requirements.txt \ - && python setup.py install -WORKDIR / -ENTRYPOINT ["kontor"] diff --git a/python/kontor-cli.backup/LICENSE.md b/python/kontor-cli.backup/LICENSE.md deleted file mode 100644 index 8b13789..0000000 --- a/python/kontor-cli.backup/LICENSE.md +++ /dev/null @@ -1 +0,0 @@ - diff --git a/python/kontor-cli.backup/MANIFEST.in b/python/kontor-cli.backup/MANIFEST.in deleted file mode 100644 index 1160952..0000000 --- a/python/kontor-cli.backup/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -recursive-include *.py -include setup.cfg -include README.md CHANGELOG.md LICENSE.md -include *.txt -recursive-include kontor/templates * diff --git a/python/kontor-cli.backup/Makefile b/python/kontor-cli.backup/Makefile deleted file mode 100644 index b016c3c..0000000 --- a/python/kontor-cli.backup/Makefile +++ /dev/null @@ -1,31 +0,0 @@ -.PHONY: clean virtualenv test docker dist dist-upload - -clean: - find . -name '*.py[co]' -delete - -virtualenv: - virtualenv --prompt '|> kontor <| ' env - env/bin/pip install -r requirements-dev.txt - env/bin/python setup.py develop - @echo - @echo "VirtualENV Setup Complete. Now run: source env/bin/activate" - @echo - -test: - python -m pytest \ - -v \ - --cov=kontor \ - --cov-report=term \ - --cov-report=html:coverage-report \ - tests/ - -docker: clean - docker build -t kontor:latest . - -dist: clean - rm -rf dist/* - python setup.py sdist - python setup.py bdist_wheel - -dist-upload: - twine upload dist/* diff --git a/python/kontor-cli.backup/README.md b/python/kontor-cli.backup/README.md deleted file mode 100644 index a95f4ea..0000000 --- a/python/kontor-cli.backup/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# Kontor CLI - -## Installation - -``` -$ pip install -r requirements.txt - -$ python setup.py install -``` - -## Development - -This project includes a number of helpers in the `Makefile` to streamline common development tasks. - -### Environment Setup - -The following demonstrates setting up and working with a development environment: - -``` -### create a virtualenv for development - -$ make virtualenv - -$ source env/bin/activate - - -### run kontor cli application - -$ kontor --help - - -### run pytest / coverage - -$ make test -``` - - -### Releasing to PyPi - -Before releasing to PyPi, you must configure your login credentials: - -**~/.pypirc**: - -``` -[pypi] -username = YOUR_USERNAME -password = YOUR_PASSWORD -``` - -Then use the included helper function via the `Makefile`: - -``` -$ make dist - -$ make dist-upload -``` - -## Deployments - -### Docker - -Included is a basic `Dockerfile` for building and distributing `Kontor CLI`, -and can be built with the included `make` helper: - -``` -$ make docker - -$ docker run -it kontor --help -``` diff --git a/python/kontor-cli.backup/config/kontor.yml.example b/python/kontor-cli.backup/config/kontor.yml.example deleted file mode 100644 index 6806151..0000000 --- a/python/kontor-cli.backup/config/kontor.yml.example +++ /dev/null @@ -1,46 +0,0 @@ -### Kontor CLI Configuration Settings ---- - -kontor: - -### Toggle application level debug (does not toggle framework debugging) -# debug: false - -### Where external (third-party) plugins are loaded from -# plugin_dir: /var/lib/kontor/plugins/ - -### Where all plugin configurations are loaded from -# plugin_config_dir: /etc/kontor/plugins.d/ - -### Where external templates are loaded from -# template_dir: /var/lib/kontor/templates/ - -### The log handler label -# log_handler: colorlog - -### The output handler label -# output_handler: jinja2 - -### sample foo option -# foo: bar - - -log.colorlog: - -### Where the log file lives (no log file by default) -# file: null - -### The level for which to log. One of: info, warning, error, fatal, debug -# level: info - -### Whether or not to log to console -# to_console: true - -### Whether or not to rotate the log file when it reaches `max_bytes` -# rotate: false - -### Max size in bytes that a log file can grow until it is rotated. -# max_bytes: 512000 - -### The maximum number of log files to maintain when rotating -# max_files: 4 diff --git a/python/kontor-cli.backup/docs/.gitkeep b/python/kontor-cli.backup/docs/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/python/kontor-cli.backup/kontor/__init__.py b/python/kontor-cli.backup/kontor/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/python/kontor-cli.backup/kontor/controllers/__init__.py b/python/kontor-cli.backup/kontor/controllers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/python/kontor-cli.backup/kontor/controllers/base.py b/python/kontor-cli.backup/kontor/controllers/base.py deleted file mode 100644 index 90153a3..0000000 --- a/python/kontor-cli.backup/kontor/controllers/base.py +++ /dev/null @@ -1,35 +0,0 @@ - -from cement import Controller, ex -from cement.utils.version import get_version_banner -from ..core.version import get_version - -VERSION_BANNER = """ -Kontor CLI %s -%s -""" % (get_version(), get_version_banner()) - - -class Base(Controller): - class Meta: - label = 'base' - - # text displayed at the top of --help output - description = 'Kontor CLI Tool' - - # text displayed at the bottom of --help output - epilog = 'Usage: kontor (database | media) [subcommands]' - - # controller level arguments. ex: 'kontor --version' - arguments = [ - ### add a version banner - (['-v', '--version'], - {'action': 'version', - 'version': VERSION_BANNER}), - (['-m', '--dry-run'], - {'action': 'store_true', - 'dest': 'dry_run'}) - ] - - def _default(self): - """Default action if no sub-command is passed.""" - self.app.args.print_help() diff --git a/python/kontor-cli.backup/kontor/controllers/database.py b/python/kontor-cli.backup/kontor/controllers/database.py deleted file mode 100644 index 5917d0a..0000000 --- a/python/kontor-cli.backup/kontor/controllers/database.py +++ /dev/null @@ -1,51 +0,0 @@ -from cement import Controller, ex -from kontor_schema import KontorDB - - -class Database(Controller): - - class Meta: - label = 'database' - stacked_type = 'nested' - stacked_on = 'base' - - @ex( - help='export database to given file', - arguments=[ - (['-f', '--file'], - {'help': 'file to store database content', - 'action': 'store', - 'dest': 'db_file'}) - ], - ) - def export(self): - data = { - 'db_file': 'data.json', - 'export_type': 'JSON', - } - if self.app.pargs.db_file is not None: - data['db_file'] = self.app.pargs.db_file - kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log) - kontor_db.export_db(data['export_type'], data['db_file']) - self.app.render(data, 'command1.jinja2') - - @ex( - label='import', - help='import data from file into database', - arguments=[ - (['-f', '--file'], - {'help': 'file to read data', - 'action': 'store', - 'dest': 'db_file'}) - ], - ) - def import_cmd(self): - data = { - 'db_file': 'data.json', - 'data_type': 'JSON', - } - if self.app.pargs.db_file is not None: - data['db_file'] = self.app.pargs.db_file - kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log) - self.app.render(data, 'import.jinja2') - kontor_db.import_db(data['db_file'], self.app.pargs.dry_run) diff --git a/python/kontor-cli.backup/kontor/controllers/media.py b/python/kontor-cli.backup/kontor/controllers/media.py deleted file mode 100644 index ecee44d..0000000 --- a/python/kontor-cli.backup/kontor/controllers/media.py +++ /dev/null @@ -1,79 +0,0 @@ -from cement import Controller, ex - -from ..database import KontorDB - - -class Media(Controller): - - class Meta: - label = 'media' - stacked_type = 'nested' - stacked_on = 'base' - - @ex( - label='update', - help='update title for mediafiles', - ) - def update_title(self): - kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log) - kontor_db.update_title(self.app.pargs.dry_run) - - @ex( - label='download', - help='download all marked videos', - arguments=[ - (['-d', '--dir'], - {'help': 'directory to store videos', - 'action': 'store', - 'dest': 'media_dir'}) - ], - ) - def download(self): - data = { - 'media_dir': '/data/media', - } - if self.app.pargs.media_dir is not None: - data['media_dir'] = self.app.pargs.media_dir - kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log) - kontor_db.download_file(self.app.pargs.dry_run) - - @ex( - help='add url to database', - arguments=[ - (['-u', '--url'], - {'help': 'link to downloadable video', - 'action': 'store', - 'dest': 'link'}) - ], - ) - def add(self): - data = { - 'link_url': None - } - if self.app.pargs.link is not None: - data['link_url'] = self.app.pargs.link - if self.app.pargs.dry_run: - print(f"add url {data['link_url']} to database") - kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log) - kontor_db.add_link(self.app.pargs.link, self.app.pargs.dry_run) - - else: - print("no url was given.") - - @ex( - help='check files if existing', - arguments=[ - (['-d', '--dir'], - {'help': 'directory to store videos', - 'action': 'store', - 'dest': 'media_dir'}) - ], - ) - def check(self): - data = { - 'media_dir': '/data/media', - } - if self.app.pargs.media_dir is not None: - data['media_dir'] = self.app.pargs.media_dir - kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log) - kontor_db.check_files() diff --git a/python/kontor-cli.backup/kontor/core/__init__.py b/python/kontor-cli.backup/kontor/core/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/python/kontor-cli.backup/kontor/core/exc.py b/python/kontor-cli.backup/kontor/core/exc.py deleted file mode 100644 index 8dd63e8..0000000 --- a/python/kontor-cli.backup/kontor/core/exc.py +++ /dev/null @@ -1,4 +0,0 @@ - -class KontorCliError(Exception): - """Generic errors.""" - pass diff --git a/python/kontor-cli.backup/kontor/core/version.py b/python/kontor-cli.backup/kontor/core/version.py deleted file mode 100644 index 846814d..0000000 --- a/python/kontor-cli.backup/kontor/core/version.py +++ /dev/null @@ -1,7 +0,0 @@ - -from cement.utils.version import get_version as cement_get_version - -VERSION = (0, 1, 0, 'alpha', 0) - -def get_version(version=VERSION): - return cement_get_version(version) diff --git a/python/kontor-cli.backup/kontor/ext/__init__.py b/python/kontor-cli.backup/kontor/ext/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/python/kontor-cli.backup/kontor/main.py b/python/kontor-cli.backup/kontor/main.py deleted file mode 100644 index c8084b8..0000000 --- a/python/kontor-cli.backup/kontor/main.py +++ /dev/null @@ -1,124 +0,0 @@ - -from cement import App, TestApp, init_defaults -from cement.core.exc import CaughtSignal -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker - -from .controllers.database import Database -from .controllers.media import Media -from .core.exc import KontorCliError -from .controllers.base import Base - -# configuration defaults -CONFIG = init_defaults('kontor', 'mariadb', 'media') -CONFIG['kontor']['foo'] = 'bar' -CONFIG['mariadb']['user'] = 'kontor' -CONFIG['mariadb']['password'] = 'kontor' -CONFIG['mariadb']['host'] = '127.0.0.1' -CONFIG['mariadb']['port'] = '3306' -CONFIG['mariadb']['database'] = 'kontor' -CONFIG['media']['yt-dlp'] = '/home/tpeetz/bin/yt-dlp' -CONFIG['media']['dir'] = '/data/media' -META = init_defaults('output.json', 'output.yaml') -META['output.json']['overridable'] = True -META['output.yaml']['overridable'] = True - - -def extend_sqlalchemy(app): - app.log.info('extending kontor application with sqlalchemy') - connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format( - app.config.get('mariadb', 'user'), - app.config.get('mariadb', 'password'), - app.config.get('mariadb', 'host'), - app.config.get('mariadb', 'port'), - app.config.get('mariadb', 'database') - )) - # engine = create_engine(connect_string, echo=True) - engine = create_engine(connect_string) - Base.metadata.create_all(bind=engine) - __session__ = sessionmaker(bind=engine) - app.extend('engine', engine) - - -class KontorCli(App): - """Kontor CLI primary application.""" - - class Meta: - label = 'kontor' - - # configuration defaults - config_defaults = CONFIG - - meta_defaults = META - - # call sys.exit() on close - exit_on_close = True - - # load additional framework extensions - extensions = [ - 'yaml', - 'json', - 'colorlog', - 'jinja2', - ] - - # configuration handler - config_handler = 'yaml' - - # configuration file suffix - config_file_suffix = '.yml' - - # set the log handler - log_handler = 'colorlog' - - # set the output handler - output_handler = 'jinja2' - - hooks = [ - ('post_setup', extend_sqlalchemy), - ] - - # register handlers - handlers = [ - Base, - Database, - Media, - ] - - -class KontorCliTest(TestApp,KontorCli): - """A sub-class of KontorCli that is better suited for testing.""" - - class Meta: - label = 'kontor' - - -def main(): - with KontorCli() as app: - try: - app.run() - - except AssertionError as e: - print('AssertionError > %s' % e.args[0]) - app.exit_code = 1 - - if app.debug is True: - import traceback - traceback.print_exc() - - except KontorCliError as e: - print('KontorCliError > %s' % e.args[0]) - app.exit_code = 1 - - if app.debug is True: - import traceback - traceback.print_exc() - - except CaughtSignal as e: - # Default Cement signals are SIGINT and SIGTERM, exit 0 (non-error) - print('\n%s' % e) - app.exit_code = 0 - - -if __name__ == '__main__': - main() diff --git a/python/kontor-cli.backup/kontor/plugins/__init__.py b/python/kontor-cli.backup/kontor/plugins/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/python/kontor-cli.backup/kontor/templates/__init__.py b/python/kontor-cli.backup/kontor/templates/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/python/kontor-cli.backup/kontor/templates/command1.jinja2 b/python/kontor-cli.backup/kontor/templates/command1.jinja2 deleted file mode 100644 index 2435e4d..0000000 --- a/python/kontor-cli.backup/kontor/templates/command1.jinja2 +++ /dev/null @@ -1,4 +0,0 @@ - -Example Template (templates/command1.jinja2) - -Foo => {{ foo }} diff --git a/python/kontor-cli.backup/kontor/templates/download.jinja2 b/python/kontor-cli.backup/kontor/templates/download.jinja2 deleted file mode 100644 index a6e3d50..0000000 --- a/python/kontor-cli.backup/kontor/templates/download.jinja2 +++ /dev/null @@ -1,2 +0,0 @@ - -Download videos to directory {{ media_dir }} diff --git a/python/kontor-cli.backup/kontor/templates/import.jinja2 b/python/kontor-cli.backup/kontor/templates/import.jinja2 deleted file mode 100644 index c54f162..0000000 --- a/python/kontor-cli.backup/kontor/templates/import.jinja2 +++ /dev/null @@ -1,2 +0,0 @@ - -Import data from {{ db_file }} diff --git a/python/kontor-cli.backup/kontor/templates/update.jinja2 b/python/kontor-cli.backup/kontor/templates/update.jinja2 deleted file mode 100644 index 992873f..0000000 --- a/python/kontor-cli.backup/kontor/templates/update.jinja2 +++ /dev/null @@ -1,2 +0,0 @@ - -Update entries of mediafile diff --git a/python/kontor-cli.backup/requirements-dev.txt b/python/kontor-cli.backup/requirements-dev.txt deleted file mode 100644 index f20606e..0000000 --- a/python/kontor-cli.backup/requirements-dev.txt +++ /dev/null @@ -1,8 +0,0 @@ --r requirements.txt - -pytest -pytest-cov -coverage -twine>=1.11.0 -setuptools>=38.6.0 -wheel>=0.31.0 diff --git a/python/kontor-cli.backup/requirements.txt b/python/kontor-cli.backup/requirements.txt deleted file mode 100644 index bee2df1..0000000 --- a/python/kontor-cli.backup/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -cement==3.0.12 -cement[jinja2] -cement[yaml] -cement[colorlog] -mariadb -sqlalchemy diff --git a/python/kontor-cli.backup/setup.cfg b/python/kontor-cli.backup/setup.cfg deleted file mode 100644 index e69de29..0000000 diff --git a/python/kontor-cli.backup/setup.py b/python/kontor-cli.backup/setup.py deleted file mode 100644 index 185d9fa..0000000 --- a/python/kontor-cli.backup/setup.py +++ /dev/null @@ -1,28 +0,0 @@ - -from setuptools import setup, find_packages -from kontor.core.version import get_version - -VERSION = get_version() - -f = open('README.md', 'r') -LONG_DESCRIPTION = f.read() -f.close() - -setup( - name='kontor', - version=VERSION, - description='Kontor CLI', - long_description=LONG_DESCRIPTION, - long_description_content_type='text/markdown', - author='Thomas Peetz', - author_email='thomas.peetz@thpeetz.de', - url='https://gitlab.com/tpeetz/kontor/', - license='MIT', - packages=find_packages(exclude=['ez_setup', 'tests*']), - package_data={'kontor': ['templates/*']}, - include_package_data=True, - entry_points=""" - [console_scripts] - kontor = kontor.main:main - """, -) diff --git a/python/kontor-cli.backup/tests/conftest.py b/python/kontor-cli.backup/tests/conftest.py deleted file mode 100644 index 5124e2e..0000000 --- a/python/kontor-cli.backup/tests/conftest.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -PyTest Fixtures. -""" - -import pytest -from cement import fs - -@pytest.fixture(scope="function") -def tmp(request): - """ - Create a `tmp` object that geneates a unique temporary directory, and file - for each test function that requires it. - """ - t = fs.Tmp() - yield t - t.remove() diff --git a/python/kontor-cli.backup/tests/test_kontor.py b/python/kontor-cli.backup/tests/test_kontor.py deleted file mode 100644 index e93dd44..0000000 --- a/python/kontor-cli.backup/tests/test_kontor.py +++ /dev/null @@ -1,36 +0,0 @@ - -from pytest import raises -from kontor.main import KontorCliTest - -def test_kontor(): - # test kontor without any subcommands or arguments - with KontorCliTest() as app: - app.run() - assert app.exit_code == 0 - - -def test_kontor_debug(): - # test that debug mode is functional - argv = ['--debug'] - with KontorCliTest(argv=argv) as app: - app.run() - assert app.debug is True - - -def test_command1(): - # test command1 without arguments - argv = ['command1'] - with KontorCliTest(argv=argv) as app: - app.run() - data,output = app.last_rendered - assert data['foo'] == 'bar' - assert output.find('Foo => bar') - - - # test command1 with arguments - argv = ['command1', '--foo', 'not-bar'] - with KontorCliTest(argv=argv) as app: - app.run() - data,output = app.last_rendered - assert data['foo'] == 'not-bar' - assert output.find('Foo => not-bar') diff --git a/python/kontor-cli/kontor/controllers/base.py b/python/kontor-cli/kontor/controllers/base.py index 0f7bdda..931db26 100644 --- a/python/kontor-cli/kontor/controllers/base.py +++ b/python/kontor-cli/kontor/controllers/base.py @@ -32,29 +32,3 @@ class CliBase(Controller): """Default action if no sub-command is passed.""" self.app.args.print_help() - - - @ex( - help='example sub command1', - - # sub-command level arguments. ex: 'kontor command1 --foo bar' - arguments=[ - ### add a sample foo option under subcommand namespace - ( [ '-f', '--foo' ], - { 'help' : 'notorious foo option', - 'action' : 'store', - 'dest' : 'foo' } ), - ], - ) - def command1(self): - """Example sub-command.""" - - data = { - 'foo' : 'bar', - } - - ### do something with arguments - if self.app.pargs.foo is not None: - data['foo'] = self.app.pargs.foo - - self.app.render(data, 'command1.jinja2') diff --git a/python/kontor-cli/kontor/controllers/database.py b/python/kontor-cli/kontor/controllers/database.py index a289c99..7f2fada 100644 --- a/python/kontor-cli/kontor/controllers/database.py +++ b/python/kontor-cli/kontor/controllers/database.py @@ -8,6 +8,11 @@ class Database(Controller): label = 'database' stacked_type = 'nested' stacked_on = 'clibase' + arguments = [ + (['-m', '--dry-run'], + {'action': 'store_true', + 'dest': 'dry_run'}), + ] @ex( help='export database to given file', @@ -25,9 +30,11 @@ class Database(Controller): } if self.app.pargs.db_file is not None: data['db_file'] = self.app.pargs.db_file - kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log) - kontor_db.export_db(data['export_type'], data['db_file']) - self.app.render(data, 'command1.jinja2') + kontor_db = KontorDB(self.app.engine) + self.app.log.info(f"export DB to {data['db_file']} as {data['export_type']}") + results = kontor_db.export_db(data['export_type'], data['db_file']) + data['results'] = results + self.app.render(data, 'export_db.jinja2') @ex( label='import', @@ -46,8 +53,7 @@ class Database(Controller): } if self.app.pargs.db_file is not None: data['db_file'] = self.app.pargs.db_file - kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log) - self.app.render(data, 'import.jinja2') + kontor_db = KontorDB(self.app.engine) kontor_db.import_db(data['db_file'], self.app.pargs.dry_run) @ex( @@ -66,7 +72,7 @@ class Database(Controller): cursor.execute("SHOW TABLES") for (tablename,) in cursor.fetchall(): table_list.append(tablename) - kontor_db = KontorDB(self.app.engine, self.app.log) + kontor_db = KontorDB(self.app.engine) table_names = kontor_db.get_table_names() for table in table_list: if table not in table_names: diff --git a/python/kontor-cli/kontor/controllers/media.py b/python/kontor-cli/kontor/controllers/media.py index 7979b25..b1fd262 100644 --- a/python/kontor-cli/kontor/controllers/media.py +++ b/python/kontor-cli/kontor/controllers/media.py @@ -1,9 +1,11 @@ +from pathlib import Path + from cement import Controller, ex from kontor_schema import KontorDB from kontor_video import VideoLink -class Media(Controller): +class Media(Controller): class Meta: label = 'media' stacked_type = 'nested' @@ -14,8 +16,14 @@ class Media(Controller): help='update title for mediafiles', ) def update_title(self): - kontor_db = KontorDB(self.app.engine, self.app.log) - kontor_db.update_title(self.app.pargs.dry_run) + kontor_db = KontorDB(self.app.engine) + updates = kontor_db.get_update_list() + self.app.log.info(f"found {len(updates)} links for update") + for file_id, url in updates.items(): + link = VideoLink(url) + title = link.get_title() + if title is not None: + kontor_db.update_entry('media_file', file_id, {'title': title, 'review': 0,}) @ex( label='download', @@ -33,11 +41,26 @@ class Media(Controller): } if self.app.pargs.media_dir is not None: data['media_dir'] = self.app.pargs.media_dir - kontor_db = KontorDB(self.app.engine, self.app.log) + kontor_db = KontorDB(self.app.engine) downloads = kontor_db.get_download_list() - for download in downloads: - link = VideoLink(download, download_dir=data['media_dir']) - link.download() + self.app.log.info(f"found {len(downloads)} links for download") + for file_id, url in downloads.items(): + link = VideoLink(url) + file_name = link.download(download_dir=data['media_dir']) + if file_name is None: + kontor_db.update_entry('media_file', file_id, {'file_name': None, 'should_download': 1}) + else: + download_file = Path(file_name) + download_file.with_name(f"{file_id}{download_file.suffix}") + link.file_name = download_file.name + link.should_download = 0 + link.cloud_link = download_file.absolute() + kontor_db.update_entry('media_file', file_id, + { + 'file_name': download_file.name, + 'should_download': 0, + 'cloud_link': download_file.absolute()} + ) @ex( help='add url to database', @@ -56,9 +79,10 @@ class Media(Controller): data['link_url'] = self.app.pargs.link if self.app.pargs.dry_run: print(f"add url {data['link_url']} to database") - kontor_db = KontorDB(self.app.engine, self.app.log) - kontor_db.add_link(self.app.pargs.link, self.app.pargs.dry_run) - + else: + kontor_db = KontorDB(self.app.engine) + result = kontor_db.add_link(self.app.pargs.link) + self.log.info(result) else: print("no url was given.") @@ -77,5 +101,5 @@ class Media(Controller): } if self.app.pargs.media_dir is not None: data['media_dir'] = self.app.pargs.media_dir - kontor_db = KontorDB(self.app.engine, self.app.log) + kontor_db = KontorDB(self.app.engine) kontor_db.check_files() diff --git a/python/kontor-cli/kontor/templates/command1.jinja2 b/python/kontor-cli/kontor/templates/command1.jinja2 deleted file mode 100644 index 2435e4d..0000000 --- a/python/kontor-cli/kontor/templates/command1.jinja2 +++ /dev/null @@ -1,4 +0,0 @@ - -Example Template (templates/command1.jinja2) - -Foo => {{ foo }} diff --git a/python/kontor-cli/kontor/templates/export_db.jinja2 b/python/kontor-cli/kontor/templates/export_db.jinja2 new file mode 100644 index 0000000..678aaa3 --- /dev/null +++ b/python/kontor-cli/kontor/templates/export_db.jinja2 @@ -0,0 +1,5 @@ + Following tables were exported: + +{% for key, value in results.items() %} +Table {{key}}: {{value}} entries +{% endfor %} diff --git a/python/kontor-cli/pyproject.toml b/python/kontor-cli/pyproject.toml new file mode 100644 index 0000000..f65e841 --- /dev/null +++ b/python/kontor-cli/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "kontor-cli" +version = "0.1.0" +description = "Kontor CLI Application" +authors = [ + {name = "Thomas Peetz", email = "thomas.peetz@thpeetz.de"}, +] +maintainers = [ + {name = "Thomas Peetz", email = "thomas.peetz@thpeetz.de"}, +] +readme = "README.md" +classifiers = [ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Programming Language :: Python", + "Programming Language :: Python :: 3.10", + "Topic :: Utilities", +] + +dependencies = [ +] +requires-python = ">= 3.10" + +[projects.scripts] +kontor = "kontor.main:main" diff --git a/python/kontor-cli/requirements.txt b/python/kontor-cli/requirements.txt index 65cbadd..58833ba 100644 --- a/python/kontor-cli/requirements.txt +++ b/python/kontor-cli/requirements.txt @@ -7,3 +7,4 @@ cement[yaml] cement[colorlog] mariadb sqlalchemy +pathlib diff --git a/python/kontor-schema/kontor_schema/__init__.py b/python/kontor-schema/kontor_schema/__init__.py index a70aa6d..a26e041 100644 --- a/python/kontor-schema/kontor_schema/__init__.py +++ b/python/kontor-schema/kontor_schema/__init__.py @@ -5,7 +5,6 @@ import uuid from datetime import datetime from pathlib import Path -import requests from sqlalchemy import Engine from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import sessionmaker @@ -20,9 +19,8 @@ from .media import MediaFile, MediaArticle, MediaVideo class KontorDB: - def __init__(self, db_engine: Engine, log): + def __init__(self, db_engine: Engine): self.engine = db_engine - self.log = log self.registry = {} self.init_registry() @@ -107,7 +105,7 @@ class KontorDB: columns[column.column_name] = {"order": column.column_order, "type": column.column_type} return columns - def get_filters(self, table_name): + def get_filters(self, table_name: str) -> dict: _filter_map = {} __session__ = sessionmaker(self.engine) with __session__() as session: @@ -116,7 +114,6 @@ class KontorDB: filter(MetaDataTable.table_name == table_name). filter(MetaDataColumn.show_filter == 1).all()): _filter_map[column.column_name] = {'label': column.filter_label, 'widget': None} - self.log.debug(f"retrieved {len(_filter_map)} filters: {_filter_map}") return _filter_map def data(self, table_name: str, columns: dict, filters: dict) -> list: @@ -135,19 +132,16 @@ class KontorDB: column_name = columns[order]['column'] if str(column_name).endswith("_id"): ref_table = column_name[:-3] - # print(f"{ref_table=}") ref = getattr(entry, ref_table) value = getattr(ref, "name") - # print(f"{value=}") row.append(value) else: row.append(getattr(entry, column_name)) - # print(repr(row)) data.append(row) return data - def export_db(self, export_type: str, export_file_name: str): - self.log.info(f"export DB to {export_file_name} as {export_type}") + def export_db(self, export_type: str, export_file_name: str) -> dict: + results = {} db = {} export_table_list = self.get_table_names() for table in export_table_list: @@ -161,8 +155,6 @@ class KontorDB: with __session__() as session: rows = session.query(model).all() entries = [] - self.log.debug(f"found {len(rows)} entries") - self.log.debug(f"found {len(columns)} columns") for row in rows: # print(row) entry = {} @@ -176,11 +168,11 @@ class KontorDB: entry[column_name] = str(value) else: entry[column_name] = value - except AttributeError as error: - self.log.debug("could not get value") + except AttributeError: + pass entries.append(entry) db[table] = entries - export_file = Path(export_file_name) + results[table] = len(entries) match export_type: case "JSON": json_dump = json.dumps(db, indent=4) @@ -190,17 +182,14 @@ class KontorDB: export_file = Path(export_file_name) case "SQLite": export_file = Path(export_file_name) - case _: - self.log.debug("unknown export type") - if export_file.exists(): - self.log.debug(f"{export_file} exists") + return results - def import_db(self, import_file_name: str, dry_run: bool): + def import_db(self, import_file_name: str, dry_run: bool) -> dict: + result = {} import_file = Path(import_file_name) if not import_file.exists(): print(f"File {import_file_name} does not exist. Do nothing.") - return - self.log.debug(f"evaluate type from file extension: {import_file.suffix}") + return result match import_file.suffix: case '.json': print("read json file") @@ -208,34 +197,44 @@ class KontorDB: json_load = json.load(json_file) for table in json_load: print(f"{table}: {len(json_load[table])}") - self.import_table(table, json_load[table], dry_run) + result[table] = self.import_table(table, json_load[table], dry_run) case '.yml': print("read yaml file") case '.yaml': print("read yaml file") case '.db': print("read sqlite file") + return result - def import_table(self, table_name, items, dry_run: bool): + def import_table(self, table_name, items, dry_run: bool) -> dict: + result = {} + updated = [] + added = [] + remaining = [] existing_ids = self.get_ids(table_name) for item in items: - # self.log.debug(f"{item}") current_id = item['id'] found_item = None __session__ = sessionmaker(self.engine) with __session__() as session: found_item = session.query(self.registry[table_name]).get(current_id) - self.log.debug(f"found: {found_item}") - if found_item is not None: - changed = self.update_entry(found_item, item, dry_run) - if changed: - print(f"{current_id} has changed") - existing_ids.remove(current_id) - else: - self.log.info("item to import not found in database, add new one...") - self.add_entry(table_name, item, session, dry_run) + if found_item is not None: + changed = self.update_entry(table_name, current_id, item) + updated.append(item) + if changed: + print(f"{current_id} has changed") + updated.append(item) + existing_ids.remove(current_id) + else: + self.add_entry(table_name, item) + added.append(item) if len(existing_ids) > 0: print("remaining items") + remaining.extend(existing_ids) + result['updated'] = updated + result['added'] = added + result['remaining'] = remaining + return result def get_ids(self, table_name: str) -> list: existing_ids = [] @@ -246,36 +245,36 @@ class KontorDB: existing_ids.append(getattr(item, 'id')) return existing_ids - def add_entry(self, table_name: str, update_item: dict, session, dry_run: bool): - add_item = self.registry[table_name]() - for key in update_item.keys(): - update_value = update_item[key] - setattr(add_item, key, update_value) - if dry_run: - self.log.info(f"add item {type(add_item)} with id {update_item['id']}") - else: - session.add(add_item) - session.commit() + def add_entry(self, table_name: str, update_item: dict): + __session__ = sessionmaker(self.engine) + with __session__() as session: + add_item = self.registry[table_name]() + for key in update_item.keys(): + update_value = update_item[key] + setattr(add_item, key, update_value) + session.add(add_item) + session.commit() - def update_entry(self, existing_item, update_item: dict, dry_run: bool) -> bool: - changed = False - for key in update_item.keys(): - update_value = update_item[key] - existing_value = getattr(existing_item, key) - if type(existing_value) is not type(update_value): - # self.log.debug(f"compare {type(existing_value)} with {type(update_value)}") - existing_value = str(existing_value) - if existing_value != update_value: - print(f"{key} has changed: {existing_value} != {update_value}") - if not dry_run: + def update_entry(self, table_name, current_id, update_item: dict) -> bool: + __session__ = sessionmaker(self.engine) + with __session__() as session: + existing_item = session.query(self.registry[table_name]).get(current_id) + changed = False + for key in update_item.keys(): + update_value = update_item[key] + existing_value = getattr(existing_item, key) + if type(existing_value) is not type(update_value): + existing_value = str(existing_value) + if existing_value != update_value: + print(f"{key} has changed: {existing_value} != {update_value}") setattr(existing_item, key, update_value) - # existing_item[key] = update_value + session.commit() changed = True - self.log.info(f"update {key} with {update_value}") + print(f"update {key} with {update_value}") return changed - def add_link(self, link: str, dry_run: bool): - self.log.info(f"add link {link} to media_file") + def add_link(self, link: str) -> dict: + result = {} __session__ = sessionmaker(self.engine) with __session__() as session: media_file = MediaFile() @@ -289,125 +288,32 @@ class KontorDB: try: session.add(media_file) session.commit() - self.log.info(f"entry {media_file} successfully added") + result['added'] = media_file except IntegrityError as error: session.rollback() - self.log.info(error.orig) + result['error'] = error.orig + return result - def update_title(self, dry_run=False): - self.log.info(f"get links to review of media_file") - __session__ = sessionmaker(self.engine) - with __session__() as session: - links = session.query(MediaFile).filter(MediaFile.review == 1).all() - self.log.info(f"try to update {len(links)} items") - for link in links: - url = link.url - if url is None: - self.log.info(f"url has not been set for {link.id}") - continue - self.log.info('get title for url {}'.format(url)) - if dry_run: - continue - try: - r = requests.get(url) - soup = BeautifulSoup(r.content, "html.parser") - title = soup.title.string - except: - self.log.info("Sorry, could not retrieve title") - continue - self.log.info('ID {} has title {}'.format(link.id, title)) - link.title = title - link.review = 0 - session.commit() - - def get_update_list(self) -> list[str]: - self.log.debug("get links marked as review") - update_list = [] + def get_update_list(self) -> dict: + update_list = {} __session__ = sessionmaker(self.engine) with __session__() as session: links = session.query(MediaFile).filter(MediaFile.review == 1).all() for link in links: url = link.url if url is None: - self.log.info(f"url has not been set for {link.id}") continue - update_list.append(url) - self.log.debug(f"found {len(update_list)} urls for updates") + update_list[link.id] = url return update_list - def get_download_list(self) -> list[str]: - self.log.debug("get links marked as should_download") - download_list = [] + def get_download_list(self) -> dict: + download_list = {} __session__ = sessionmaker(self.engine) with __session__() as session: links = session.query(MediaFile).filter(MediaFile.should_download == 1).all() for link in links: url = link.url if url is None: - self.log.info(f"url has not been set for {link.id}") continue - download_list.append(url) - self.log.debug(f"found {len(download_list)} urls for downloads") + download_list[link.id] = url return download_list - - def download_file(self, dry_run=False): - self.log.info(f"download marked urls of media_file") - __session__ = sessionmaker(self.engine) - with __session__() as session: - links = session.query(MediaFile).filter(MediaFile.should_download == 1).all() - self.log.info(f"try to download {len(links)} items") - for link in links: - url = link.url - if url is None: - self.log.info(f"url has not been set for {link.id}") - continue - if dry_run: - self.log.info(f"download {link.url} to {self.config.get('media', 'dir')}") - continue - filename = self.download_url(link) - if filename is None: - link.file_name = filename - link.should_download = 1 - else: - download_file = Path(filename) - download_file.with_name(f"{link.id}{download_file.suffix}") - link.file_name = download_file.name - link.should_download = 0 - link.cloud_link = download_file.absolute() - session.commit() - - def parse_output(self, lines_list): - file_name = "" - for line in lines_list: - if 'has already been downloaded' in line: - end_len = len(' has already been downloaded') - file_name = line[11:-end_len] - self.log.info('found file: "%s"', file_name) - if 'Destination' in line: - line_len = len(line) - start_len = len('[download] Destination: ') - file_len = line_len - start_len - file_name = line[-file_len:] - self.log.info('new file: "%s"', file_name) - return file_name - - def download_url(self, video_url): - media_dir = Path(self.config.get('media', 'dir')) - if not media_dir.exists(): - media_dir = Path().absolute() - self.log.info(f"download video to {media_dir}") - result = subprocess.run([self.config.get('media', 'yt-dlp'), video_url], cwd=media_dir, capture_output=True, - text=True) - if result.returncode == 0: - output = result.stdout - output = re.sub(' +', ' ', output) - lines_list = output.splitlines() - return self.parse_output(lines_list) - else: - return None - - def check_files(self): - media_dir = Path(self.config.get('media', 'dir')) - if not media_dir.exists(): - return - self.log.info(f"check files in {media_dir}") diff --git a/python/kontor-video/kontor_video/__init__.py b/python/kontor-video/kontor_video/__init__.py index 6fd7843..8cfb548 100644 --- a/python/kontor-video/kontor_video/__init__.py +++ b/python/kontor-video/kontor_video/__init__.py @@ -1,21 +1,63 @@ +import re +import subprocess +from pathlib import Path + import requests from bs4 import BeautifulSoup class VideoLink: - def __init__(self, url: str, log): + def __init__(self, url: str, dl_tool: str, table: str): + self.file_name = None self.url = url self.title = None - self.log = log + self.dl_tool = dl_tool + self.table = table - def get_title(self): + def get_title(self) -> str: try: r = requests.get(self.url) soup = BeautifulSoup(r.content, "html.parser") title = soup.title.string except: - self.log.info("Sorry, could not retrieve title") + title = None + return title + def download(self, download_dir=None): - self.log.info(f"download {self.url} to {download_dir}") + if download_dir is None: + download_dir = Path.cwd() + result = subprocess.run([self.dl_tool, self.url], cwd=download_dir, capture_output=True, text=True) + if result.returncode == 0: + output = result.stdout + output = re.sub(' +', ' ', output) + lines_list = output.splitlines() + return self.__parse_output__(lines_list) + else: + return None + + def __parse_output__(self, lines_list): + self.file_name = "" + for line in lines_list: + if 'has already been downloaded' in line: + end_len = len(' has already been downloaded') + self.file_name = line[11:-end_len] + if 'Destination' in line: + line_len = len(line) + start_len = len('[download] Destination: ') + file_len = line_len - start_len + self.file_name = line[-file_len:] + return self.file_name + + +class MediaFile(VideoLink): + + def __init__(self, url: str, dl_tool='yt-dlp'): + super().__init__(url, dl_tool, 'media_file') + + +class MediaVideo(VideoLink): + + def __init__(self, url: str, dl_tool='yt-dlp'): + super().__init__(url, dl_tool, 'media_video')