python/kontor-cli.backup/.gitignore

@@ -1,105 +0,0 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-coverage-report/
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-.hypothesis/
-.pytest_cache/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# pyenv
-.python-version
-
-# celery beat schedule file
-celerybeat-schedule
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/

python/kontor-cli.backup/CHANGELOG.md

@@ -1,5 +0,0 @@
-# Kontor CLI Change History
-
-## 0.0.1
-
-Initial release.

python/kontor-cli.backup/Dockerfile

@@ -1,10 +0,0 @@
-FROM python:3.9-alpine
-LABEL MAINTAINER="Thomas Peetz <thomas.peetz@thpeetz.de>"
-ENV PS1="\[\e[0;33m\]|> kontor <| \[\e[1;35m\]\W\[\e[0m\] \[\e[0m\]# "
-
-WORKDIR /src
-COPY . /src
-RUN pip install --no-cache-dir -r requirements.txt \
-    && python setup.py install
-WORKDIR /
-ENTRYPOINT ["kontor"]

python/kontor-cli.backup/LICENSE.md

@@ -1 +0,0 @@
-

python/kontor-cli.backup/MANIFEST.in

@@ -1,5 +0,0 @@
-recursive-include *.py
-include setup.cfg
-include README.md CHANGELOG.md LICENSE.md
-include *.txt
-recursive-include kontor/templates *

python/kontor-cli.backup/Makefile

@@ -1,31 +0,0 @@
-.PHONY: clean virtualenv test docker dist dist-upload
-
-clean:
-	find . -name '*.py[co]' -delete
-
-virtualenv:
-	virtualenv --prompt '|> kontor <| ' env
-	env/bin/pip install -r requirements-dev.txt
-	env/bin/python setup.py develop
-	@echo
-	@echo "VirtualENV Setup Complete. Now run: source env/bin/activate"
-	@echo
-
-test:
-	python -m pytest \
-		-v \
-		--cov=kontor \
-		--cov-report=term \
-		--cov-report=html:coverage-report \
-		tests/
-
-docker: clean
-	docker build -t kontor:latest .
-
-dist: clean
-	rm -rf dist/*
-	python setup.py sdist
-	python setup.py bdist_wheel
-
-dist-upload:
-	twine upload dist/*

python/kontor-cli.backup/README.md

@@ -1,69 +0,0 @@
-# Kontor CLI
-
-## Installation
-
-```
-$ pip install -r requirements.txt
-
-$ python setup.py install
-```
-
-## Development
-
-This project includes a number of helpers in the `Makefile` to streamline common development tasks.
-
-### Environment Setup
-
-The following demonstrates setting up and working with a development environment:
-
-```
-### create a virtualenv for development
-
-$ make virtualenv
-
-$ source env/bin/activate
-
-
-### run kontor cli application
-
-$ kontor --help
-
-
-### run pytest / coverage
-
-$ make test
-```
-
-
-### Releasing to PyPi
-
-Before releasing to PyPi, you must configure your login credentials:
-
-**~/.pypirc**:
-
-```
-[pypi]
-username = YOUR_USERNAME
-password = YOUR_PASSWORD
-```
-
-Then use the included helper function via the `Makefile`:
-
-```
-$ make dist
-
-$ make dist-upload
-```
-
-## Deployments
-
-### Docker
-
-Included is a basic `Dockerfile` for building and distributing `Kontor CLI`,
-and can be built with the included `make` helper:
-
-```
-$ make docker
-
-$ docker run -it kontor --help
-```

python/kontor-cli.backup/config/kontor.yml.example

@@ -1,46 +0,0 @@
-### Kontor CLI Configuration Settings
----
-
-kontor:
-
-### Toggle application level debug (does not toggle framework debugging)
-# debug: false
-
-### Where external (third-party) plugins are loaded from
-# plugin_dir: /var/lib/kontor/plugins/
-
-### Where all plugin configurations are loaded from
-# plugin_config_dir: /etc/kontor/plugins.d/
-
-### Where external templates are loaded from
-# template_dir: /var/lib/kontor/templates/
-
-### The log handler label
-# log_handler: colorlog
-
-### The output handler label
-# output_handler: jinja2
-
-### sample foo option
-# foo: bar
-
-
-log.colorlog:
-
-### Where the log file lives (no log file by default)
-# file: null
-
-### The level for which to log.  One of: info, warning, error, fatal, debug
-# level: info
-
-### Whether or not to log to console
-# to_console: true
-
-### Whether or not to rotate the log file when it reaches `max_bytes`
-# rotate: false
-
-### Max size in bytes that a log file can grow until it is rotated.
-# max_bytes: 512000
-
-### The maximum number of log files to maintain when rotating
-# max_files: 4

python/kontor-cli.backup/kontor/controllers/base.py

@@ -1,35 +0,0 @@
-
-from cement import Controller, ex
-from cement.utils.version import get_version_banner
-from ..core.version import get_version
-
-VERSION_BANNER = """
-Kontor CLI %s
-%s
-""" % (get_version(), get_version_banner())
-
-
-class Base(Controller):
-    class Meta:
-        label = 'base'
-
-        # text displayed at the top of --help output
-        description = 'Kontor CLI Tool'
-
-        # text displayed at the bottom of --help output
-        epilog = 'Usage: kontor (database | media) [subcommands]'
-
-        # controller level arguments. ex: 'kontor --version'
-        arguments = [
-            ### add a version banner
-            (['-v', '--version'],
-             {'action': 'version',
-              'version': VERSION_BANNER}),
-            (['-m', '--dry-run'],
-             {'action': 'store_true',
-              'dest': 'dry_run'})
-        ]
-
-    def _default(self):
-        """Default action if no sub-command is passed."""
-        self.app.args.print_help()

python/kontor-cli.backup/kontor/controllers/database.py

@@ -1,51 +0,0 @@
-from cement import Controller, ex
-from kontor_schema import KontorDB
-
-
-class Database(Controller):
-
-    class Meta:
-        label = 'database'
-        stacked_type = 'nested'
-        stacked_on = 'base'
-
-    @ex(
-        help='export database to given file',
-        arguments=[
-            (['-f', '--file'],
-             {'help': 'file to store database content',
-              'action': 'store',
-              'dest': 'db_file'})
-        ],
-    )
-    def export(self):
-        data = {
-            'db_file': 'data.json',
-            'export_type': 'JSON',
-        }
-        if self.app.pargs.db_file is not None:
-            data['db_file'] = self.app.pargs.db_file
-        kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log)
-        kontor_db.export_db(data['export_type'], data['db_file'])
-        self.app.render(data, 'command1.jinja2')
-
-    @ex(
-        label='import',
-        help='import data from file into database',
-        arguments=[
-            (['-f', '--file'],
-             {'help': 'file to read data',
-              'action': 'store',
-              'dest': 'db_file'})
-        ],
-    )
-    def import_cmd(self):
-        data = {
-            'db_file': 'data.json',
-            'data_type': 'JSON',
-        }
-        if self.app.pargs.db_file is not None:
-            data['db_file'] = self.app.pargs.db_file
-        kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log)
-        self.app.render(data, 'import.jinja2')
-        kontor_db.import_db(data['db_file'], self.app.pargs.dry_run)

python/kontor-cli.backup/kontor/controllers/media.py

@@ -1,79 +0,0 @@
-from cement import Controller, ex
-
-from ..database import KontorDB
-
-
-class Media(Controller):
-
-    class Meta:
-        label = 'media'
-        stacked_type = 'nested'
-        stacked_on = 'base'
-
-    @ex(
-        label='update',
-        help='update title for mediafiles',
-    )
-    def update_title(self):
-        kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log)
-        kontor_db.update_title(self.app.pargs.dry_run)
-
-    @ex(
-        label='download',
-        help='download all marked videos',
-        arguments=[
-            (['-d', '--dir'],
-             {'help': 'directory to store videos',
-              'action': 'store',
-              'dest': 'media_dir'})
-        ],
-    )
-    def download(self):
-        data = {
-            'media_dir': '/data/media',
-        }
-        if self.app.pargs.media_dir is not None:
-            data['media_dir'] = self.app.pargs.media_dir
-        kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log)
-        kontor_db.download_file(self.app.pargs.dry_run)
-
-    @ex(
-        help='add url to database',
-        arguments=[
-            (['-u', '--url'],
-             {'help': 'link to downloadable video',
-              'action': 'store',
-              'dest': 'link'})
-        ],
-    )
-    def add(self):
-        data = {
-            'link_url': None
-        }
-        if self.app.pargs.link is not None:
-            data['link_url'] = self.app.pargs.link
-            if self.app.pargs.dry_run:
-                print(f"add url {data['link_url']} to database")
-            kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log)
-            kontor_db.add_link(self.app.pargs.link, self.app.pargs.dry_run)
-
-        else:
-            print("no url was given.")
-
-    @ex(
-        help='check files if existing',
-        arguments=[
-            (['-d', '--dir'],
-             {'help': 'directory to store videos',
-              'action': 'store',
-              'dest': 'media_dir'})
-        ],
-    )
-    def check(self):
-        data = {
-            'media_dir': '/data/media',
-        }
-        if self.app.pargs.media_dir is not None:
-            data['media_dir'] = self.app.pargs.media_dir
-        kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log)
-        kontor_db.check_files()

python/kontor-cli.backup/kontor/core/exc.py

@@ -1,4 +0,0 @@
-
-class KontorCliError(Exception):
-    """Generic errors."""
-    pass

python/kontor-cli.backup/kontor/core/version.py

@@ -1,7 +0,0 @@
-
-from cement.utils.version import get_version as cement_get_version
-
-VERSION = (0, 1, 0, 'alpha', 0)
-
-def get_version(version=VERSION):
-    return cement_get_version(version)

python/kontor-cli.backup/kontor/main.py

@@ -1,124 +0,0 @@
-
-from cement import App, TestApp, init_defaults
-from cement.core.exc import CaughtSignal
-from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker
-
-from .controllers.database import Database
-from .controllers.media import Media
-from .core.exc import KontorCliError
-from .controllers.base import Base
-
-# configuration defaults
-CONFIG = init_defaults('kontor', 'mariadb', 'media')
-CONFIG['kontor']['foo'] = 'bar'
-CONFIG['mariadb']['user'] = 'kontor'
-CONFIG['mariadb']['password'] = 'kontor'
-CONFIG['mariadb']['host'] = '127.0.0.1'
-CONFIG['mariadb']['port'] = '3306'
-CONFIG['mariadb']['database'] = 'kontor'
-CONFIG['media']['yt-dlp'] = '/home/tpeetz/bin/yt-dlp'
-CONFIG['media']['dir'] = '/data/media'
-META =  init_defaults('output.json', 'output.yaml')
-META['output.json']['overridable'] = True
-META['output.yaml']['overridable'] = True
-
-
-def extend_sqlalchemy(app):
-    app.log.info('extending kontor application with sqlalchemy')
-    connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format(
-        app.config.get('mariadb', 'user'),
-        app.config.get('mariadb', 'password'),
-        app.config.get('mariadb', 'host'),
-        app.config.get('mariadb', 'port'),
-        app.config.get('mariadb', 'database')
-    ))
-    # engine = create_engine(connect_string, echo=True)
-    engine = create_engine(connect_string)
-    Base.metadata.create_all(bind=engine)
-    __session__ = sessionmaker(bind=engine)
-    app.extend('engine', engine)
-
-
-class KontorCli(App):
-    """Kontor CLI primary application."""
-
-    class Meta:
-        label = 'kontor'
-
-        # configuration defaults
-        config_defaults = CONFIG
-
-        meta_defaults = META
-
-        # call sys.exit() on close
-        exit_on_close = True
-
-        # load additional framework extensions
-        extensions = [
-            'yaml',
-            'json',
-            'colorlog',
-            'jinja2',
-        ]
-
-        # configuration handler
-        config_handler = 'yaml'
-
-        # configuration file suffix
-        config_file_suffix = '.yml'
-
-        # set the log handler
-        log_handler = 'colorlog'
-
-        # set the output handler
-        output_handler = 'jinja2'
-
-        hooks = [
-            ('post_setup', extend_sqlalchemy),
-        ]
-
-        # register handlers
-        handlers = [
-            Base,
-            Database,
-            Media,
-        ]
-
-
-class KontorCliTest(TestApp,KontorCli):
-    """A sub-class of KontorCli that is better suited for testing."""
-
-    class Meta:
-        label = 'kontor'
-
-
-def main():
-    with KontorCli() as app:
-        try:
-            app.run()
-
-        except AssertionError as e:
-            print('AssertionError > %s' % e.args[0])
-            app.exit_code = 1
-
-            if app.debug is True:
-                import traceback
-                traceback.print_exc()
-
-        except KontorCliError as e:
-            print('KontorCliError > %s' % e.args[0])
-            app.exit_code = 1
-
-            if app.debug is True:
-                import traceback
-                traceback.print_exc()
-
-        except CaughtSignal as e:
-            # Default Cement signals are SIGINT and SIGTERM, exit 0 (non-error)
-            print('\n%s' % e)
-            app.exit_code = 0
-
-
-if __name__ == '__main__':
-    main()

python/kontor-cli.backup/kontor/templates/command1.jinja2

@@ -1,4 +0,0 @@
-
-Example Template (templates/command1.jinja2)
-
-Foo => {{ foo }}

python/kontor-cli.backup/kontor/templates/download.jinja2

@@ -1,2 +0,0 @@
-
-Download videos to directory {{ media_dir }}

python/kontor-cli.backup/kontor/templates/import.jinja2

@@ -1,2 +0,0 @@
-
-Import data from {{ db_file }}

python/kontor-cli.backup/kontor/templates/update.jinja2

@@ -1,2 +0,0 @@
-
-Update entries of mediafile

python/kontor-cli.backup/requirements-dev.txt

@@ -1,8 +0,0 @@
--r requirements.txt
-
-pytest
-pytest-cov
-coverage
-twine>=1.11.0
-setuptools>=38.6.0
-wheel>=0.31.0

python/kontor-cli.backup/requirements.txt

@@ -1,6 +0,0 @@
-cement==3.0.12
-cement[jinja2]
-cement[yaml]
-cement[colorlog]
-mariadb
-sqlalchemy

python/kontor-cli.backup/setup.py

@@ -1,28 +0,0 @@
-
-from setuptools import setup, find_packages
-from kontor.core.version import get_version
-
-VERSION = get_version()
-
-f = open('README.md', 'r')
-LONG_DESCRIPTION = f.read()
-f.close()
-
-setup(
-    name='kontor',
-    version=VERSION,
-    description='Kontor CLI',
-    long_description=LONG_DESCRIPTION,
-    long_description_content_type='text/markdown',
-    author='Thomas Peetz',
-    author_email='thomas.peetz@thpeetz.de',
-    url='https://gitlab.com/tpeetz/kontor/',
-    license='MIT',
-    packages=find_packages(exclude=['ez_setup', 'tests*']),
-    package_data={'kontor': ['templates/*']},
-    include_package_data=True,
-    entry_points="""
-        [console_scripts]
-        kontor = kontor.main:main
-    """,
-)

python/kontor-cli.backup/tests/conftest.py

@@ -1,16 +0,0 @@
-"""
-PyTest Fixtures.
-"""
-
-import pytest
-from cement import fs
-
-@pytest.fixture(scope="function")
-def tmp(request):
-    """
-    Create a `tmp` object that geneates a unique temporary directory, and file
-    for each test function that requires it.
-    """
-    t = fs.Tmp()
-    yield t
-    t.remove()

python/kontor-cli.backup/tests/test_kontor.py

@@ -1,36 +0,0 @@
-
-from pytest import raises
-from kontor.main import KontorCliTest
-
-def test_kontor():
-    # test kontor without any subcommands or arguments
-    with KontorCliTest() as app:
-        app.run()
-        assert app.exit_code == 0
-
-
-def test_kontor_debug():
-    # test that debug mode is functional
-    argv = ['--debug']
-    with KontorCliTest(argv=argv) as app:
-        app.run()
-        assert app.debug is True
-
-
-def test_command1():
-    # test command1 without arguments
-    argv = ['command1']
-    with KontorCliTest(argv=argv) as app:
-        app.run()
-        data,output = app.last_rendered
-        assert data['foo'] == 'bar'
-        assert output.find('Foo => bar')
-
-
-    # test command1 with arguments
-    argv = ['command1', '--foo', 'not-bar']
-    with KontorCliTest(argv=argv) as app:
-        app.run()
-        data,output = app.last_rendered
-        assert data['foo'] == 'not-bar'
-        assert output.find('Foo => not-bar')

python/kontor-cli/kontor/controllers/base.py

@@ -32,29 +32,3 @@ class CliBase(Controller):
         """Default action if no sub-command is passed."""
 
         self.app.args.print_help()
-
-
-    @ex(
-        help='example sub command1',
-
-        # sub-command level arguments. ex: 'kontor command1 --foo bar'
-        arguments=[
-            ### add a sample foo option under subcommand namespace
-            ( [ '-f', '--foo' ],
-              { 'help' : 'notorious foo option',
-                'action'  : 'store',
-                'dest' : 'foo' } ),
-        ],
-    )
-    def command1(self):
-        """Example sub-command."""
-
-        data = {
-            'foo' : 'bar',
-        }
-
-        ### do something with arguments
-        if self.app.pargs.foo is not None:
-            data['foo'] = self.app.pargs.foo
-
-        self.app.render(data, 'command1.jinja2')

python/kontor-cli/kontor/controllers/database.py

@@ -8,6 +8,11 @@ class Database(Controller):
         label = 'database'
         stacked_type = 'nested'
         stacked_on = 'clibase'
+        arguments = [
+            (['-m', '--dry-run'],
+             {'action': 'store_true',
+              'dest': 'dry_run'}),
+        ]
 
     @ex(
         help='export database to given file',
@@ -25,9 +30,11 @@ class Database(Controller):
         }
         if self.app.pargs.db_file is not None:
             data['db_file'] = self.app.pargs.db_file
-        kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log)
+        kontor_db = KontorDB(self.app.engine)
-        kontor_db.export_db(data['export_type'], data['db_file'])
+        self.app.log.info(f"export DB to {data['db_file']} as {data['export_type']}")
-        self.app.render(data, 'command1.jinja2')
+        results = kontor_db.export_db(data['export_type'], data['db_file'])
+        data['results'] = results
+        self.app.render(data, 'export_db.jinja2')
 
     @ex(
         label='import',
@@ -46,8 +53,7 @@ class Database(Controller):
         }
         if self.app.pargs.db_file is not None:
             data['db_file'] = self.app.pargs.db_file
-        kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log)
+        kontor_db = KontorDB(self.app.engine)
-        self.app.render(data, 'import.jinja2')
         kontor_db.import_db(data['db_file'], self.app.pargs.dry_run)
 
     @ex(
@@ -66,7 +72,7 @@ class Database(Controller):
         cursor.execute("SHOW TABLES")
         for (tablename,) in cursor.fetchall():
             table_list.append(tablename)
-        kontor_db = KontorDB(self.app.engine, self.app.log)
+        kontor_db = KontorDB(self.app.engine)
         table_names = kontor_db.get_table_names()
         for table in table_list:
             if table not in table_names:

python/kontor-cli/kontor/controllers/media.py

@@ -1,9 +1,11 @@
+from pathlib import Path
+
 from cement import Controller, ex
 from kontor_schema import KontorDB
 from kontor_video import VideoLink
 
-class Media(Controller):
 
+class Media(Controller):
     class Meta:
         label = 'media'
         stacked_type = 'nested'
@@ -14,8 +16,14 @@ class Media(Controller):
         help='update title for mediafiles',
     )
     def update_title(self):
-        kontor_db = KontorDB(self.app.engine, self.app.log)
+        kontor_db = KontorDB(self.app.engine)
-        kontor_db.update_title(self.app.pargs.dry_run)
+        updates = kontor_db.get_update_list()
+        self.app.log.info(f"found {len(updates)} links for update")
+        for file_id, url in updates.items():
+            link = VideoLink(url)
+            title = link.get_title()
+            if title is not None:
+                kontor_db.update_entry('media_file', file_id, {'title': title, 'review': 0,})
 
     @ex(
         label='download',
@@ -33,11 +41,26 @@ class Media(Controller):
         }
         if self.app.pargs.media_dir is not None:
             data['media_dir'] = self.app.pargs.media_dir
-        kontor_db = KontorDB(self.app.engine, self.app.log)
+        kontor_db = KontorDB(self.app.engine)
         downloads = kontor_db.get_download_list()
-        for download in downloads:
+        self.app.log.info(f"found {len(downloads)} links for download")
-            link = VideoLink(download, download_dir=data['media_dir'])
+        for file_id, url in downloads.items():
-            link.download()
+            link = VideoLink(url)
+            file_name = link.download(download_dir=data['media_dir'])
+            if file_name is None:
+                kontor_db.update_entry('media_file', file_id, {'file_name': None, 'should_download': 1})
+            else:
+                download_file = Path(file_name)
+                download_file.with_name(f"{file_id}{download_file.suffix}")
+                link.file_name = download_file.name
+                link.should_download = 0
+                link.cloud_link = download_file.absolute()
+                kontor_db.update_entry('media_file', file_id,
+                                       {
+                                           'file_name': download_file.name,
+                                           'should_download': 0,
+                                           'cloud_link': download_file.absolute()}
+                                       )
 
     @ex(
         help='add url to database',
@@ -56,9 +79,10 @@ class Media(Controller):
             data['link_url'] = self.app.pargs.link
             if self.app.pargs.dry_run:
                 print(f"add url {data['link_url']} to database")
-            kontor_db = KontorDB(self.app.engine, self.app.log)
+            else:
-            kontor_db.add_link(self.app.pargs.link, self.app.pargs.dry_run)
+                kontor_db = KontorDB(self.app.engine)
-
+                result = kontor_db.add_link(self.app.pargs.link)
+                self.log.info(result)
         else:
             print("no url was given.")
 
@@ -77,5 +101,5 @@ class Media(Controller):
         }
         if self.app.pargs.media_dir is not None:
             data['media_dir'] = self.app.pargs.media_dir
-        kontor_db = KontorDB(self.app.engine, self.app.log)
+        kontor_db = KontorDB(self.app.engine)
         kontor_db.check_files()

python/kontor-cli/kontor/templates/command1.jinja2

@@ -1,4 +0,0 @@
-
-Example Template (templates/command1.jinja2)
-
-Foo => {{ foo }}

python/kontor-cli/kontor/templates/export_db.jinja2

@@ -0,0 +1,5 @@
+ Following tables were exported:
+
+{% for key, value in results.items() %}
+Table {{key}}: {{value}} entries
+{% endfor %}

python/kontor-cli/pyproject.toml

@@ -0,0 +1,29 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "kontor-cli"
+version = "0.1.0"
+description = "Kontor CLI Application"
+authors = [
+    {name = "Thomas Peetz", email = "thomas.peetz@thpeetz.de"},
+]
+maintainers = [
+    {name = "Thomas Peetz", email = "thomas.peetz@thpeetz.de"},
+]
+readme = "README.md"
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3.10",
+    "Topic :: Utilities",
+]
+
+dependencies = [
+]
+requires-python = ">= 3.10"
+
+[projects.scripts]
+kontor = "kontor.main:main"

python/kontor-cli/requirements.txt

@@ -7,3 +7,4 @@ cement[yaml]
 cement[colorlog]
 mariadb
 sqlalchemy
+pathlib

python/kontor-schema/kontor_schema/__init__.py

@@ -5,7 +5,6 @@ import uuid
 from datetime import datetime
 from pathlib import Path
 
-import requests
 from sqlalchemy import Engine
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import sessionmaker
@@ -20,9 +19,8 @@ from .media import MediaFile, MediaArticle, MediaVideo
 
 class KontorDB:
 
-    def __init__(self, db_engine: Engine, log):
+    def __init__(self, db_engine: Engine):
         self.engine = db_engine
-        self.log = log
         self.registry = {}
         self.init_registry()
 
@@ -107,7 +105,7 @@ class KontorDB:
                 columns[column.column_name] = {"order": column.column_order, "type": column.column_type}
         return columns
 
-    def get_filters(self, table_name):
+    def get_filters(self, table_name: str) -> dict:
         _filter_map = {}
         __session__ = sessionmaker(self.engine)
         with __session__() as session:
@@ -116,7 +114,6 @@ class KontorDB:
                     filter(MetaDataTable.table_name == table_name).
                     filter(MetaDataColumn.show_filter == 1).all()):
                 _filter_map[column.column_name] = {'label': column.filter_label, 'widget': None}
-            self.log.debug(f"retrieved {len(_filter_map)} filters: {_filter_map}")
         return _filter_map
 
     def data(self, table_name: str, columns: dict, filters: dict) -> list:
@@ -135,19 +132,16 @@ class KontorDB:
                     column_name = columns[order]['column']
                     if str(column_name).endswith("_id"):
                         ref_table = column_name[:-3]
-                        # print(f"{ref_table=}")
                         ref = getattr(entry, ref_table)
                         value = getattr(ref, "name")
-                        # print(f"{value=}")
                         row.append(value)
                     else:
                         row.append(getattr(entry, column_name))
-                # print(repr(row))
                 data.append(row)
         return data
 
-    def export_db(self, export_type: str, export_file_name: str):
+    def export_db(self, export_type: str, export_file_name: str) -> dict:
-        self.log.info(f"export DB to {export_file_name} as {export_type}")
+        results = {}
         db = {}
         export_table_list = self.get_table_names()
         for table in export_table_list:
@@ -161,8 +155,6 @@ class KontorDB:
             with __session__() as session:
                 rows = session.query(model).all()
                 entries = []
-                self.log.debug(f"found {len(rows)} entries")
-                self.log.debug(f"found {len(columns)} columns")
                 for row in rows:
                     # print(row)
                     entry = {}
@@ -176,11 +168,11 @@ class KontorDB:
                                 entry[column_name] = str(value)
                             else:
                                 entry[column_name] = value
-                        except AttributeError as error:
+                        except AttributeError:
-                            self.log.debug("could not get value")
+                            pass
                     entries.append(entry)
                 db[table] = entries
-        export_file = Path(export_file_name)
+                results[table] = len(entries)
         match export_type:
             case "JSON":
                 json_dump = json.dumps(db, indent=4)
@@ -190,17 +182,14 @@ class KontorDB:
                 export_file = Path(export_file_name)
             case "SQLite":
                 export_file = Path(export_file_name)
-            case _:
+        return results
-                self.log.debug("unknown export type")
-        if export_file.exists():
-            self.log.debug(f"{export_file} exists")
 
-    def import_db(self, import_file_name: str, dry_run: bool):
+    def import_db(self, import_file_name: str, dry_run: bool) -> dict:
+        result = {}
         import_file = Path(import_file_name)
         if not import_file.exists():
             print(f"File {import_file_name} does not exist. Do nothing.")
-            return
+            return result
-        self.log.debug(f"evaluate type from file extension: {import_file.suffix}")
         match import_file.suffix:
             case '.json':
                 print("read json file")
@@ -208,34 +197,44 @@ class KontorDB:
                     json_load = json.load(json_file)
                 for table in json_load:
                     print(f"{table}: {len(json_load[table])}")
-                    self.import_table(table, json_load[table], dry_run)
+                    result[table] = self.import_table(table, json_load[table], dry_run)
             case '.yml':
                 print("read yaml file")
             case '.yaml':
                 print("read yaml file")
             case '.db':
                 print("read sqlite file")
+        return result
 
-    def import_table(self, table_name, items, dry_run: bool):
+    def import_table(self, table_name, items, dry_run: bool) -> dict:
+        result = {}
+        updated = []
+        added = []
+        remaining = []
         existing_ids = self.get_ids(table_name)
         for item in items:
-            # self.log.debug(f"{item}")
             current_id = item['id']
             found_item = None
             __session__ = sessionmaker(self.engine)
             with __session__() as session:
                 found_item = session.query(self.registry[table_name]).get(current_id)
-                self.log.debug(f"found: {found_item}")
+            if found_item is not None:
-                if found_item is not None:
+                changed = self.update_entry(table_name, current_id, item)
-                    changed = self.update_entry(found_item, item, dry_run)
+                updated.append(item)
-                    if changed:
+                if changed:
-                        print(f"{current_id} has changed")
+                    print(f"{current_id} has changed")
-                    existing_ids.remove(current_id)
+                    updated.append(item)
-                else:
+                existing_ids.remove(current_id)
-                    self.log.info("item to import not found in database, add new one...")
+            else:
-                    self.add_entry(table_name, item, session, dry_run)
+                self.add_entry(table_name, item)
+                added.append(item)
         if len(existing_ids) > 0:
             print("remaining items")
+            remaining.extend(existing_ids)
+        result['updated'] = updated
+        result['added'] = added
+        result['remaining'] = remaining
+        return result
 
     def get_ids(self, table_name: str) -> list:
         existing_ids = []
@@ -246,36 +245,36 @@ class KontorDB:
                 existing_ids.append(getattr(item, 'id'))
         return existing_ids
 
-    def add_entry(self, table_name: str, update_item: dict, session, dry_run: bool):
+    def add_entry(self, table_name: str, update_item: dict):
-        add_item = self.registry[table_name]()
+        __session__ = sessionmaker(self.engine)
-        for key in update_item.keys():
+        with __session__() as session:
-            update_value = update_item[key]
+            add_item = self.registry[table_name]()
-            setattr(add_item, key, update_value)
+            for key in update_item.keys():
-        if dry_run:
+                update_value = update_item[key]
-            self.log.info(f"add item {type(add_item)} with id {update_item['id']}")
+                setattr(add_item, key, update_value)
-        else:
+                session.add(add_item)
-            session.add(add_item)
+                session.commit()
-            session.commit()
 
-    def update_entry(self, existing_item, update_item: dict, dry_run: bool) -> bool:
+    def update_entry(self, table_name, current_id, update_item: dict) -> bool:
-        changed = False
+        __session__ = sessionmaker(self.engine)
-        for key in update_item.keys():
+        with __session__() as session:
-            update_value = update_item[key]
+            existing_item = session.query(self.registry[table_name]).get(current_id)
-            existing_value = getattr(existing_item, key)
+            changed = False
-            if type(existing_value) is not type(update_value):
+            for key in update_item.keys():
-                # self.log.debug(f"compare {type(existing_value)} with {type(update_value)}")
+                update_value = update_item[key]
-                existing_value = str(existing_value)
+                existing_value = getattr(existing_item, key)
-            if existing_value != update_value:
+                if type(existing_value) is not type(update_value):
-                print(f"{key} has changed: {existing_value} != {update_value}")
+                    existing_value = str(existing_value)
-                if not dry_run:
+                if existing_value != update_value:
+                    print(f"{key} has changed: {existing_value} != {update_value}")
                     setattr(existing_item, key, update_value)
-                    # existing_item[key] = update_value
+                    session.commit()
                     changed = True
-                self.log.info(f"update {key} with {update_value}")
+                    print(f"update {key} with {update_value}")
         return changed
 
-    def add_link(self, link: str, dry_run: bool):
+    def add_link(self, link: str) -> dict:
-        self.log.info(f"add link {link} to media_file")
+        result = {}
         __session__ = sessionmaker(self.engine)
         with __session__() as session:
             media_file = MediaFile()
@@ -289,125 +288,32 @@ class KontorDB:
             try:
                 session.add(media_file)
                 session.commit()
-                self.log.info(f"entry {media_file} successfully added")
+                result['added'] = media_file
             except IntegrityError as error:
                 session.rollback()
-                self.log.info(error.orig)
+                result['error'] = error.orig
+        return result
 
-    def update_title(self, dry_run=False):
+    def get_update_list(self) -> dict:
-        self.log.info(f"get links to review of media_file")
+        update_list = {}
-        __session__ = sessionmaker(self.engine)
-        with __session__() as session:
-            links = session.query(MediaFile).filter(MediaFile.review == 1).all()
-            self.log.info(f"try to update {len(links)} items")
-            for link in links:
-                url = link.url
-                if url is None:
-                    self.log.info(f"url has not been set for {link.id}")
-                    continue
-                self.log.info('get title for url {}'.format(url))
-                if dry_run:
-                    continue
-                try:
-                    r = requests.get(url)
-                    soup = BeautifulSoup(r.content, "html.parser")
-                    title = soup.title.string
-                except:
-                    self.log.info("Sorry, could not retrieve title")
-                    continue
-                self.log.info('ID {} has title {}'.format(link.id, title))
-                link.title = title
-                link.review = 0
-                session.commit()
-
-    def get_update_list(self) -> list[str]:
-        self.log.debug("get links marked as review")
-        update_list = []
         __session__ = sessionmaker(self.engine)
         with __session__() as session:
             links = session.query(MediaFile).filter(MediaFile.review == 1).all()
             for link in links:
                 url = link.url
                 if url is None:
-                    self.log.info(f"url has not been set for {link.id}")
                     continue
-                update_list.append(url)
+                update_list[link.id] = url
-        self.log.debug(f"found {len(update_list)} urls for updates")
         return update_list
 
-    def get_download_list(self) -> list[str]:
+    def get_download_list(self) -> dict:
-        self.log.debug("get links marked as should_download")
+        download_list = {}
-        download_list = []
         __session__ = sessionmaker(self.engine)
         with __session__() as session:
             links = session.query(MediaFile).filter(MediaFile.should_download == 1).all()
             for link in links:
                 url = link.url
                 if url is None:
-                    self.log.info(f"url has not been set for {link.id}")
                     continue
-                download_list.append(url)
+                download_list[link.id] = url
-        self.log.debug(f"found {len(download_list)} urls for downloads")
         return download_list
-
-    def download_file(self, dry_run=False):
-        self.log.info(f"download marked urls of media_file")
-        __session__ = sessionmaker(self.engine)
-        with __session__() as session:
-            links = session.query(MediaFile).filter(MediaFile.should_download == 1).all()
-            self.log.info(f"try to download {len(links)} items")
-            for link in links:
-                url = link.url
-                if url is None:
-                    self.log.info(f"url has not been set for {link.id}")
-                    continue
-                if dry_run:
-                    self.log.info(f"download {link.url} to {self.config.get('media', 'dir')}")
-                    continue
-                filename = self.download_url(link)
-                if filename is None:
-                    link.file_name = filename
-                    link.should_download = 1
-                else:
-                    download_file = Path(filename)
-                    download_file.with_name(f"{link.id}{download_file.suffix}")
-                    link.file_name = download_file.name
-                    link.should_download = 0
-                    link.cloud_link = download_file.absolute()
-                session.commit()
-
-    def parse_output(self, lines_list):
-        file_name = ""
-        for line in lines_list:
-            if 'has already been downloaded' in line:
-                end_len = len(' has already been downloaded')
-                file_name = line[11:-end_len]
-                self.log.info('found file: "%s"', file_name)
-            if 'Destination' in line:
-                line_len = len(line)
-                start_len = len('[download] Destination: ')
-                file_len = line_len - start_len
-                file_name = line[-file_len:]
-                self.log.info('new file: "%s"', file_name)
-        return file_name
-
-    def download_url(self, video_url):
-        media_dir = Path(self.config.get('media', 'dir'))
-        if not media_dir.exists():
-            media_dir = Path().absolute()
-        self.log.info(f"download video to {media_dir}")
-        result = subprocess.run([self.config.get('media', 'yt-dlp'), video_url], cwd=media_dir, capture_output=True,
-                                text=True)
-        if result.returncode == 0:
-            output = result.stdout
-            output = re.sub(' +', ' ', output)
-            lines_list = output.splitlines()
-            return self.parse_output(lines_list)
-        else:
-            return None
-
-    def check_files(self):
-        media_dir = Path(self.config.get('media', 'dir'))
-        if not media_dir.exists():
-            return
-        self.log.info(f"check files in {media_dir}")

python/kontor-video/kontor_video/__init__.py

@@ -1,21 +1,63 @@
+import re
+import subprocess
+from pathlib import Path
+
 import requests
 from bs4 import BeautifulSoup
 
 
 class VideoLink:
 
-    def __init__(self, url: str, log):
+    def __init__(self, url: str, dl_tool: str, table: str):
+        self.file_name = None
         self.url = url
         self.title = None
-        self.log = log
+        self.dl_tool = dl_tool
+        self.table = table
 
-    def get_title(self):
+    def get_title(self) -> str:
         try:
             r = requests.get(self.url)
             soup = BeautifulSoup(r.content, "html.parser")
             title = soup.title.string
         except:
-            self.log.info("Sorry, could not retrieve title")
+            title = None
+        return title
+
 
     def download(self, download_dir=None):
-        self.log.info(f"download {self.url} to {download_dir}")
+        if download_dir is None:
+            download_dir = Path.cwd()
+        result = subprocess.run([self.dl_tool, self.url], cwd=download_dir, capture_output=True, text=True)
+        if result.returncode == 0:
+            output = result.stdout
+            output = re.sub(' +', ' ', output)
+            lines_list = output.splitlines()
+            return self.__parse_output__(lines_list)
+        else:
+            return None
+
+    def __parse_output__(self, lines_list):
+        self.file_name = ""
+        for line in lines_list:
+            if 'has already been downloaded' in line:
+                end_len = len(' has already been downloaded')
+                self.file_name = line[11:-end_len]
+            if 'Destination' in line:
+                line_len = len(line)
+                start_len = len('[download] Destination: ')
+                file_len = line_len - start_len
+                self.file_name = line[-file_len:]
+        return self.file_name
+
+
+class MediaFile(VideoLink):
+
+    def __init__(self, url: str, dl_tool='yt-dlp'):
+        super().__init__(url, dl_tool, 'media_file')
+
+
+class MediaVideo(VideoLink):
+
+    def __init__(self, url: str, dl_tool='yt-dlp'):
+        super().__init__(url, dl_tool, 'media_video')