tpeetz/kontor
1
0
Fork 0
Code Issues 6 Pull Requests 2 Actions Packages Projects Releases 2 Wiki Activity

remove kontor-cli.backup

Browse Source
This commit is contained in:
Thomas Peetz
2025-01-21 09:42:33 +01:00
parent 65288a53a1
commit 4c330a1138
40 changed files with 196 additions and 889 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/kontor-cli.backup/.gitignore
-105
View File
@@ -1,105 +0,0 @@
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
coverage-report/
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
python/kontor-cli.backup/CHANGELOG.md
-5
View File
@@ -1,5 +0,0 @@
# Kontor CLI Change History
## 0.0.1
Initial release.
python/kontor-cli.backup/Dockerfile
-10
View File
@@ -1,10 +0,0 @@
FROM python:3.9-alpine
LABEL MAINTAINER="Thomas Peetz <thomas.peetz@thpeetz.de>"
ENV PS1="\[\e[0;33m\]|> kontor <| \[\e[1;35m\]\W\[\e[0m\] \[\e[0m\]# "
WORKDIR /src
COPY . /src
RUN pip install --no-cache-dir -r requirements.txt \
&& python setup.py install
WORKDIR /
ENTRYPOINT ["kontor"]
python/kontor-cli.backup/LICENSE.md
-1
View File
@@ -1 +0,0 @@
python/kontor-cli.backup/MANIFEST.in
-5
View File
@@ -1,5 +0,0 @@
recursive-include *.py
include setup.cfg
include README.md CHANGELOG.md LICENSE.md
include *.txt
recursive-include kontor/templates *
python/kontor-cli.backup/Makefile
-31
View File
@@ -1,31 +0,0 @@
.PHONY: clean virtualenv test docker dist dist-upload
clean:
find . -name '*.py[co]' -delete
virtualenv:
virtualenv --prompt '|> kontor <| ' env
env/bin/pip install -r requirements-dev.txt
env/bin/python setup.py develop
@echo
@echo "VirtualENV Setup Complete. Now run: source env/bin/activate"
@echo
test:
python -m pytest \
-v \
--cov=kontor \
--cov-report=term \
--cov-report=html:coverage-report \
tests/
docker: clean
docker build -t kontor:latest .
dist: clean
rm -rf dist/*
python setup.py sdist
python setup.py bdist_wheel
dist-upload:
twine upload dist/*
python/kontor-cli.backup/README.md
-69
View File
@@ -1,69 +0,0 @@
# Kontor CLI
## Installation
```
$ pip install -r requirements.txt
$ python setup.py install
```
## Development
This project includes a number of helpers in the `Makefile` to streamline common development tasks.
### Environment Setup
The following demonstrates setting up and working with a development environment:
```
### create a virtualenv for development
$ make virtualenv
$ source env/bin/activate
### run kontor cli application
$ kontor --help
### run pytest / coverage
$ make test
```
### Releasing to PyPi
Before releasing to PyPi, you must configure your login credentials:
**~/.pypirc**:
```
[pypi]
username = YOUR_USERNAME
password = YOUR_PASSWORD
```
Then use the included helper function via the `Makefile`:
```
$ make dist
$ make dist-upload
```
## Deployments
### Docker
Included is a basic `Dockerfile` for building and distributing `Kontor CLI`,
and can be built with the included `make` helper:
```
$ make docker
$ docker run -it kontor --help
```
python/kontor-cli.backup/config/kontor.yml.example
-46
View File
@@ -1,46 +0,0 @@
### Kontor CLI Configuration Settings
---
kontor:
### Toggle application level debug (does not toggle framework debugging)
# debug: false
### Where external (third-party) plugins are loaded from
# plugin_dir: /var/lib/kontor/plugins/
### Where all plugin configurations are loaded from
# plugin_config_dir: /etc/kontor/plugins.d/
### Where external templates are loaded from
# template_dir: /var/lib/kontor/templates/
### The log handler label
# log_handler: colorlog
### The output handler label
# output_handler: jinja2
### sample foo option
# foo: bar
log.colorlog:
### Where the log file lives (no log file by default)
# file: null
### The level for which to log. One of: info, warning, error, fatal, debug
# level: info
### Whether or not to log to console
# to_console: true
### Whether or not to rotate the log file when it reaches `max_bytes`
# rotate: false
### Max size in bytes that a log file can grow until it is rotated.
# max_bytes: 512000
### The maximum number of log files to maintain when rotating
# max_files: 4
python/kontor-cli.backup/docs/.gitkeep
View File
python/kontor-cli.backup/kontor/__init__.py
View File
python/kontor-cli.backup/kontor/controllers/__init__.py
View File
python/kontor-cli.backup/kontor/controllers/base.py
-35
View File
@@ -1,35 +0,0 @@
from cement import Controller, ex
from cement.utils.version import get_version_banner
from ..core.version import get_version
VERSION_BANNER = """
Kontor CLI %s
%s
""" % (get_version(), get_version_banner())
class Base(Controller):
class Meta:
label = 'base'
# text displayed at the top of --help output
description = 'Kontor CLI Tool'
# text displayed at the bottom of --help output
epilog = 'Usage: kontor (database | media) [subcommands]'
# controller level arguments. ex: 'kontor --version'
arguments = [
### add a version banner
(['-v', '--version'],
{'action': 'version',
'version': VERSION_BANNER}),
(['-m', '--dry-run'],
{'action': 'store_true',
'dest': 'dry_run'})
]
def _default(self):
"""Default action if no sub-command is passed."""
self.app.args.print_help()
python/kontor-cli.backup/kontor/controllers/database.py
-51
View File
@@ -1,51 +0,0 @@
from cement import Controller, ex
from kontor_schema import KontorDB
class Database(Controller):
class Meta:
label = 'database'
stacked_type = 'nested'
stacked_on = 'base'
@ex(
help='export database to given file',
arguments=[
(['-f', '--file'],
{'help': 'file to store database content',
'action': 'store',
'dest': 'db_file'})
],
)
def export(self):
data = {
'db_file': 'data.json',
'export_type': 'JSON',
}
if self.app.pargs.db_file is not None:
data['db_file'] = self.app.pargs.db_file
kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log)
kontor_db.export_db(data['export_type'], data['db_file'])
self.app.render(data, 'command1.jinja2')
@ex(
label='import',
help='import data from file into database',
arguments=[
(['-f', '--file'],
{'help': 'file to read data',
'action': 'store',
'dest': 'db_file'})
],
)
def import_cmd(self):
data = {
'db_file': 'data.json',
'data_type': 'JSON',
}
if self.app.pargs.db_file is not None:
data['db_file'] = self.app.pargs.db_file
kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log)
self.app.render(data, 'import.jinja2')
kontor_db.import_db(data['db_file'], self.app.pargs.dry_run)
python/kontor-cli.backup/kontor/controllers/media.py
-79
View File
@@ -1,79 +0,0 @@
from cement import Controller, ex
from ..database import KontorDB
class Media(Controller):
class Meta:
label = 'media'
stacked_type = 'nested'
stacked_on = 'base'
@ex(
label='update',
help='update title for mediafiles',
)
def update_title(self):
kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log)
kontor_db.update_title(self.app.pargs.dry_run)
@ex(
label='download',
help='download all marked videos',
arguments=[
(['-d', '--dir'],
{'help': 'directory to store videos',
'action': 'store',
'dest': 'media_dir'})
],
)
def download(self):
data = {
'media_dir': '/data/media',
}
if self.app.pargs.media_dir is not None:
data['media_dir'] = self.app.pargs.media_dir
kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log)
kontor_db.download_file(self.app.pargs.dry_run)
@ex(
help='add url to database',
arguments=[
(['-u', '--url'],
{'help': 'link to downloadable video',
'action': 'store',
'dest': 'link'})
],
)
def add(self):
data = {
'link_url': None
}
if self.app.pargs.link is not None:
data['link_url'] = self.app.pargs.link
if self.app.pargs.dry_run:
print(f"add url {data['link_url']} to database")
kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log)
kontor_db.add_link(self.app.pargs.link, self.app.pargs.dry_run)
else:
print("no url was given.")
@ex(
help='check files if existing',
arguments=[
(['-d', '--dir'],
{'help': 'directory to store videos',
'action': 'store',
'dest': 'media_dir'})
],
)
def check(self):
data = {
'media_dir': '/data/media',
}
if self.app.pargs.media_dir is not None:
data['media_dir'] = self.app.pargs.media_dir
kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log)
kontor_db.check_files()
python/kontor-cli.backup/kontor/core/__init__.py
View File
python/kontor-cli.backup/kontor/core/exc.py
-4
View File
@@ -1,4 +0,0 @@
class KontorCliError(Exception):
"""Generic errors."""
pass
python/kontor-cli.backup/kontor/core/version.py
-7
View File
@@ -1,7 +0,0 @@
from cement.utils.version import get_version as cement_get_version
VERSION = (0, 1, 0, 'alpha', 0)
def get_version(version=VERSION):
return cement_get_version(version)
python/kontor-cli.backup/kontor/ext/__init__.py
View File
python/kontor-cli.backup/kontor/main.py
-124
View File
@@ -1,124 +0,0 @@
from cement import App, TestApp, init_defaults
from cement.core.exc import CaughtSignal
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from .controllers.database import Database
from .controllers.media import Media
from .core.exc import KontorCliError
from .controllers.base import Base
# configuration defaults
CONFIG = init_defaults('kontor', 'mariadb', 'media')
CONFIG['kontor']['foo'] = 'bar'
CONFIG['mariadb']['user'] = 'kontor'
CONFIG['mariadb']['password'] = 'kontor'
CONFIG['mariadb']['host'] = '127.0.0.1'
CONFIG['mariadb']['port'] = '3306'
CONFIG['mariadb']['database'] = 'kontor'
CONFIG['media']['yt-dlp'] = '/home/tpeetz/bin/yt-dlp'
CONFIG['media']['dir'] = '/data/media'
META = init_defaults('output.json', 'output.yaml')
META['output.json']['overridable'] = True
META['output.yaml']['overridable'] = True
def extend_sqlalchemy(app):
app.log.info('extending kontor application with sqlalchemy')
connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format(
app.config.get('mariadb', 'user'),
app.config.get('mariadb', 'password'),
app.config.get('mariadb', 'host'),
app.config.get('mariadb', 'port'),
app.config.get('mariadb', 'database')
))
# engine = create_engine(connect_string, echo=True)
engine = create_engine(connect_string)
Base.metadata.create_all(bind=engine)
__session__ = sessionmaker(bind=engine)
app.extend('engine', engine)
class KontorCli(App):
"""Kontor CLI primary application."""
class Meta:
label = 'kontor'
# configuration defaults
config_defaults = CONFIG
meta_defaults = META
# call sys.exit() on close
exit_on_close = True
# load additional framework extensions
extensions = [
'yaml',
'json',
'colorlog',
'jinja2',
]
# configuration handler
config_handler = 'yaml'
# configuration file suffix
config_file_suffix = '.yml'
# set the log handler
log_handler = 'colorlog'
# set the output handler
output_handler = 'jinja2'
hooks = [
('post_setup', extend_sqlalchemy),
]
# register handlers
handlers = [
Base,
Database,
Media,
]
class KontorCliTest(TestApp,KontorCli):
"""A sub-class of KontorCli that is better suited for testing."""
class Meta:
label = 'kontor'
def main():
with KontorCli() as app:
try:
app.run()
except AssertionError as e:
print('AssertionError > %s' % e.args[0])
app.exit_code = 1
if app.debug is True:
import traceback
traceback.print_exc()
except KontorCliError as e:
print('KontorCliError > %s' % e.args[0])
app.exit_code = 1
if app.debug is True:
import traceback
traceback.print_exc()
except CaughtSignal as e:
# Default Cement signals are SIGINT and SIGTERM, exit 0 (non-error)
print('\n%s' % e)
app.exit_code = 0
if __name__ == '__main__':
main()
python/kontor-cli.backup/kontor/plugins/__init__.py
View File
python/kontor-cli.backup/kontor/templates/__init__.py
View File
python/kontor-cli.backup/kontor/templates/command1.jinja2
-4
View File
@@ -1,4 +0,0 @@
Example Template (templates/command1.jinja2)
Foo => {{ foo }}
python/kontor-cli.backup/kontor/templates/download.jinja2
-2
View File
@@ -1,2 +0,0 @@
Download videos to directory {{ media_dir }}
python/kontor-cli.backup/kontor/templates/import.jinja2
-2
View File
@@ -1,2 +0,0 @@
Import data from {{ db_file }}
python/kontor-cli.backup/kontor/templates/update.jinja2
-2
View File
@@ -1,2 +0,0 @@
Update entries of mediafile
python/kontor-cli.backup/requirements-dev.txt
-8
View File
@@ -1,8 +0,0 @@
-r requirements.txt
pytest
pytest-cov
coverage
twine>=1.11.0
setuptools>=38.6.0
wheel>=0.31.0
python/kontor-cli.backup/requirements.txt
-6
View File
@@ -1,6 +0,0 @@
cement==3.0.12
cement[jinja2]
cement[yaml]
cement[colorlog]
mariadb
sqlalchemy
python/kontor-cli.backup/setup.cfg
View File
python/kontor-cli.backup/setup.py
-28
View File
@@ -1,28 +0,0 @@
from setuptools import setup, find_packages
from kontor.core.version import get_version
VERSION = get_version()
f = open('README.md', 'r')
LONG_DESCRIPTION = f.read()
f.close()
setup(
name='kontor',
version=VERSION,
description='Kontor CLI',
long_description=LONG_DESCRIPTION,
long_description_content_type='text/markdown',
author='Thomas Peetz',
author_email='thomas.peetz@thpeetz.de',
url='https://gitlab.com/tpeetz/kontor/',
license='MIT',
packages=find_packages(exclude=['ez_setup', 'tests*']),
package_data={'kontor': ['templates/*']},
include_package_data=True,
entry_points="""
[console_scripts]
kontor = kontor.main:main
""",
)
python/kontor-cli.backup/tests/conftest.py
-16
View File
@@ -1,16 +0,0 @@
"""
PyTest Fixtures.
"""
import pytest
from cement import fs
@pytest.fixture(scope="function")
def tmp(request):
"""
Create a `tmp` object that geneates a unique temporary directory, and file
for each test function that requires it.
"""
t = fs.Tmp()
yield t
t.remove()
python/kontor-cli.backup/tests/test_kontor.py
-36
View File
@@ -1,36 +0,0 @@
from pytest import raises
from kontor.main import KontorCliTest
def test_kontor():
# test kontor without any subcommands or arguments
with KontorCliTest() as app:
app.run()
assert app.exit_code == 0
def test_kontor_debug():
# test that debug mode is functional
argv = ['--debug']
with KontorCliTest(argv=argv) as app:
app.run()
assert app.debug is True
def test_command1():
# test command1 without arguments
argv = ['command1']
with KontorCliTest(argv=argv) as app:
app.run()
data,output = app.last_rendered
assert data['foo'] == 'bar'
assert output.find('Foo => bar')
# test command1 with arguments
argv = ['command1', '--foo', 'not-bar']
with KontorCliTest(argv=argv) as app:
app.run()
data,output = app.last_rendered
assert data['foo'] == 'not-bar'
assert output.find('Foo => not-bar')
python/kontor-cli/kontor/controllers/base.py
-26
View File
@@ -32,29 +32,3 @@ class CliBase(Controller):
"""Default action if no sub-command is passed."""
self.app.args.print_help()
@ex(
help='example sub command1',
# sub-command level arguments. ex: 'kontor command1 --foo bar'
arguments=[
### add a sample foo option under subcommand namespace
( [ '-f', '--foo' ],
{ 'help' : 'notorious foo option',
'action' : 'store',
'dest' : 'foo' } ),
],
)
def command1(self):
"""Example sub-command."""
data = {
'foo' : 'bar',
}
### do something with arguments
if self.app.pargs.foo is not None:
data['foo'] = self.app.pargs.foo
self.app.render(data, 'command1.jinja2')
python/kontor-cli/kontor/controllers/database.py
+12 -6
View File
@@ -8,6 +8,11 @@ class Database(Controller):
label = 'database'
stacked_type = 'nested'
stacked_on = 'clibase'
arguments = [
(['-m', '--dry-run'],
{'action': 'store_true',
'dest': 'dry_run'}),
]
@ex(
help='export database to given file',
@@ -25,9 +30,11 @@ class Database(Controller):
}
if self.app.pargs.db_file is not None:
data['db_file'] = self.app.pargs.db_file
kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log)
kontor_db.export_db(data['export_type'], data['db_file'])
self.app.render(data, 'command1.jinja2')
kontor_db = KontorDB(self.app.engine)
self.app.log.info(f"export DB to {data['db_file']} as {data['export_type']}")
results = kontor_db.export_db(data['export_type'], data['db_file'])
data['results'] = results
self.app.render(data, 'export_db.jinja2')
@ex(
label='import',
@@ -46,8 +53,7 @@ class Database(Controller):
}
if self.app.pargs.db_file is not None:
data['db_file'] = self.app.pargs.db_file
kontor_db = KontorDB(self.app.engine, self.app.config, self.app.log)
self.app.render(data, 'import.jinja2')
kontor_db = KontorDB(self.app.engine)
kontor_db.import_db(data['db_file'], self.app.pargs.dry_run)
@ex(
@@ -66,7 +72,7 @@ class Database(Controller):
cursor.execute("SHOW TABLES")
for (tablename,) in cursor.fetchall():
table_list.append(tablename)
kontor_db = KontorDB(self.app.engine, self.app.log)
kontor_db = KontorDB(self.app.engine)
table_names = kontor_db.get_table_names()
for table in table_list:
if table not in table_names:
python/kontor-cli/kontor/controllers/media.py
+35 -11
View File
@@ -1,9 +1,11 @@
from pathlib import Path
from cement import Controller, ex
from kontor_schema import KontorDB
from kontor_video import VideoLink
class Media(Controller):
class Media(Controller):
class Meta:
label = 'media'
stacked_type = 'nested'
@@ -14,8 +16,14 @@ class Media(Controller):
help='update title for mediafiles',
)
def update_title(self):
kontor_db = KontorDB(self.app.engine, self.app.log)
kontor_db.update_title(self.app.pargs.dry_run)
kontor_db = KontorDB(self.app.engine)
updates = kontor_db.get_update_list()
self.app.log.info(f"found {len(updates)} links for update")
for file_id, url in updates.items():
link = VideoLink(url)
title = link.get_title()
if title is not None:
kontor_db.update_entry('media_file', file_id, {'title': title, 'review': 0,})
@ex(
label='download',
@@ -33,11 +41,26 @@ class Media(Controller):
}
if self.app.pargs.media_dir is not None:
data['media_dir'] = self.app.pargs.media_dir
kontor_db = KontorDB(self.app.engine, self.app.log)
kontor_db = KontorDB(self.app.engine)
downloads = kontor_db.get_download_list()
for download in downloads:
link = VideoLink(download, download_dir=data['media_dir'])
link.download()
self.app.log.info(f"found {len(downloads)} links for download")
for file_id, url in downloads.items():
link = VideoLink(url)
file_name = link.download(download_dir=data['media_dir'])
if file_name is None:
kontor_db.update_entry('media_file', file_id, {'file_name': None, 'should_download': 1})
else:
download_file = Path(file_name)
download_file.with_name(f"{file_id}{download_file.suffix}")
link.file_name = download_file.name
link.should_download = 0
link.cloud_link = download_file.absolute()
kontor_db.update_entry('media_file', file_id,
{
'file_name': download_file.name,
'should_download': 0,
'cloud_link': download_file.absolute()}
)
@ex(
help='add url to database',
@@ -56,9 +79,10 @@ class Media(Controller):
data['link_url'] = self.app.pargs.link
if self.app.pargs.dry_run:
print(f"add url {data['link_url']} to database")
kontor_db = KontorDB(self.app.engine, self.app.log)
kontor_db.add_link(self.app.pargs.link, self.app.pargs.dry_run)
else:
kontor_db = KontorDB(self.app.engine)
result = kontor_db.add_link(self.app.pargs.link)
self.log.info(result)
else:
print("no url was given.")
@@ -77,5 +101,5 @@ class Media(Controller):
}
if self.app.pargs.media_dir is not None:
data['media_dir'] = self.app.pargs.media_dir
kontor_db = KontorDB(self.app.engine, self.app.log)
kontor_db = KontorDB(self.app.engine)
kontor_db.check_files()
python/kontor-cli/kontor/templates/command1.jinja2
-4
View File
@@ -1,4 +0,0 @@
Example Template (templates/command1.jinja2)
Foo => {{ foo }}
python/kontor-cli/kontor/templates/export_db.jinja2
+5
View File
@@ -0,0 +1,5 @@
Following tables were exported:
{% for key, value in results.items() %}
Table {{key}}: {{value}} entries
{% endfor %}
python/kontor-cli/pyproject.toml
+29
View File
@@ -0,0 +1,29 @@
[build-system]
requires = ["setuptools"]
build-backend = "setuptools.build_meta"
[project]
name = "kontor-cli"
version = "0.1.0"
description = "Kontor CLI Application"
authors = [
{name = "Thomas Peetz", email = "thomas.peetz@thpeetz.de"},
]
maintainers = [
{name = "Thomas Peetz", email = "thomas.peetz@thpeetz.de"},
]
readme = "README.md"
classifiers = [
"Development Status :: 3 - Alpha",
"Environment :: Console",
"Programming Language :: Python",
"Programming Language :: Python :: 3.10",
"Topic :: Utilities",
]
dependencies = [
]
requires-python = ">= 3.10"
[projects.scripts]
kontor = "kontor.main:main"
python/kontor-cli/requirements.txt
+1
View File
@@ -7,3 +7,4 @@ cement[yaml]
cement[colorlog]
mariadb
sqlalchemy
pathlib
python/kontor-schema/kontor_schema/__init__.py
+67 -161
View File
@@ -5,7 +5,6 @@ import uuid
from datetime import datetime
from pathlib import Path
import requests
from sqlalchemy import Engine
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import sessionmaker
@@ -20,9 +19,8 @@ from .media import MediaFile, MediaArticle, MediaVideo
class KontorDB:
def __init__(self, db_engine: Engine, log):
def __init__(self, db_engine: Engine):
self.engine = db_engine
self.log = log
self.registry = {}
self.init_registry()
@@ -107,7 +105,7 @@ class KontorDB:
columns[column.column_name] = {"order": column.column_order, "type": column.column_type}
return columns
def get_filters(self, table_name):
def get_filters(self, table_name: str) -> dict:
_filter_map = {}
__session__ = sessionmaker(self.engine)
with __session__() as session:
@@ -116,7 +114,6 @@ class KontorDB:
filter(MetaDataTable.table_name == table_name).
filter(MetaDataColumn.show_filter == 1).all()):
_filter_map[column.column_name] = {'label': column.filter_label, 'widget': None}
self.log.debug(f"retrieved {len(_filter_map)} filters: {_filter_map}")
return _filter_map
def data(self, table_name: str, columns: dict, filters: dict) -> list:
@@ -135,19 +132,16 @@ class KontorDB:
column_name = columns[order]['column']
if str(column_name).endswith("_id"):
ref_table = column_name[:-3]
# print(f"{ref_table=}")
ref = getattr(entry, ref_table)
value = getattr(ref, "name")
# print(f"{value=}")
row.append(value)
else:
row.append(getattr(entry, column_name))
# print(repr(row))
data.append(row)
return data
def export_db(self, export_type: str, export_file_name: str):
self.log.info(f"export DB to {export_file_name} as {export_type}")
def export_db(self, export_type: str, export_file_name: str) -> dict:
results = {}
db = {}
export_table_list = self.get_table_names()
for table in export_table_list:
@@ -161,8 +155,6 @@ class KontorDB:
with __session__() as session:
rows = session.query(model).all()
entries = []
self.log.debug(f"found {len(rows)} entries")
self.log.debug(f"found {len(columns)} columns")
for row in rows:
# print(row)
entry = {}
@@ -176,11 +168,11 @@ class KontorDB:
entry[column_name] = str(value)
else:
entry[column_name] = value
except AttributeError as error:
self.log.debug("could not get value")
except AttributeError:
pass
entries.append(entry)
db[table] = entries
export_file = Path(export_file_name)
results[table] = len(entries)
match export_type:
case "JSON":
json_dump = json.dumps(db, indent=4)
@@ -190,17 +182,14 @@ class KontorDB:
export_file = Path(export_file_name)
case "SQLite":
export_file = Path(export_file_name)
case _:
self.log.debug("unknown export type")
if export_file.exists():
self.log.debug(f"{export_file} exists")
return results
def import_db(self, import_file_name: str, dry_run: bool):
def import_db(self, import_file_name: str, dry_run: bool) -> dict:
result = {}
import_file = Path(import_file_name)
if not import_file.exists():
print(f"File {import_file_name} does not exist. Do nothing.")
return
self.log.debug(f"evaluate type from file extension: {import_file.suffix}")
return result
match import_file.suffix:
case '.json':
print("read json file")
@@ -208,34 +197,44 @@ class KontorDB:
json_load = json.load(json_file)
for table in json_load:
print(f"{table}: {len(json_load[table])}")
self.import_table(table, json_load[table], dry_run)
result[table] = self.import_table(table, json_load[table], dry_run)
case '.yml':
print("read yaml file")
case '.yaml':
print("read yaml file")
case '.db':
print("read sqlite file")
return result
def import_table(self, table_name, items, dry_run: bool):
def import_table(self, table_name, items, dry_run: bool) -> dict:
result = {}
updated = []
added = []
remaining = []
existing_ids = self.get_ids(table_name)
for item in items:
# self.log.debug(f"{item}")
current_id = item['id']
found_item = None
__session__ = sessionmaker(self.engine)
with __session__() as session:
found_item = session.query(self.registry[table_name]).get(current_id)
self.log.debug(f"found: {found_item}")
if found_item is not None:
changed = self.update_entry(found_item, item, dry_run)
if changed:
print(f"{current_id} has changed")
existing_ids.remove(current_id)
else:
self.log.info("item to import not found in database, add new one...")
self.add_entry(table_name, item, session, dry_run)
if found_item is not None:
changed = self.update_entry(table_name, current_id, item)
updated.append(item)
if changed:
print(f"{current_id} has changed")
updated.append(item)
existing_ids.remove(current_id)
else:
self.add_entry(table_name, item)
added.append(item)
if len(existing_ids) > 0:
print("remaining items")
remaining.extend(existing_ids)
result['updated'] = updated
result['added'] = added
result['remaining'] = remaining
return result
def get_ids(self, table_name: str) -> list:
existing_ids = []
@@ -246,36 +245,36 @@ class KontorDB:
existing_ids.append(getattr(item, 'id'))
return existing_ids
def add_entry(self, table_name: str, update_item: dict, session, dry_run: bool):
add_item = self.registry[table_name]()
for key in update_item.keys():
update_value = update_item[key]
setattr(add_item, key, update_value)
if dry_run:
self.log.info(f"add item {type(add_item)} with id {update_item['id']}")
else:
session.add(add_item)
session.commit()
def add_entry(self, table_name: str, update_item: dict):
__session__ = sessionmaker(self.engine)
with __session__() as session:
add_item = self.registry[table_name]()
for key in update_item.keys():
update_value = update_item[key]
setattr(add_item, key, update_value)
session.add(add_item)
session.commit()
def update_entry(self, existing_item, update_item: dict, dry_run: bool) -> bool:
changed = False
for key in update_item.keys():
update_value = update_item[key]
existing_value = getattr(existing_item, key)
if type(existing_value) is not type(update_value):
# self.log.debug(f"compare {type(existing_value)} with {type(update_value)}")
existing_value = str(existing_value)
if existing_value != update_value:
print(f"{key} has changed: {existing_value} != {update_value}")
if not dry_run:
def update_entry(self, table_name, current_id, update_item: dict) -> bool:
__session__ = sessionmaker(self.engine)
with __session__() as session:
existing_item = session.query(self.registry[table_name]).get(current_id)
changed = False
for key in update_item.keys():
update_value = update_item[key]
existing_value = getattr(existing_item, key)
if type(existing_value) is not type(update_value):
existing_value = str(existing_value)
if existing_value != update_value:
print(f"{key} has changed: {existing_value} != {update_value}")
setattr(existing_item, key, update_value)
# existing_item[key] = update_value
session.commit()
changed = True
self.log.info(f"update {key} with {update_value}")
print(f"update {key} with {update_value}")
return changed
def add_link(self, link: str, dry_run: bool):
self.log.info(f"add link {link} to media_file")
def add_link(self, link: str) -> dict:
result = {}
__session__ = sessionmaker(self.engine)
with __session__() as session:
media_file = MediaFile()
@@ -289,125 +288,32 @@ class KontorDB:
try:
session.add(media_file)
session.commit()
self.log.info(f"entry {media_file} successfully added")
result['added'] = media_file
except IntegrityError as error:
session.rollback()
self.log.info(error.orig)
result['error'] = error.orig
return result
def update_title(self, dry_run=False):
self.log.info(f"get links to review of media_file")
__session__ = sessionmaker(self.engine)
with __session__() as session:
links = session.query(MediaFile).filter(MediaFile.review == 1).all()
self.log.info(f"try to update {len(links)} items")
for link in links:
url = link.url
if url is None:
self.log.info(f"url has not been set for {link.id}")
continue
self.log.info('get title for url {}'.format(url))
if dry_run:
continue
try:
r = requests.get(url)
soup = BeautifulSoup(r.content, "html.parser")
title = soup.title.string
except:
self.log.info("Sorry, could not retrieve title")
continue
self.log.info('ID {} has title {}'.format(link.id, title))
link.title = title
link.review = 0
session.commit()
def get_update_list(self) -> list[str]:
self.log.debug("get links marked as review")
update_list = []
def get_update_list(self) -> dict:
update_list = {}
__session__ = sessionmaker(self.engine)
with __session__() as session:
links = session.query(MediaFile).filter(MediaFile.review == 1).all()
for link in links:
url = link.url
if url is None:
self.log.info(f"url has not been set for {link.id}")
continue
update_list.append(url)
self.log.debug(f"found {len(update_list)} urls for updates")
update_list[link.id] = url
return update_list
def get_download_list(self) -> list[str]:
self.log.debug("get links marked as should_download")
download_list = []
def get_download_list(self) -> dict:
download_list = {}
__session__ = sessionmaker(self.engine)
with __session__() as session:
links = session.query(MediaFile).filter(MediaFile.should_download == 1).all()
for link in links:
url = link.url
if url is None:
self.log.info(f"url has not been set for {link.id}")
continue
download_list.append(url)
self.log.debug(f"found {len(download_list)} urls for downloads")
download_list[link.id] = url
return download_list
def download_file(self, dry_run=False):
self.log.info(f"download marked urls of media_file")
__session__ = sessionmaker(self.engine)
with __session__() as session:
links = session.query(MediaFile).filter(MediaFile.should_download == 1).all()
self.log.info(f"try to download {len(links)} items")
for link in links:
url = link.url
if url is None:
self.log.info(f"url has not been set for {link.id}")
continue
if dry_run:
self.log.info(f"download {link.url} to {self.config.get('media', 'dir')}")
continue
filename = self.download_url(link)
if filename is None:
link.file_name = filename
link.should_download = 1
else:
download_file = Path(filename)
download_file.with_name(f"{link.id}{download_file.suffix}")
link.file_name = download_file.name
link.should_download = 0
link.cloud_link = download_file.absolute()
session.commit()
def parse_output(self, lines_list):
file_name = ""
for line in lines_list:
if 'has already been downloaded' in line:
end_len = len(' has already been downloaded')
file_name = line[11:-end_len]
self.log.info('found file: "%s"', file_name)
if 'Destination' in line:
line_len = len(line)
start_len = len('[download] Destination: ')
file_len = line_len - start_len
file_name = line[-file_len:]
self.log.info('new file: "%s"', file_name)
return file_name
def download_url(self, video_url):
media_dir = Path(self.config.get('media', 'dir'))
if not media_dir.exists():
media_dir = Path().absolute()
self.log.info(f"download video to {media_dir}")
result = subprocess.run([self.config.get('media', 'yt-dlp'), video_url], cwd=media_dir, capture_output=True,
text=True)
if result.returncode == 0:
output = result.stdout
output = re.sub(' +', ' ', output)
lines_list = output.splitlines()
return self.parse_output(lines_list)
else:
return None
def check_files(self):
media_dir = Path(self.config.get('media', 'dir'))
if not media_dir.exists():
return
self.log.info(f"check files in {media_dir}")
python/kontor-video/kontor_video/__init__.py
+47 -5
View File
@@ -1,21 +1,63 @@
import re
import subprocess
from pathlib import Path
import requests
from bs4 import BeautifulSoup
class VideoLink:
def __init__(self, url: str, log):
def __init__(self, url: str, dl_tool: str, table: str):
self.file_name = None
self.url = url
self.title = None
self.log = log
self.dl_tool = dl_tool
self.table = table
def get_title(self):
def get_title(self) -> str:
try:
r = requests.get(self.url)
soup = BeautifulSoup(r.content, "html.parser")
title = soup.title.string
except:
self.log.info("Sorry, could not retrieve title")
title = None
return title
def download(self, download_dir=None):
self.log.info(f"download {self.url} to {download_dir}")
if download_dir is None:
download_dir = Path.cwd()
result = subprocess.run([self.dl_tool, self.url], cwd=download_dir, capture_output=True, text=True)
if result.returncode == 0:
output = result.stdout
output = re.sub(' +', ' ', output)
lines_list = output.splitlines()
return self.__parse_output__(lines_list)
else:
return None
def __parse_output__(self, lines_list):
self.file_name = ""
for line in lines_list:
if 'has already been downloaded' in line:
end_len = len(' has already been downloaded')
self.file_name = line[11:-end_len]
if 'Destination' in line:
line_len = len(line)
start_len = len('[download] Destination: ')
file_len = line_len - start_len
self.file_name = line[-file_len:]
return self.file_name
class MediaFile(VideoLink):
def __init__(self, url: str, dl_tool='yt-dlp'):
super().__init__(url, dl_tool, 'media_file')
class MediaVideo(VideoLink):
def __init__(self, url: str, dl_tool='yt-dlp'):
super().__init__(url, dl_tool, 'media_video')