tpeetz/kontor
1
0
Fork 0
Code Issues 6 Pull Requests 2 Actions Packages Projects Releases 2 Wiki Activity

moved update and download functionality to kontor-schema

Browse Source
This commit is contained in:
Thomas Peetz
2025-01-28 15:10:10 +01:00
parent c61e49720e
commit e733fa21e6
14 changed files with 118 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/kontor-cli/kontor/controllers/media.py
+7 -30
View File
@@ -1,14 +1,4 @@
from enum import Enum
from pathlib import Path
from cement import Controller, ex
from kontor_schema import KontorDB
from kontor_video import VideoLink, MediaVideo
class VideoType(Enum):
MEDIA_FILE = "media_file"
MEDIA_VIDEO = "media_video"
class Media(Controller):
@@ -23,8 +13,8 @@ class Media(Controller):
)
def update_title(self):
db = self.app.kontor_db
updates = db.get_update_list()
self.app.log.info(f"found {len(updates)} links for update")
updates = db.update_titles()
self.app.log.info(f"{len(updates)} entries updated")
@ex(
label='download',
@@ -43,25 +33,12 @@ class Media(Controller):
if self.app.pargs.media_dir is not None:
data['media_dir'] = self.app.pargs.media_dir
db = self.app.kontor_db
downloads = db.get_download_list(data['media_dir'])
downloads = db.get_download_list()
self.app.log.info(f"found {len(downloads)} links for download")
#for file_id, url in downloads.items():
# link = VideoLink(url, VideoType.MEDIA_FILE)
# file_name = link.download(download_dir=data['media_dir'])
# if file_name is None:
# db.update_entry('media_file', file_id, {'file_name': None, 'should_download': 1})
# else:
# download_file = Path(file_name)
# download_file.with_name(f"{file_id}{download_file.suffix}")
# link.file_name = download_file.name
# link.should_download = 0
# link.cloud_link = download_file.absolute()
# db.update_entry('media_file', file_id,
# {
# 'file_name': download_file.name,
# 'should_download': 0,
# 'cloud_link': download_file.absolute()}
# )
for entry_id in downloads:
result = db.download_file(entry_id, download_dir=data['media_dir'])
if result is not None:
self.app.log.info(f"file {result} successfully downloaded")
@ex(
help='add url to database',
python/kontor-cli/requirements.txt
-1
View File
@@ -1,5 +1,4 @@
-e ../kontor-schema
-e ../kontor-video
cement==3.0.12
cement[jinja2]
python/kontor-gui/gui/main_window.py
+30 -22
View File
@@ -1,3 +1,4 @@
from PySide6.QtCore import Qt, QThreadPool
from PySide6.QtGui import QAction, QIcon, QGuiApplication
from PySide6.QtWidgets import QWidget, QVBoxLayout, QMenu, QMessageBox, QTabWidget, QTableView, QProgressBar, QMdiArea
from PySide6.QtWidgets import QLabel, QMainWindow
@@ -10,6 +11,7 @@ from .progress import ProgressUpdate
from .dialogs import ExportKontorDialog, ImportKontorDialog
from .model_config import KontorModelConfig
from .table_model import KontorTableModel
from .worker import VideoDownloader
class MainWindow(QMainWindow):
@@ -17,6 +19,7 @@ class MainWindow(QMainWindow):
def __init__(self, engine: Engine, log):
super().__init__()
self.downloader = None
self.tick = QIcon('res/tick.png')
self.cross = QIcon('res/cross.png')
self.import_icon = QIcon("res/application-import.png")
@@ -28,29 +31,20 @@ class MainWindow(QMainWindow):
self.kontor_db = KontorDB(engine, log)
self.log = log
self._subwindows = {}
self.media_dir = "/data/media"
self.dl_tool = "yt-dlp"
self._setup_ui()
#self.tabs = QTabWidget()
#self.tabs.addTab(self.generate_data_tab("comic"), "Comics")
#self.tabs.addTab(self.generate_data_tab("media_file"), "MediaFile")
#self.tabs.currentChanged.connect(self._tab_changed)
#label.setAlignment(Qt.AlignmentFlag.AlignCenter)
#parent_layout.addWidget(self.tabs)
self.setCentralWidget(self.central_widget)
def _setup_ui(self):
self.setWindowTitle("Kontor")
self.setMinimumSize(1200, 800)
self._create_actions()
self.central_widget = QWidget()
# parent_layout = QVBoxLayout()
# self.central_widget.setLayout(parent_layout)
self.mdi_area = QMdiArea(self.central_widget)
self.mdi_area = QMdiArea()
self.setCentralWidget(self.mdi_area)
self.mdi_area.setObjectName('mdi_area')
self.setCentralWidget(self.central_widget)
self.mdi_area.setHorizontalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAsNeeded)
self.mdi_area.setVerticalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAsNeeded)
self._create_menubar()
self._create_toolbars()
self.status_progress = QProgressBar()
@@ -77,7 +71,7 @@ class MainWindow(QMainWindow):
self.updateTitleAction = QAction("&Update Titles", self)
self.updateTitleAction.triggered.connect(self.update_title)
self.downloadAction = QAction("&Download Videos", self)
self.downloadAction.triggered.connect(self.download_file)
self.downloadAction.triggered.connect(self.start_download)
self.checkFileAction = QAction("&Check files", self)
self.checkFileAction.triggered.connect(self.check_files)
self.exitAction = QAction("&Beenden", self)
@@ -189,18 +183,32 @@ class MainWindow(QMainWindow):
self.log.info("update title for table MediaFile")
self.statusBar.showMessage("update title for table MediaFile", 3000)
self.status_progress.setEnabled(True)
self.kontor_db.update_title()
self.kontor_db.update_titles()
self.status_progress.setEnabled(False)
self.refresh()
def download_file(self):
self.log.info("download videos for table MediaFile")
self.statusBar.showMessage("download videos for table MediaFile", 3000)
def start_download(self):
self.status_progress.setEnabled(True)
self.kontor_db.download_file(False, self.progress_update)
self.status_progress.setEnabled(False)
self.statusBar.showMessage("download videos for table MediaFile", 3000)
self.downloader = VideoDownloader(self.kontor_db, self.log)
self.downloader.setTotalProgress.connect(self.status_progress.setMaximum)
self.downloader.setCurrentProgress.connect(self.downloadProgress)
self.downloader.succeeded.connect(self.downloadSucceeded)
self.downloader.finished.connect(self.downloadFinished)
self.downloader.start()
def downloadProgress(self, value: int):
self.status_progress.setValue(value)
self.refresh()
def downloadSucceeded(self):
self.status_progress.setValue(self.status_progress.maximum())
self.statusBar.showMessage("Download succeeded", 3000)
def downloadFinished(self):
self.status_progress.setEnabled(False)
del self.downloader
def check_files(self):
self.log.info("check files")
self.statusBar.showMessage("check files for table MediaFile", 3000)
python/kontor-gui/gui/media_window.py
+1 -1
View File
@@ -35,7 +35,7 @@ class MediaWindow(QMdiSubWindow):
self._main_window.remove_sub_window('comic')
def refresh(self):
# self.log.info("refresh")
self.log.info("MediaWindow.refresh")
self.data_views[self.tabs.currentIndex()].refresh()
def _tab_changed(self, tab_index):
python/kontor-gui/gui/worker.py
+28
View File
@@ -0,0 +1,28 @@
import sys
from PySide6.QtCore import QObject, Signal, QRunnable, Slot, QThread
class VideoDownloader(QThread):
# Signal for the window to establish the maximum value
# of the progress bar.
setTotalProgress = Signal(int)
# Signal to increase the progress.
setCurrentProgress = Signal(int)
# Signal to be emitted when the file has been downloaded successfully.
succeeded = Signal()
def __init__(self, kontor_db, log):
super().__init__()
self.kontor_db = kontor_db
self.log = log
def run(self):
self.log.info("download videos for table MediaFile")
download_entries = self.kontor_db.get_download_list()
self.setTotalProgress.emit(len(download_entries))
for index, entry in enumerate(download_entries):
self.kontor_db.download_file(entry)
self.setCurrentProgress.emit(index)
self.succeeded.emit()
python/kontor-gui/requirements.txt
-3
View File
@@ -1,9 +1,6 @@
-e ../kontor-schema
-e ../kontor-video
platformdirs
pyyaml
PySide6
beautifulsoup4
requests
python/kontor-schema/kontor_schema/__init__.py
+13 -5
View File
@@ -313,7 +313,7 @@ class KontorDB:
result['error'] = error.orig
return result
def get_update_list(self) -> dict:
def update_titles(self) -> dict:
update_list = {}
__session__ = sessionmaker(self.engine)
with __session__() as session:
@@ -327,8 +327,8 @@ class KontorDB:
update_list[link.id] = link.title
return update_list
def get_download_list(self, download_dir: str) -> dict:
download_list = {}
def get_download_list(self) -> list:
download_list = []
__session__ = sessionmaker(self.engine)
with __session__() as session:
links = session.query(MediaFile).filter(MediaFile.should_download == 1).all()
@@ -336,10 +336,18 @@ class KontorDB:
url = link.url
if url is None:
continue
link.download_file(download_dir)
download_list[link.id] = link.file_name
download_list.append(link.id)
return download_list
def download_file(self, entry_id: str, download_dir = "/data/media", dl_tool = "yt-dlp") -> str:
__session__ = sessionmaker(self.engine)
with __session__() as session:
link = session.query(MediaFile).get(entry_id)
link.download_file(download_dir, dl_tool)
session.commit()
file_name = link.file_name
return file_name
def delete_entries(self):
for (table_name, table) in self.registry.items():
# self.log.info("delete entries from table %s", table_name)
python/kontor-schema/kontor_schema/media.py
+37 -2
View File
@@ -1,3 +1,8 @@
import re
import subprocess
from datetime import datetime
from pathlib import Path
import requests
from bs4 import BeautifulSoup
from sqlalchemy import Column, DateTime, Integer, String
@@ -26,9 +31,39 @@ class MediaFile(Base, BaseMixin, BaseVideoMixin):
except:
self.title = None
self.review = 1
self.last_modified_date = datetime.now()
def download_file(self, download_dir: str):
print(f"download file for {self.url}")
def download_file(self, download_dir: str, dl_tool: str):
print(f"download file for {self.url} to {download_dir}")
result = subprocess.run([dl_tool, self.url], cwd=download_dir, capture_output=True, text=True)
if result.returncode == 0:
output = result.stdout
output = re.sub(' +', ' ', output)
lines_list = output.splitlines()
file_name = self.__parse_output__(lines_list)
if file_name is None:
self.review = 1
self.should_download = 1
self.file_name = None
else:
download_file = Path(file_name)
self.should_download = 0
self.file_name = download_file.name
self.cloud_link = str(download_file.absolute())
self.last_modified_date = datetime.now()
def __parse_output__(self, lines_list):
self.file_name = None
for line in lines_list:
if 'has already been downloaded' in line:
end_len = len(' has already been downloaded')
self.file_name = line[11:-end_len]
if 'Destination' in line:
line_len = len(line)
start_len = len('[download] Destination: ')
file_len = line_len - start_len
self.file_name = line[-file_len:]
return self.file_name
class MediaArticle(Base, BaseMixin):
python/kontor-schema/setup.py
+2 -2
View File
@@ -8,7 +8,7 @@ long_description = ( here / "README.md").read_text(encoding="utf-8")
setup(
name='kontor_schema',
version='0.1.0',
description='Schema for Konotor DB',
description='Schema for Kontor DB',
long_description=long_description,
long_description_content_type="text/markdown",
author='Thomas Peetz',
@@ -18,6 +18,6 @@ setup(
"License :: OSI Approved :: MIT License",
"Programming Language :: Python :: 3.11",
],
install_requires=["sqlalchemy", "mariadb"],
install_requires=["sqlalchemy", "mariadb", "requests", "beautifulsoup4"],
packages=find_packages(),
)
python/kontor-video/README.md
-3
View File
@@ -1,3 +0,0 @@
# Kontor Video
This project provides helper methods to handle video links, like Youtube or ZDF Mediathek.
python/kontor-video/kontor_video/__init__.py
-63
View File
@@ -1,63 +0,0 @@
import re
import subprocess
from pathlib import Path
import requests
from bs4 import BeautifulSoup
class VideoLink:
def __init__(self, url: str, dl_tool: str, table: str):
self.file_name = None
self.url = url
self.title = None
self.dl_tool = dl_tool
self.table = table
def get_title(self) -> str:
try:
r = requests.get(self.url)
soup = BeautifulSoup(r.content, "html.parser")
title = soup.title.string
except:
title = None
return title
def download(self, download_dir=None):
if download_dir is None:
download_dir = Path.cwd()
result = subprocess.run([self.dl_tool, self.url], cwd=download_dir, capture_output=True, text=True)
if result.returncode == 0:
output = result.stdout
output = re.sub(' +', ' ', output)
lines_list = output.splitlines()
return self.__parse_output__(lines_list)
else:
return None
def __parse_output__(self, lines_list):
self.file_name = ""
for line in lines_list:
if 'has already been downloaded' in line:
end_len = len(' has already been downloaded')
self.file_name = line[11:-end_len]
if 'Destination' in line:
line_len = len(line)
start_len = len('[download] Destination: ')
file_len = line_len - start_len
self.file_name = line[-file_len:]
return self.file_name
class MediaFile(VideoLink):
def __init__(self, url: str, dl_tool='yt-dlp'):
super().__init__(url, dl_tool, 'media_file')
class MediaVideo(VideoLink):
def __init__(self, url: str, dl_tool='yt-dlp'):
super().__init__(url, dl_tool, 'media_video')
python/kontor-video/pyvenv.cfg
-5
View File
@@ -1,5 +0,0 @@
home = /usr/bin
include-system-site-packages = false
version = 3.11.2
executable = /usr/bin/python3.11
command = /usr/bin/python -m venv /home/tpeetz/projects/kontor/python/kontor-video
python/kontor-video/requirements.txt
-2
View File
@@ -1,2 +0,0 @@
beautifulsoup4
requests
python/kontor-video/setup.py
-23
View File
@@ -1,23 +0,0 @@
from setuptools import setup, find_packages
import pathlib
here = pathlib.Path(__file__).parent.resolve()
long_description = ( here / "README.md").read_text(encoding="utf-8")
setup(
name='kontor_video',
version='0.1.0',
description='Helper methods to download videos',
long_description=long_description,
long_description_content_type="text/markdown",
author='Thomas Peetz',
classifiers=[
"Development Status :: 3 - Alpha",
"Intended Audience :: Developers",
"License :: OSI Approved :: MIT License",
"Programming Language :: Python :: 3.11",
],
install_requires=["beautifulsoup4"],
packages=find_packages(),
)