diff --git a/python/kontor-cli/kontor/controllers/media.py b/python/kontor-cli/kontor/controllers/media.py index 135366e..55148fc 100644 --- a/python/kontor-cli/kontor/controllers/media.py +++ b/python/kontor-cli/kontor/controllers/media.py @@ -1,14 +1,4 @@ -from enum import Enum -from pathlib import Path - from cement import Controller, ex -from kontor_schema import KontorDB -from kontor_video import VideoLink, MediaVideo - - -class VideoType(Enum): - MEDIA_FILE = "media_file" - MEDIA_VIDEO = "media_video" class Media(Controller): @@ -23,8 +13,8 @@ class Media(Controller): ) def update_title(self): db = self.app.kontor_db - updates = db.get_update_list() - self.app.log.info(f"found {len(updates)} links for update") + updates = db.update_titles() + self.app.log.info(f"{len(updates)} entries updated") @ex( label='download', @@ -43,25 +33,12 @@ class Media(Controller): if self.app.pargs.media_dir is not None: data['media_dir'] = self.app.pargs.media_dir db = self.app.kontor_db - downloads = db.get_download_list(data['media_dir']) + downloads = db.get_download_list() self.app.log.info(f"found {len(downloads)} links for download") - #for file_id, url in downloads.items(): - # link = VideoLink(url, VideoType.MEDIA_FILE) - # file_name = link.download(download_dir=data['media_dir']) - # if file_name is None: - # db.update_entry('media_file', file_id, {'file_name': None, 'should_download': 1}) - # else: - # download_file = Path(file_name) - # download_file.with_name(f"{file_id}{download_file.suffix}") - # link.file_name = download_file.name - # link.should_download = 0 - # link.cloud_link = download_file.absolute() - # db.update_entry('media_file', file_id, - # { - # 'file_name': download_file.name, - # 'should_download': 0, - # 'cloud_link': download_file.absolute()} - # ) + for entry_id in downloads: + result = db.download_file(entry_id, download_dir=data['media_dir']) + if result is not None: + self.app.log.info(f"file {result} successfully downloaded") @ex( help='add url to database', diff --git a/python/kontor-cli/requirements.txt b/python/kontor-cli/requirements.txt index 58833ba..814fe5a 100644 --- a/python/kontor-cli/requirements.txt +++ b/python/kontor-cli/requirements.txt @@ -1,5 +1,4 @@ -e ../kontor-schema --e ../kontor-video cement==3.0.12 cement[jinja2] diff --git a/python/kontor-gui/gui/main_window.py b/python/kontor-gui/gui/main_window.py index 5a97c6a..6ddd612 100644 --- a/python/kontor-gui/gui/main_window.py +++ b/python/kontor-gui/gui/main_window.py @@ -1,3 +1,4 @@ +from PySide6.QtCore import Qt, QThreadPool from PySide6.QtGui import QAction, QIcon, QGuiApplication from PySide6.QtWidgets import QWidget, QVBoxLayout, QMenu, QMessageBox, QTabWidget, QTableView, QProgressBar, QMdiArea from PySide6.QtWidgets import QLabel, QMainWindow @@ -10,6 +11,7 @@ from .progress import ProgressUpdate from .dialogs import ExportKontorDialog, ImportKontorDialog from .model_config import KontorModelConfig from .table_model import KontorTableModel +from .worker import VideoDownloader class MainWindow(QMainWindow): @@ -17,6 +19,7 @@ class MainWindow(QMainWindow): def __init__(self, engine: Engine, log): super().__init__() + self.downloader = None self.tick = QIcon('res/tick.png') self.cross = QIcon('res/cross.png') self.import_icon = QIcon("res/application-import.png") @@ -28,29 +31,20 @@ class MainWindow(QMainWindow): self.kontor_db = KontorDB(engine, log) self.log = log self._subwindows = {} + self.media_dir = "/data/media" + self.dl_tool = "yt-dlp" self._setup_ui() - - #self.tabs = QTabWidget() - #self.tabs.addTab(self.generate_data_tab("comic"), "Comics") - #self.tabs.addTab(self.generate_data_tab("media_file"), "MediaFile") - #self.tabs.currentChanged.connect(self._tab_changed) - #label.setAlignment(Qt.AlignmentFlag.AlignCenter) - #parent_layout.addWidget(self.tabs) - - self.setCentralWidget(self.central_widget) - def _setup_ui(self): self.setWindowTitle("Kontor") self.setMinimumSize(1200, 800) self._create_actions() - self.central_widget = QWidget() - # parent_layout = QVBoxLayout() - # self.central_widget.setLayout(parent_layout) - self.mdi_area = QMdiArea(self.central_widget) + self.mdi_area = QMdiArea() + self.setCentralWidget(self.mdi_area) self.mdi_area.setObjectName('mdi_area') - self.setCentralWidget(self.central_widget) + self.mdi_area.setHorizontalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAsNeeded) + self.mdi_area.setVerticalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAsNeeded) self._create_menubar() self._create_toolbars() self.status_progress = QProgressBar() @@ -77,7 +71,7 @@ class MainWindow(QMainWindow): self.updateTitleAction = QAction("&Update Titles", self) self.updateTitleAction.triggered.connect(self.update_title) self.downloadAction = QAction("&Download Videos", self) - self.downloadAction.triggered.connect(self.download_file) + self.downloadAction.triggered.connect(self.start_download) self.checkFileAction = QAction("&Check files", self) self.checkFileAction.triggered.connect(self.check_files) self.exitAction = QAction("&Beenden", self) @@ -189,18 +183,32 @@ class MainWindow(QMainWindow): self.log.info("update title for table MediaFile") self.statusBar.showMessage("update title for table MediaFile", 3000) self.status_progress.setEnabled(True) - self.kontor_db.update_title() + self.kontor_db.update_titles() self.status_progress.setEnabled(False) self.refresh() - def download_file(self): - self.log.info("download videos for table MediaFile") - self.statusBar.showMessage("download videos for table MediaFile", 3000) + def start_download(self): self.status_progress.setEnabled(True) - self.kontor_db.download_file(False, self.progress_update) - self.status_progress.setEnabled(False) + self.statusBar.showMessage("download videos for table MediaFile", 3000) + self.downloader = VideoDownloader(self.kontor_db, self.log) + self.downloader.setTotalProgress.connect(self.status_progress.setMaximum) + self.downloader.setCurrentProgress.connect(self.downloadProgress) + self.downloader.succeeded.connect(self.downloadSucceeded) + self.downloader.finished.connect(self.downloadFinished) + self.downloader.start() + + def downloadProgress(self, value: int): + self.status_progress.setValue(value) self.refresh() + def downloadSucceeded(self): + self.status_progress.setValue(self.status_progress.maximum()) + self.statusBar.showMessage("Download succeeded", 3000) + + def downloadFinished(self): + self.status_progress.setEnabled(False) + del self.downloader + def check_files(self): self.log.info("check files") self.statusBar.showMessage("check files for table MediaFile", 3000) diff --git a/python/kontor-gui/gui/media_window.py b/python/kontor-gui/gui/media_window.py index 6518953..44c30a8 100644 --- a/python/kontor-gui/gui/media_window.py +++ b/python/kontor-gui/gui/media_window.py @@ -35,7 +35,7 @@ class MediaWindow(QMdiSubWindow): self._main_window.remove_sub_window('comic') def refresh(self): - # self.log.info("refresh") + self.log.info("MediaWindow.refresh") self.data_views[self.tabs.currentIndex()].refresh() def _tab_changed(self, tab_index): diff --git a/python/kontor-gui/gui/worker.py b/python/kontor-gui/gui/worker.py new file mode 100644 index 0000000..96ecf12 --- /dev/null +++ b/python/kontor-gui/gui/worker.py @@ -0,0 +1,28 @@ +import sys + +from PySide6.QtCore import QObject, Signal, QRunnable, Slot, QThread + + +class VideoDownloader(QThread): + # Signal for the window to establish the maximum value + # of the progress bar. + setTotalProgress = Signal(int) + # Signal to increase the progress. + setCurrentProgress = Signal(int) + # Signal to be emitted when the file has been downloaded successfully. + succeeded = Signal() + + def __init__(self, kontor_db, log): + super().__init__() + self.kontor_db = kontor_db + self.log = log + + def run(self): + self.log.info("download videos for table MediaFile") + download_entries = self.kontor_db.get_download_list() + self.setTotalProgress.emit(len(download_entries)) + for index, entry in enumerate(download_entries): + self.kontor_db.download_file(entry) + self.setCurrentProgress.emit(index) + self.succeeded.emit() + diff --git a/python/kontor-gui/requirements.txt b/python/kontor-gui/requirements.txt index 6c37bb1..a6e9a93 100644 --- a/python/kontor-gui/requirements.txt +++ b/python/kontor-gui/requirements.txt @@ -1,9 +1,6 @@ -e ../kontor-schema --e ../kontor-video platformdirs pyyaml PySide6 -beautifulsoup4 -requests diff --git a/python/kontor-schema/kontor_schema/__init__.py b/python/kontor-schema/kontor_schema/__init__.py index f14b16a..1e747a1 100644 --- a/python/kontor-schema/kontor_schema/__init__.py +++ b/python/kontor-schema/kontor_schema/__init__.py @@ -313,7 +313,7 @@ class KontorDB: result['error'] = error.orig return result - def get_update_list(self) -> dict: + def update_titles(self) -> dict: update_list = {} __session__ = sessionmaker(self.engine) with __session__() as session: @@ -327,8 +327,8 @@ class KontorDB: update_list[link.id] = link.title return update_list - def get_download_list(self, download_dir: str) -> dict: - download_list = {} + def get_download_list(self) -> list: + download_list = [] __session__ = sessionmaker(self.engine) with __session__() as session: links = session.query(MediaFile).filter(MediaFile.should_download == 1).all() @@ -336,10 +336,18 @@ class KontorDB: url = link.url if url is None: continue - link.download_file(download_dir) - download_list[link.id] = link.file_name + download_list.append(link.id) return download_list + def download_file(self, entry_id: str, download_dir = "/data/media", dl_tool = "yt-dlp") -> str: + __session__ = sessionmaker(self.engine) + with __session__() as session: + link = session.query(MediaFile).get(entry_id) + link.download_file(download_dir, dl_tool) + session.commit() + file_name = link.file_name + return file_name + def delete_entries(self): for (table_name, table) in self.registry.items(): # self.log.info("delete entries from table %s", table_name) diff --git a/python/kontor-schema/kontor_schema/media.py b/python/kontor-schema/kontor_schema/media.py index ab1e8b4..10338b4 100644 --- a/python/kontor-schema/kontor_schema/media.py +++ b/python/kontor-schema/kontor_schema/media.py @@ -1,3 +1,8 @@ +import re +import subprocess +from datetime import datetime +from pathlib import Path + import requests from bs4 import BeautifulSoup from sqlalchemy import Column, DateTime, Integer, String @@ -26,9 +31,39 @@ class MediaFile(Base, BaseMixin, BaseVideoMixin): except: self.title = None self.review = 1 + self.last_modified_date = datetime.now() - def download_file(self, download_dir: str): - print(f"download file for {self.url}") + def download_file(self, download_dir: str, dl_tool: str): + print(f"download file for {self.url} to {download_dir}") + result = subprocess.run([dl_tool, self.url], cwd=download_dir, capture_output=True, text=True) + if result.returncode == 0: + output = result.stdout + output = re.sub(' +', ' ', output) + lines_list = output.splitlines() + file_name = self.__parse_output__(lines_list) + if file_name is None: + self.review = 1 + self.should_download = 1 + self.file_name = None + else: + download_file = Path(file_name) + self.should_download = 0 + self.file_name = download_file.name + self.cloud_link = str(download_file.absolute()) + self.last_modified_date = datetime.now() + + def __parse_output__(self, lines_list): + self.file_name = None + for line in lines_list: + if 'has already been downloaded' in line: + end_len = len(' has already been downloaded') + self.file_name = line[11:-end_len] + if 'Destination' in line: + line_len = len(line) + start_len = len('[download] Destination: ') + file_len = line_len - start_len + self.file_name = line[-file_len:] + return self.file_name class MediaArticle(Base, BaseMixin): diff --git a/python/kontor-schema/setup.py b/python/kontor-schema/setup.py index 7f76710..99f68b9 100644 --- a/python/kontor-schema/setup.py +++ b/python/kontor-schema/setup.py @@ -8,7 +8,7 @@ long_description = ( here / "README.md").read_text(encoding="utf-8") setup( name='kontor_schema', version='0.1.0', - description='Schema for Konotor DB', + description='Schema for Kontor DB', long_description=long_description, long_description_content_type="text/markdown", author='Thomas Peetz', @@ -18,6 +18,6 @@ setup( "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3.11", ], - install_requires=["sqlalchemy", "mariadb"], + install_requires=["sqlalchemy", "mariadb", "requests", "beautifulsoup4"], packages=find_packages(), ) diff --git a/python/kontor-video/README.md b/python/kontor-video/README.md deleted file mode 100644 index c52965f..0000000 --- a/python/kontor-video/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Kontor Video - -This project provides helper methods to handle video links, like Youtube or ZDF Mediathek. diff --git a/python/kontor-video/kontor_video/__init__.py b/python/kontor-video/kontor_video/__init__.py deleted file mode 100644 index 8cfb548..0000000 --- a/python/kontor-video/kontor_video/__init__.py +++ /dev/null @@ -1,63 +0,0 @@ -import re -import subprocess -from pathlib import Path - -import requests -from bs4 import BeautifulSoup - - -class VideoLink: - - def __init__(self, url: str, dl_tool: str, table: str): - self.file_name = None - self.url = url - self.title = None - self.dl_tool = dl_tool - self.table = table - - def get_title(self) -> str: - try: - r = requests.get(self.url) - soup = BeautifulSoup(r.content, "html.parser") - title = soup.title.string - except: - title = None - return title - - - def download(self, download_dir=None): - if download_dir is None: - download_dir = Path.cwd() - result = subprocess.run([self.dl_tool, self.url], cwd=download_dir, capture_output=True, text=True) - if result.returncode == 0: - output = result.stdout - output = re.sub(' +', ' ', output) - lines_list = output.splitlines() - return self.__parse_output__(lines_list) - else: - return None - - def __parse_output__(self, lines_list): - self.file_name = "" - for line in lines_list: - if 'has already been downloaded' in line: - end_len = len(' has already been downloaded') - self.file_name = line[11:-end_len] - if 'Destination' in line: - line_len = len(line) - start_len = len('[download] Destination: ') - file_len = line_len - start_len - self.file_name = line[-file_len:] - return self.file_name - - -class MediaFile(VideoLink): - - def __init__(self, url: str, dl_tool='yt-dlp'): - super().__init__(url, dl_tool, 'media_file') - - -class MediaVideo(VideoLink): - - def __init__(self, url: str, dl_tool='yt-dlp'): - super().__init__(url, dl_tool, 'media_video') diff --git a/python/kontor-video/pyvenv.cfg b/python/kontor-video/pyvenv.cfg deleted file mode 100644 index e789070..0000000 --- a/python/kontor-video/pyvenv.cfg +++ /dev/null @@ -1,5 +0,0 @@ -home = /usr/bin -include-system-site-packages = false -version = 3.11.2 -executable = /usr/bin/python3.11 -command = /usr/bin/python -m venv /home/tpeetz/projects/kontor/python/kontor-video diff --git a/python/kontor-video/requirements.txt b/python/kontor-video/requirements.txt deleted file mode 100644 index 1f3e778..0000000 --- a/python/kontor-video/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -beautifulsoup4 -requests diff --git a/python/kontor-video/setup.py b/python/kontor-video/setup.py deleted file mode 100644 index 1362fdf..0000000 --- a/python/kontor-video/setup.py +++ /dev/null @@ -1,23 +0,0 @@ -from setuptools import setup, find_packages -import pathlib - -here = pathlib.Path(__file__).parent.resolve() - -long_description = ( here / "README.md").read_text(encoding="utf-8") - -setup( - name='kontor_video', - version='0.1.0', - description='Helper methods to download videos', - long_description=long_description, - long_description_content_type="text/markdown", - author='Thomas Peetz', - classifiers=[ - "Development Status :: 3 - Alpha", - "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3.11", - ], - install_requires=["beautifulsoup4"], - packages=find_packages(), -)