moved update and download functionality to kontor-schema

python/kontor-cli/kontor/controllers/media.py

@@ -1,14 +1,4 @@
-from enum import Enum
-from pathlib import Path
-
 from cement import Controller, ex
-from kontor_schema import KontorDB
-from kontor_video import VideoLink, MediaVideo
-
-
-class VideoType(Enum):
-    MEDIA_FILE = "media_file"
-    MEDIA_VIDEO = "media_video"
 
 
 class Media(Controller):
@@ -23,8 +13,8 @@ class Media(Controller):
     )
     def update_title(self):
         db = self.app.kontor_db
-        updates = db.get_update_list()
+        updates = db.update_titles()
-        self.app.log.info(f"found {len(updates)} links for update")
+        self.app.log.info(f"{len(updates)} entries updated")
 
     @ex(
         label='download',
@@ -43,25 +33,12 @@ class Media(Controller):
         if self.app.pargs.media_dir is not None:
             data['media_dir'] = self.app.pargs.media_dir
         db = self.app.kontor_db
-        downloads = db.get_download_list(data['media_dir'])
+        downloads = db.get_download_list()
         self.app.log.info(f"found {len(downloads)} links for download")
-        #for file_id, url in downloads.items():
+        for entry_id in downloads:
-        #    link = VideoLink(url, VideoType.MEDIA_FILE)
+            result = db.download_file(entry_id, download_dir=data['media_dir'])
-        #    file_name = link.download(download_dir=data['media_dir'])
+            if result is not None:
-        #    if file_name is None:
+                self.app.log.info(f"file {result} successfully downloaded")
-        #        db.update_entry('media_file', file_id, {'file_name': None, 'should_download': 1})
-        #    else:
-        #        download_file = Path(file_name)
-        #        download_file.with_name(f"{file_id}{download_file.suffix}")
-        #        link.file_name = download_file.name
-        #        link.should_download = 0
-        #        link.cloud_link = download_file.absolute()
-        #        db.update_entry('media_file', file_id,
-        #                               {
-        #                                   'file_name': download_file.name,
-        #                                   'should_download': 0,
-        #                                   'cloud_link': download_file.absolute()}
-        #                               )
 
     @ex(
         help='add url to database',

python/kontor-cli/requirements.txt

@@ -1,5 +1,4 @@
 -e ../kontor-schema
--e ../kontor-video
  
 cement==3.0.12
 cement[jinja2]

python/kontor-gui/gui/main_window.py

@@ -1,3 +1,4 @@
+from PySide6.QtCore import Qt, QThreadPool
 from PySide6.QtGui import QAction, QIcon, QGuiApplication
 from PySide6.QtWidgets import QWidget, QVBoxLayout, QMenu, QMessageBox, QTabWidget, QTableView, QProgressBar, QMdiArea
 from PySide6.QtWidgets import QLabel, QMainWindow
@@ -10,6 +11,7 @@ from .progress import ProgressUpdate
 from .dialogs import ExportKontorDialog, ImportKontorDialog
 from .model_config import KontorModelConfig
 from .table_model import KontorTableModel
+from .worker import VideoDownloader
 
 
 class MainWindow(QMainWindow):
@@ -17,6 +19,7 @@ class MainWindow(QMainWindow):
     def __init__(self, engine: Engine, log):
         super().__init__()
 
+        self.downloader = None
         self.tick = QIcon('res/tick.png')
         self.cross = QIcon('res/cross.png')
         self.import_icon = QIcon("res/application-import.png")
@@ -28,29 +31,20 @@ class MainWindow(QMainWindow):
         self.kontor_db = KontorDB(engine, log)
         self.log = log
         self._subwindows = {}
+        self.media_dir = "/data/media"
+        self.dl_tool = "yt-dlp"
 
         self._setup_ui()
 
-
-        #self.tabs = QTabWidget()
-        #self.tabs.addTab(self.generate_data_tab("comic"), "Comics")
-        #self.tabs.addTab(self.generate_data_tab("media_file"), "MediaFile")
-        #self.tabs.currentChanged.connect(self._tab_changed)
-        #label.setAlignment(Qt.AlignmentFlag.AlignCenter)
-        #parent_layout.addWidget(self.tabs)
-
-        self.setCentralWidget(self.central_widget)
-
     def _setup_ui(self):
         self.setWindowTitle("Kontor")
         self.setMinimumSize(1200, 800)
         self._create_actions()
-        self.central_widget = QWidget()
+        self.mdi_area = QMdiArea()
-        # parent_layout = QVBoxLayout()
+        self.setCentralWidget(self.mdi_area)
-        # self.central_widget.setLayout(parent_layout)
-        self.mdi_area = QMdiArea(self.central_widget)
         self.mdi_area.setObjectName('mdi_area')
-        self.setCentralWidget(self.central_widget)
+        self.mdi_area.setHorizontalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAsNeeded)
+        self.mdi_area.setVerticalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAsNeeded)
         self._create_menubar()
         self._create_toolbars()
         self.status_progress = QProgressBar()
@@ -77,7 +71,7 @@ class MainWindow(QMainWindow):
         self.updateTitleAction = QAction("&Update Titles", self)
         self.updateTitleAction.triggered.connect(self.update_title)
         self.downloadAction = QAction("&Download Videos", self)
-        self.downloadAction.triggered.connect(self.download_file)
+        self.downloadAction.triggered.connect(self.start_download)
         self.checkFileAction = QAction("&Check files", self)
         self.checkFileAction.triggered.connect(self.check_files)
         self.exitAction = QAction("&Beenden", self)
@@ -189,18 +183,32 @@ class MainWindow(QMainWindow):
         self.log.info("update title for table MediaFile")
         self.statusBar.showMessage("update title for table MediaFile", 3000)
         self.status_progress.setEnabled(True)
-        self.kontor_db.update_title()
+        self.kontor_db.update_titles()
         self.status_progress.setEnabled(False)
         self.refresh()
 
-    def download_file(self):
+    def start_download(self):
-        self.log.info("download videos for table MediaFile")
-        self.statusBar.showMessage("download videos for table MediaFile", 3000)
         self.status_progress.setEnabled(True)
-        self.kontor_db.download_file(False, self.progress_update)
+        self.statusBar.showMessage("download videos for table MediaFile", 3000)
-        self.status_progress.setEnabled(False)
+        self.downloader = VideoDownloader(self.kontor_db, self.log)
+        self.downloader.setTotalProgress.connect(self.status_progress.setMaximum)
+        self.downloader.setCurrentProgress.connect(self.downloadProgress)
+        self.downloader.succeeded.connect(self.downloadSucceeded)
+        self.downloader.finished.connect(self.downloadFinished)
+        self.downloader.start()
+
+    def downloadProgress(self, value: int):
+        self.status_progress.setValue(value)
         self.refresh()
 
+    def downloadSucceeded(self):
+        self.status_progress.setValue(self.status_progress.maximum())
+        self.statusBar.showMessage("Download succeeded", 3000)
+
+    def downloadFinished(self):
+        self.status_progress.setEnabled(False)
+        del self.downloader
+
     def check_files(self):
         self.log.info("check files")
         self.statusBar.showMessage("check files for table MediaFile", 3000)

python/kontor-gui/gui/media_window.py

@@ -35,7 +35,7 @@ class MediaWindow(QMdiSubWindow):
         self._main_window.remove_sub_window('comic')
 
     def refresh(self):
-        # self.log.info("refresh")
+        self.log.info("MediaWindow.refresh")
         self.data_views[self.tabs.currentIndex()].refresh()
 
     def _tab_changed(self, tab_index):

python/kontor-gui/gui/worker.py

@@ -0,0 +1,28 @@
+import sys
+
+from PySide6.QtCore import QObject, Signal, QRunnable, Slot, QThread
+
+
+class VideoDownloader(QThread):
+    # Signal for the window to establish the maximum value
+    # of the progress bar.
+    setTotalProgress = Signal(int)
+    # Signal to increase the progress.
+    setCurrentProgress = Signal(int)
+    # Signal to be emitted when the file has been downloaded successfully.
+    succeeded = Signal()
+
+    def __init__(self, kontor_db, log):
+        super().__init__()
+        self.kontor_db = kontor_db
+        self.log = log
+
+    def run(self):
+        self.log.info("download videos for table MediaFile")
+        download_entries = self.kontor_db.get_download_list()
+        self.setTotalProgress.emit(len(download_entries))
+        for index, entry in enumerate(download_entries):
+            self.kontor_db.download_file(entry)
+            self.setCurrentProgress.emit(index)
+        self.succeeded.emit()
+

python/kontor-gui/requirements.txt

@@ -1,9 +1,6 @@
 -e ../kontor-schema
--e ../kontor-video
  
 platformdirs
 pyyaml
 PySide6
-beautifulsoup4
-requests
 

python/kontor-schema/kontor_schema/__init__.py

@@ -313,7 +313,7 @@ class KontorDB:
                 result['error'] = error.orig
         return result
 
-    def get_update_list(self) -> dict:
+    def update_titles(self) -> dict:
         update_list = {}
         __session__ = sessionmaker(self.engine)
         with __session__() as session:
@@ -327,8 +327,8 @@ class KontorDB:
                 update_list[link.id] = link.title
         return update_list
 
-    def get_download_list(self, download_dir: str) -> dict:
+    def get_download_list(self) -> list:
-        download_list = {}
+        download_list = []
         __session__ = sessionmaker(self.engine)
         with __session__() as session:
             links = session.query(MediaFile).filter(MediaFile.should_download == 1).all()
@@ -336,10 +336,18 @@ class KontorDB:
                 url = link.url
                 if url is None:
                     continue
-                link.download_file(download_dir)
+                download_list.append(link.id)
-                download_list[link.id] = link.file_name
         return download_list
 
+    def download_file(self, entry_id: str, download_dir = "/data/media", dl_tool = "yt-dlp") -> str:
+        __session__ = sessionmaker(self.engine)
+        with __session__() as session:
+            link = session.query(MediaFile).get(entry_id)
+            link.download_file(download_dir, dl_tool)
+            session.commit()
+            file_name = link.file_name
+        return file_name
+
     def delete_entries(self):
         for (table_name, table) in self.registry.items():
             # self.log.info("delete entries from table %s", table_name)

python/kontor-schema/kontor_schema/media.py

@@ -1,3 +1,8 @@
+import re
+import subprocess
+from datetime import datetime
+from pathlib import Path
+
 import requests
 from bs4 import BeautifulSoup
 from sqlalchemy import Column, DateTime, Integer, String
@@ -26,9 +31,39 @@ class MediaFile(Base, BaseMixin, BaseVideoMixin):
         except:
             self.title = None
             self.review = 1
+        self.last_modified_date = datetime.now()
 
-    def download_file(self, download_dir: str):
+    def download_file(self, download_dir: str, dl_tool: str):
-        print(f"download file for {self.url}")
+        print(f"download file for {self.url} to {download_dir}")
+        result = subprocess.run([dl_tool, self.url], cwd=download_dir, capture_output=True, text=True)
+        if result.returncode == 0:
+            output = result.stdout
+            output = re.sub(' +', ' ', output)
+            lines_list = output.splitlines()
+            file_name = self.__parse_output__(lines_list)
+            if file_name is None:
+                self.review = 1
+                self.should_download = 1
+                self.file_name = None
+            else:
+                download_file = Path(file_name)
+                self.should_download = 0
+                self.file_name = download_file.name
+                self.cloud_link = str(download_file.absolute())
+            self.last_modified_date = datetime.now()
+
+    def __parse_output__(self, lines_list):
+        self.file_name = None
+        for line in lines_list:
+            if 'has already been downloaded' in line:
+                end_len = len(' has already been downloaded')
+                self.file_name = line[11:-end_len]
+            if 'Destination' in line:
+                line_len = len(line)
+                start_len = len('[download] Destination: ')
+                file_len = line_len - start_len
+                self.file_name = line[-file_len:]
+        return self.file_name
 
 
 class MediaArticle(Base, BaseMixin):

python/kontor-schema/setup.py

@@ -8,7 +8,7 @@ long_description = ( here / "README.md").read_text(encoding="utf-8")
 setup(
     name='kontor_schema',
     version='0.1.0',
-    description='Schema for Konotor DB',
+    description='Schema for Kontor DB',
     long_description=long_description,
     long_description_content_type="text/markdown",
     author='Thomas Peetz',
@@ -18,6 +18,6 @@ setup(
         "License :: OSI Approved :: MIT License",
         "Programming Language :: Python :: 3.11",
     ],
-    install_requires=["sqlalchemy", "mariadb"],
+    install_requires=["sqlalchemy", "mariadb", "requests", "beautifulsoup4"],
     packages=find_packages(),
 )

python/kontor-video/README.md

@@ -1,3 +0,0 @@
-# Kontor Video
-
-This project provides helper methods to handle video links, like Youtube or ZDF Mediathek.

python/kontor-video/kontor_video/__init__.py

@@ -1,63 +0,0 @@
-import re
-import subprocess
-from pathlib import Path
-
-import requests
-from bs4 import BeautifulSoup
-
-
-class VideoLink:
-
-    def __init__(self, url: str, dl_tool: str, table: str):
-        self.file_name = None
-        self.url = url
-        self.title = None
-        self.dl_tool = dl_tool
-        self.table = table
-
-    def get_title(self) -> str:
-        try:
-            r = requests.get(self.url)
-            soup = BeautifulSoup(r.content, "html.parser")
-            title = soup.title.string
-        except:
-            title = None
-        return title
-
-
-    def download(self, download_dir=None):
-        if download_dir is None:
-            download_dir = Path.cwd()
-        result = subprocess.run([self.dl_tool, self.url], cwd=download_dir, capture_output=True, text=True)
-        if result.returncode == 0:
-            output = result.stdout
-            output = re.sub(' +', ' ', output)
-            lines_list = output.splitlines()
-            return self.__parse_output__(lines_list)
-        else:
-            return None
-
-    def __parse_output__(self, lines_list):
-        self.file_name = ""
-        for line in lines_list:
-            if 'has already been downloaded' in line:
-                end_len = len(' has already been downloaded')
-                self.file_name = line[11:-end_len]
-            if 'Destination' in line:
-                line_len = len(line)
-                start_len = len('[download] Destination: ')
-                file_len = line_len - start_len
-                self.file_name = line[-file_len:]
-        return self.file_name
-
-
-class MediaFile(VideoLink):
-
-    def __init__(self, url: str, dl_tool='yt-dlp'):
-        super().__init__(url, dl_tool, 'media_file')
-
-
-class MediaVideo(VideoLink):
-
-    def __init__(self, url: str, dl_tool='yt-dlp'):
-        super().__init__(url, dl_tool, 'media_video')

python/kontor-video/pyvenv.cfg

@@ -1,5 +0,0 @@
-home = /usr/bin
-include-system-site-packages = false
-version = 3.11.2
-executable = /usr/bin/python3.11
-command = /usr/bin/python -m venv /home/tpeetz/projects/kontor/python/kontor-video

python/kontor-video/requirements.txt

@@ -1,2 +0,0 @@
-beautifulsoup4
-requests

python/kontor-video/setup.py

@@ -1,23 +0,0 @@
-from setuptools import setup, find_packages
-import pathlib
-
-here = pathlib.Path(__file__).parent.resolve()
-
-long_description = ( here / "README.md").read_text(encoding="utf-8")
-
-setup(
-    name='kontor_video',
-    version='0.1.0',
-    description='Helper methods to download videos',
-    long_description=long_description,
-    long_description_content_type="text/markdown",
-    author='Thomas Peetz',
-    classifiers=[
-        "Development Status :: 3 - Alpha",
-        "Intended Audience :: Developers",
-        "License :: OSI Approved :: MIT License",
-        "Programming Language :: Python :: 3.11",
-    ],
-    install_requires=["beautifulsoup4"],
-    packages=find_packages(),
-)