moved update and download functionality to kontor-schema

python/kontor-schema/kontor_schema/__init__.py

@@ -313,7 +313,7 @@ class KontorDB:
                 result['error'] = error.orig
         return result
 
-    def get_update_list(self) -> dict:
+    def update_titles(self) -> dict:
         update_list = {}
         __session__ = sessionmaker(self.engine)
         with __session__() as session:
@@ -327,8 +327,8 @@ class KontorDB:
                 update_list[link.id] = link.title
         return update_list
 
-    def get_download_list(self, download_dir: str) -> dict:
-        download_list = {}
+    def get_download_list(self) -> list:
+        download_list = []
         __session__ = sessionmaker(self.engine)
         with __session__() as session:
             links = session.query(MediaFile).filter(MediaFile.should_download == 1).all()
@@ -336,10 +336,18 @@ class KontorDB:
                 url = link.url
                 if url is None:
                     continue
-                link.download_file(download_dir)
-                download_list[link.id] = link.file_name
+                download_list.append(link.id)
         return download_list
 
+    def download_file(self, entry_id: str, download_dir = "/data/media", dl_tool = "yt-dlp") -> str:
+        __session__ = sessionmaker(self.engine)
+        with __session__() as session:
+            link = session.query(MediaFile).get(entry_id)
+            link.download_file(download_dir, dl_tool)
+            session.commit()
+            file_name = link.file_name
+        return file_name
+
     def delete_entries(self):
         for (table_name, table) in self.registry.items():
             # self.log.info("delete entries from table %s", table_name)

python/kontor-schema/kontor_schema/media.py

@@ -1,3 +1,8 @@
+import re
+import subprocess
+from datetime import datetime
+from pathlib import Path
+
 import requests
 from bs4 import BeautifulSoup
 from sqlalchemy import Column, DateTime, Integer, String
@@ -26,9 +31,39 @@ class MediaFile(Base, BaseMixin, BaseVideoMixin):
         except:
             self.title = None
             self.review = 1
+        self.last_modified_date = datetime.now()
 
-    def download_file(self, download_dir: str):
-        print(f"download file for {self.url}")
+    def download_file(self, download_dir: str, dl_tool: str):
+        print(f"download file for {self.url} to {download_dir}")
+        result = subprocess.run([dl_tool, self.url], cwd=download_dir, capture_output=True, text=True)
+        if result.returncode == 0:
+            output = result.stdout
+            output = re.sub(' +', ' ', output)
+            lines_list = output.splitlines()
+            file_name = self.__parse_output__(lines_list)
+            if file_name is None:
+                self.review = 1
+                self.should_download = 1
+                self.file_name = None
+            else:
+                download_file = Path(file_name)
+                self.should_download = 0
+                self.file_name = download_file.name
+                self.cloud_link = str(download_file.absolute())
+            self.last_modified_date = datetime.now()
+
+    def __parse_output__(self, lines_list):
+        self.file_name = None
+        for line in lines_list:
+            if 'has already been downloaded' in line:
+                end_len = len(' has already been downloaded')
+                self.file_name = line[11:-end_len]
+            if 'Destination' in line:
+                line_len = len(line)
+                start_len = len('[download] Destination: ')
+                file_len = line_len - start_len
+                self.file_name = line[-file_len:]
+        return self.file_name
 
 
 class MediaArticle(Base, BaseMixin):

python/kontor-schema/setup.py

@@ -8,7 +8,7 @@ long_description = ( here / "README.md").read_text(encoding="utf-8")
 setup(
     name='kontor_schema',
     version='0.1.0',
-    description='Schema for Konotor DB',
+    description='Schema for Kontor DB',
     long_description=long_description,
     long_description_content_type="text/markdown",
     author='Thomas Peetz',
@@ -18,6 +18,6 @@ setup(
         "License :: OSI Approved :: MIT License",
         "Programming Language :: Python :: 3.11",
     ],
-    install_requires=["sqlalchemy", "mariadb"],
+    install_requires=["sqlalchemy", "mariadb", "requests", "beautifulsoup4"],
     packages=find_packages(),
 )