reorganize python projects

scripts/download.py

@@ -1,73 +1,58 @@
 """
 download files with URLs from DB
 """
-import re
-import subprocess
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
-import mariadb
-from setup import get_database_cursors, create_tables, get_logger
+from platformdirs import PlatformDirs
+from pathlib import Path
+import yaml
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import sessionmaker
+from schema import Base, KontorDB, MediaFile
+from config import get_logger
+
 
 parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
 parser.add_argument('--verbose', '-v', action='count', default=0)
+parser.add_argument('--config', '-c', default='kontor-docker')
 parser.add_argument('--dir', '-d', default='/data/media')
+parser.add_argument('--tool', '-t', default='yt-dlp')
 parser.add_argument('--dry-run', '-m', action='store_true')
 parser.add_argument('--rename', '-r', action='store_true')
 args = parser.parse_args()
 
 
-def parse_output(lines_list, log):
-    file_name = ""
-    for line in lines_list:
-        if 'has already been downloaded' in line:
-            end_len = len(' has already been downloaded')
-            file_name = line[11:-end_len]
-            log.info('found file: "%s"', file_name)
-        if 'Destination' in line:
-            line_len = len(line)
-            start_len = len('[download] Destination: ')
-            file_len = line_len-start_len
-            file_name = line[-file_len:]
-            log.info('new file: "%s"', file_name)
-    return file_name
-
-
-def download_url(video_url, log):
-    result = subprocess.run(["/home/tpeetz/bin/yt-dlp", video_url], cwd=args.dir, capture_output=True, text=True)
-    if result.returncode == 0:
-        output = result.stdout
-        output = re.sub(' +', ' ', output)
-        lines_list = output.splitlines()
-        return parse_output(lines_list, log)
-    else:
-        return None
-
-
-def download_and_update(link, entry_id, conn):
-    m_cursor = conn.cursor()
-    filename = download_url(link, logger)
-    if filename is None:
-        update_statement = 'UPDATE media_file set review = true WHERE id = ?'
-        logger.debug(f'entry {entry_id} could not downloaded, set to Review')
-        m_cursor.execute(update_statement, (entry_id,))
-    else:
-        update_statement = 'UPDATE media_file set file_name = ?, should_download = false, review = false WHERE id = ?'
-        logger.debug(f'entry {entry_id} successfully downloaded, set review and should_download to false')
-        m_cursor.execute(update_statement, (filename, entry_id))
-    conn.commit()
-
-
 if __name__ == '__main__':
-    logger = get_logger(args.verbose)
-    logger.info('kontor.download started')
-    s_conn, m_conn = get_database_cursors(logger)
-    cursor = m_conn.cursor()
-    cursor.execute('SELECT id, url FROM media_file where should_download is true')
-    for (link_id, url) in cursor.fetchall():
-        if url is None:
-            logger.info('There is no url for id {}'.format(link_id))
-        else:
-            if args.dry_run:
-                logger.info(f'download {url} for {link_id}')
-            else:
-                download_and_update(url, link_id, m_conn)
-    logger.info('kontor.download finished')
+    log = get_logger(args.verbose, args.config)
+    log.info('kontor.download started')
+    dirs = PlatformDirs(args.config)
+    database_config = Path(dirs.user_config_dir, 'database-config.yaml')
+    with open(database_config, 'rt') as f:
+        db_config = yaml.safe_load(f.read())
+    print(db_config)
+    connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format(
+        db_config['mariadb']['user'],
+        db_config['mariadb']['password'],
+        db_config['mariadb']['host'],
+        db_config['mariadb']['port'],
+        db_config['mariadb']['database']
+    ))
+    engine = create_engine(connect_string)
+    Base.metadata.create_all(bind=engine, checkfirst=True)
+    __session__ = sessionmaker(bind=engine)
+    _filter = {'should_download': 1}
+    with __session__() as session:
+        files = session.query(MediaFile).filter_by(**_filter).all()
+        log.info("found %d entries", len(files))
+        files2 = session.query(MediaFile).filter(MediaFile.should_download == 1).all()
+        log.info("found %d entries", len(files2))
+        for mediafile in files2:
+            mediafile.download_file(download_dir=args.dir, dl_tool="yt-dlp")
+            log.info("Datei {} erfolgreich heruntergeladen".format(mediafile.file_name))
+            if args.rename:
+                current_file = Path(mediafile.file_name)
+                new_file_path = current_file.with_name(f"{mediafile.id}{current_file.suffix}")
+                current_file.rename(Path(new_file_path))
+                mediafile.cloud_link = new_file_path
+            session.add(mediafile)
+        session.commit()
+    log.info('kontor.download finished')