fix problem in download.py when title has not been set

2025-08-19 08:55:45 +02:00
parent ddc5797549
commit a6a03e3f04
13 changed files with 221 additions and 50 deletions
@@ -21,10 +21,10 @@ class BaseMixin:


 class BaseVideoMixin:
-    cloud_link = Column(String)
-    file_name = Column(String)
+    cloud_link = Column(String, nullable=True)
+    file_name = Column(String, nullable=True)
    path = Column(String)
    review = Column(Boolean)
    title = Column(String)
-    url = Column(String, unique=True)
+    url = Column(String, nullable=True)
    should_download = Column(Boolean)
@@ -3,25 +3,46 @@ download files with URLs from DB
 """
 import logging.config
 import requests
-import yaml
+import re
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
-from pathlib import Path

 from bs4 import BeautifulSoup
-from platformdirs import PlatformDirs

 parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
 parser.add_argument('--verbose', '-v', action='count', default=0)
-parser.add_argument('--config', '-c', default='kontor-docker')
+parser.add_argument('--all', '-a', action='store_true')
 args = parser.parse_args()

-def get_logger(level: int, config: str):
-    dirs = PlatformDirs(config)
-    logging_config = Path(dirs.user_config_dir, 'logging-config.yaml')
-    with open(logging_config, 'rt') as f:
-        configDict = yaml.safe_load(f.read())
-    logging.config.dictConfig(configDict)
-    logger = logging.getLogger('development')
+def get_logger(level: int) -> logging.Logger:
+    logging.config.dictConfig({
+        'version': 1,
+        'disable_existing_loggers': False,
+        'formatters': {
+            'simple': {
+                'format': '[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s',
+                'datefmt': '%Y-%m-%d %H:%M:%S',
+            },
+        },
+        'handlers': {
+            'console': {
+                'class': logging.StreamHandler,
+                'level': logging.DEBUG,
+                'formatter': 'simple',
+                'stream': 'ext://sys.stdout'
+            },
+        },
+        'loggers': {
+            'urllib3.connectionpool': {
+                'level': 'WARNING',
+                'propagate': False,
+            },
+            'root': {
+                'level': 'DEBUG',
+                'handlers': ['console'],
+            },
+        },
+    })
+    logger = logging.getLogger(__file__)
    if level is not None:
        match level:
            case 0:
@@ -32,35 +53,78 @@ def get_logger(level: int, config: str):
                logger.setLevel(logging.CRITICAL)
    return logger

+def update_file(log: logging.Logger, media_file):
+    update = requests.put(f"http://127.0.0.1:8800/api/media/files/{media_file['id']}", json=media_file)
+    log.info(f"update status: {update.status_code}")
+    log.info(f"update result: {update.json()}")
+
+def get_actor_links(log: logging.Logger, media_file_url: str) -> list:
+    try:
+        r = requests.get(media_file_url)
+        soup = BeautifulSoup(r.content, "html.parser")
+        error404 = soup.css.select_one('.error404-title')
+        if error404 and error404.get_text() == "Video nicht gefunden":
+            log.info(f"{error404.get_text()}")
+            item['url'] = None
+            item['review'] = False
+            update_file(log, item)
+            return []
+        anchors = soup.find_all('a', attrs={'href': re.compile("^https://.*pornstars/.*")})
+        actor_links = []
+        for anchor in anchors:
+            link_url = anchor.get('href')
+            if link_url.endswith('all/countries'):
+                continue
+            actor_links.append(link_url)
+        log.info(f"links({len(actor_links)}): {actor_links}")
+        return actor_links
+    except Exception as error:
+        log.info(f"something went wrong: {error}")
+        return []
+

 if __name__ == '__main__':
-    log = get_logger(args.verbose, args.config)
-    log.info('kontor.update_titles started')
-    response = requests.get("http://127.0.0.1:8800/api/media/files?review=true")
+    log = get_logger(args.verbose)
+    log.info('kontor.find_links started')
+    log.info('get all actors')
+    response = requests.get("http://127.0.0.1:8800/api/media/actors")
+    data = response.json()
+    actors = {}
+    for item in data:
+        actor = {}
+        actor['id'] = item['id']
+        actor['name'] = item['name']
+        actor['url'] = item['url']
+        actors[item['url']] = actor
+    log.debug(f'all actors: {actors}')
+    files_url = ""
+    if args.all:
+        files_url= "http://127.0.0.1:8800/api/media/files"
+    else:
+        files_url = "http://127.0.0.1:8800/api/media/files?review=true"
+    response = requests.get(files_url)
    log.info(f"Status: {response.status_code}")
    data = response.json()
    log.info(f"data: {len(data)}")
    for item in data:
        link = item['url']
+        if not link:
+            continue
+        if str(link) == "None":
+            continue
        log.info(f"{item['id']} - {str(link)}")
-        try:
-            r = requests.get(link)
-            soup = BeautifulSoup(r.content, "html.parser")
-            title = soup.title.string
-            anchors = soup.find_all('a')
-            for anchor in anchors:
-                if anchor.has_attr('href'):
-                    link_url = anchor['href']
-                    if link_url and link_url.__contains__('pornstars/'):
-                        log.info(link_url)
-            item['title'] = title
-            item['review'] = False
-        except Exception as error:
-            log.info(f"something went wrong: {error} {anchor}")
-            item['title'] = None
-            item['review'] = True
-        #update = requests.put(f"http://127.0.0.1:8800/api/media/files/{item['id']}", json=item)
-        #log.info(f"update status: {update.status_code}")
-        #log.info(f"update result: {update.json()}")
-    log.info('kontor.update_titles finished')
-
+        actor_links = get_actor_links(log, link)
+        actor_list = []
+        for actor_link in actor_links:
+            if actor_link in actors:
+                log.info(f"found actor with id: {actors[actor_link]['id']}")
+                actor_list.append(actors[actor_link])
+        actor_response = requests.put(f"http://127.0.0.1:8800/api/media/files/{item['id']}/actors", json=actor_list)
+        actor_data = actor_response.json()
+        log.info(f"found {len(actor_data)} actors")
+        log.info(f"found actors: {actor_data}")
+        item['review'] = False
+        update = requests.put(f"http://127.0.0.1:8800/api/media/files/{item['id']}", json=item)
+        log.info(f"update status: {update.status_code}")
+        log.info(f"update result: {update.json()}")
+    log.info('kontor.find_links finished')
@@ -43,9 +43,14 @@ if __name__ == '__main__':
    for item in data:
        link = item['url']
        log.info(f"{item['id']} - {str(link)}")
+        if not link:
+            continue
        try:
            r = requests.get(link)
            soup = BeautifulSoup(r.content, "html.parser")
+            title_tag = soup.find('title')
+            if title_tag:
+                title= title_tag.get_text()
            title = soup.title.string
            item['title'] = title
            item['review'] = False