tpeetz/kontor
1
0
Fork 0
Code Issues 6 Pull Requests 2 Actions Packages Projects Releases 2 Wiki Activity

add authentication for kontor-scripts

Browse Source
This commit is contained in:
Thomas Peetz
2026-01-08 07:05:33 +01:00
parent d63629ba5c
commit 0dbd108051
3 changed files with 118 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kontor-scripts/config.py
+37
View File
@@ -1,11 +1,14 @@
"""
Setup database connections
"""
from logging import Logger
import sqlite3
from typing import Any, Dict
import psycopg2
import logging.config
from platformdirs import PlatformDirs
from pathlib import Path
import requests
import yaml
@@ -54,3 +57,37 @@ def get_logger(level, config: str):
logger.setLevel(logging.INFO)
return logger
def get_api_config(log: Logger, config: str) -> Dict[str, Any]:
api_data: Dict[str, Any] = {}
token: str | None = None
host: str | None = None
port: int = 0
dirs = PlatformDirs(config)
api_config = Path(dirs.user_config_dir, 'api.yaml')
with open(api_config, 'rt') as f:
api_data = yaml.safe_load(f.read())
if not api_data:
log.fatal("API configuration is missing")
return api_data
host = api_data["host"]
port = api_data["login_port"]
if not token:
log.info("Call login first")
login_url = f"http://{host}:{port}/login"
login_data = {}
login_data['user'] = api_data["user"]
login_data['pass'] = api_data["pass"]
response = requests.post(login_url, data=login_data)
status = response.status_code
log.info(f"Status: {status}")
if status != 200:
log.fatal("authentication failed")
return api_data
data = response.json()
token = data['token']
api_data['token'] = token
with open(api_config, 'w') as f:
yaml.dump(api_data, f)
else:
token = api_data['token']
return api_data
kontor-scripts/download.py
+12 -38
View File
@@ -17,7 +17,7 @@ from platformdirs import PlatformDirs
import requests
import yaml
from config import get_logger
from config import get_api_config, get_logger
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument("--verbose", "-v", action="count", default=0)
@@ -106,9 +106,11 @@ def is_file_downloaded(media_file: dict, dir: Path) -> FileStatus:
return FileStatus.UNKNOWN
def update_status(item_id: UUID, file_info: dict):
url: str = f"http://127.0.0.1:8800/api/media/files/{item_id}"
headers: Dict[str, str] = {"Authorization": f"Bearer {bearer_token}"}
def update_status(item_id: UUID, file_info: dict, api_data: Dict[str, Any]):
host = api_data["host"]
token = api_data['token']
url: str = f"http://{host}:{port}/api/media/files/{item_id}"
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
update = requests.put(url, headers=headers, json=file_info)
log.info(f"update status: {update.status_code}")
log.info(f"update result: {update.json()}")
@@ -132,39 +134,11 @@ def rename_file(file_info: dict):
if __name__ == "__main__":
log = get_logger(args.verbose, args.config)
log.info("kontor.download started")
api_data: Dict[str, Any] = {}
token: str | None = None
host: str | None = None
port: int = 0
dirs = PlatformDirs(args.config)
api_config = Path(dirs.user_config_dir, 'api.yaml')
with open(api_config, 'rt') as f:
api_data = yaml.safe_load(f.read())
if not api_data:
log.fatal("API configuration is missing")
sys.exit(1)
api_data = get_api_config(log, args.config)
host = api_data["host"]
port = api_data["port"]
if not token:
log.info("Call login first")
login_url = f"http://{host}:{port}/login"
login_data = {}
login_data['user'] = api_data["user"]
login_data['pass'] = api_data["pass"]
response = requests.post(login_url, data=login_data)
status = response.status_code
log.info(f"Status: {status}")
if status != 200:
log.fatal("authentication failed")
sys.exit(0)
data = response.json()
token = data['token']
api_data['token'] = token
with open(api_config, 'w') as f:
yaml.dump(api_data, f)
else:
token = api_data['token']
url: str = f"http://{host}:8500/api/media/files?download=true"
token = api_data['token']
url: str = f"http://{host}:{port}/api/media/files?download=true"
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
response = requests.get(url, headers=headers)
log.info(f"Status: {response.status_code}")
@@ -186,15 +160,15 @@ if __name__ == "__main__":
match download_status:
case FileStatus.DOWNLOADED:
rename_file(item)
update_status(file_id, item)
update_status(file_id, item, api_data)
case FileStatus.RENAMED:
log.info("update status")
update_status(file_id, item)
update_status(file_id, item, api_data)
case FileStatus.UNKNOWN:
download_file(link, item, args.dir)
rename_file(item)
log.info(f"{item}")
update_status(file_id, item)
update_status(file_id, item, api_data)
log.warning(f"processed {mediafile_index}/{entries_count}")
if args.limit and args.limit <= mediafile_index:
break
kontor-scripts/find_links.py
+69 -26
View File
@@ -3,15 +3,18 @@ download files with URLs from DB
"""
import logging.config
import sys
from typing import Any
from typing import Any, Dict
import requests
import re
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from bs4 import BeautifulSoup
from config import get_api_config
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--verbose', '-v', action='count', default=0)
parser.add_argument("--config", "-c", default="kontor-docker")
parser.add_argument('--all', '-a', action='store_true')
parser.add_argument('--limit', '-l', type=int, help='maximum number of links to check')
parser.add_argument('--add-actor', action='store_true', help='add missing actors')
@@ -60,12 +63,17 @@ def get_logger(level: int) -> logging.Logger:
logger.setLevel(logging.CRITICAL)
return logger
def update_file(log: logging.Logger, media_file):
update = requests.put(f"http://127.0.0.1:8800/api/media/files/{media_file['id']}", json=media_file)
def update_file(log: logging.Logger, media_file, api_data: Dict[str, Any]):
host = api_data["host"]
port = api_data["port"]
token = api_data['token']
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
url = f"http://{host}:{port}/api/media/files/{media_file['id']}"
update = requests.put(url, headers=headers, json=media_file)
log.debug(f"update status: {update.status_code}")
log.debug(f"update result: {update.json()}")
def get_actor_links(log: logging.Logger, media_file_url: str) -> list[str]:
def get_actor_links(log: logging.Logger, media_file_url: str, api_data: Dict[str, Any]) -> list[str]:
try:
r = requests.get(media_file_url)
soup = BeautifulSoup(r.content, "html.parser")
@@ -74,7 +82,7 @@ def get_actor_links(log: logging.Logger, media_file_url: str) -> list[str]:
log.warning(f"{error404.get_text()}")
media_file['url'] = None
media_file['review'] = False
update_file(log, media_file)
update_file(log, media_file, api_data=api_data)
return []
anchors = soup.find_all('a', attrs={'href': re.compile("^https://.*pornstars/.*")})
actor_links = []
@@ -91,19 +99,28 @@ def get_actor_links(log: logging.Logger, media_file_url: str) -> list[str]:
log.warning(f"something went wrong: {error}")
return []
def get_media_files(all_files: bool)-> Any:
def get_media_files(all_files: bool, api_data: Dict[str, Any])-> Any:
files_url = ""
host = api_data["host"]
port = api_data["port"]
token = api_data['token']
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
if all_files:
files_url= "http://127.0.0.1:8800/api/media/files"
files_url= f"http://{host}:{port}/api/media/files"
else:
files_url = "http://127.0.0.1:8800/api/media/files?review=true"
response = requests.get(files_url)
files_url = f"http://{host}:{port}/api/media/files?review=true"
response = requests.get(files_url, headers=headers)
log.debug(f"Status: {response.status_code}")
data = response.json()
return data
def update_media_file(item, log: logging.Logger) -> Any:
update = requests.put(f"http://127.0.0.1:8800/api/media/files/{item['id']}", json=item)
def update_media_file(item, log: logging.Logger, api_data: Dict[str, Any]) -> Any:
host = api_data["host"]
port = api_data["port"]
token = api_data['token']
url: str = f"http://{host}:{port}/api/media/files/{item['id']}"
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
update = requests.put(url, headers=headers, json=item)
log.debug(f"update status: {update.status_code}")
log.debug(f"update result: {update.json()}")
return update.json()
@@ -112,9 +129,15 @@ def update_media_file_actors(mediafile: dict,
actor_id_list: list[dict[str, str]],
actor_links: list[str],
map_ids_actor: dict[str, str],
log: logging.Logger):
log: logging.Logger,
api_data: Dict[str, Any]):
media_file_id = mediafile['id']
actor_response = requests.put(f"http://127.0.0.1:8800/api/media/files/{media_file_id}/actors", json=actor_id_list)
host = api_data["host"]
port = api_data["port"]
token = api_data['token']
url: str = f"http://{host}:{port}/api/media/files/{media_file_id}/actors"
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
actor_response = requests.put(url, headers=headers, json=actor_id_list)
files_actor_list = actor_response.json()
persisted_actor_links_count: int = len(files_actor_list)
found_actor_links_count: int = len(actor_links)
@@ -129,15 +152,20 @@ def update_media_file_actors(mediafile: dict,
log.debug(f"check if actor({actor_id}) with {actor_url} in list")
if actor_url not in actor_links:
log.info(f"actor not found in links, delete relation {file_actor['id']}")
delete_media_file_actor(file_actor['id'], log)
delete_media_file_actor(file_actor['id'], log, api_data)
mediafile['review'] = True
else:
mediafile['review'] = False
log.debug(f"found {persisted_actor_links_count} actors")
log.debug(f"found actors: {files_actor_list}")
def delete_media_file_actor(media_actor_file_id: str, log: logging.Logger):
delete_response = requests.delete(f"http://127.0.0.1:8800/api/media/actorfiles/{media_actor_file_id}")
def delete_media_file_actor(media_actor_file_id: str, log: logging.Logger, api_data: Dict[str, Any]):
host = api_data["host"]
port = api_data["port"]
token = api_data['token']
url: str = f"http://{host}:{port}/api/media/actorfiles/{media_actor_file_id}"
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
delete_response = requests.delete(url, headers=headers)
if delete_response.status_code == 204:
log.info(f"actor file relation with id {media_actor_file_id} successfully deleted")
@@ -205,11 +233,16 @@ def get_persisted_actor(actor_url: str,
log.info(f"found actor {actor_url} missing")
return None
def get_actors(log: logging.Logger):
def get_actors(log: logging.Logger, api_data: Dict[str, Any]):
actors_url = {}
actors_id = {}
actors_path = {}
response = requests.get("http://127.0.0.1:8800/api/media/actors")
host = api_data["host"]
port = api_data["port"]
token = api_data['token']
url: str = f"http://{host}:{port}/api/media/actors"
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
response = requests.get(url, headers=headers)
data = response.json()
for media_actor in data:
actor_id = media_actor['id']
@@ -238,9 +271,14 @@ def get_actor_name(actor_url: str, log: logging.Logger) -> str | None:
log.warning(f"something went wrong: {error}")
return None
def create_actor(actor_url: str, actor_name: str, log: logging.Logger):
def create_actor(actor_url: str, actor_name: str, log: logging.Logger, api_data: Dict[str, Any]):
new_actor = { 'name': actor_name, 'url': actor_url}
actor_response = requests.post(f"http://127.0.0.1:8800/api/media/actors", json=new_actor)
host = api_data["host"]
port = api_data["port"]
token = api_data['token']
url: str = f"http://{host}:{port}/api/media/actors"
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
actor_response = requests.post(url, headers=headers, json=new_actor)
log.warning(f"add status: {actor_response.status_code}")
if actor_response.status_code == 201:
actor_data = actor_response.json()
@@ -253,8 +291,13 @@ if __name__ == '__main__':
log = get_logger(args.verbose)
log.warning('kontor.find_links started')
log.debug('get all actors')
(actors_url, actors_id, actors_path) = get_actors(log)
data = get_media_files(args.all)
api_data = get_api_config(log, args.config)
host = api_data["host"]
port = api_data["port"]
token = api_data['token']
(actors_url, actors_id, actors_path) = get_actors(log, api_data=api_data)
data = get_media_files(args.all, api_data)
entries_count = len(data)
mediafile_index = 1
log.debug(f"data: {len(data)}")
@@ -269,10 +312,10 @@ if __name__ == '__main__':
if str(link) == "None":
continue
log.warning(f"{media_file['id']} - {str(link)}")
actor_links: list[str] = get_actor_links(log, link)
actor_links: list[str] = get_actor_links(log, link, api_data=api_data)
actor_id_list = get_actor_ids(actor_links, actors_url, actors_id, actors_path, missing_actors, log)
update_media_file_actors(media_file, actor_id_list, actor_links, actors_id, log)
result = update_media_file(media_file, log)
update_media_file_actors(media_file, actor_id_list, actor_links, actors_id, log, api_data=api_data)
result = update_media_file(media_file, log, api_data=api_data)
log.warning(f"processed {mediafile_index}/{entries_count}")
if args.limit and args.limit <= mediafile_index:
break
@@ -281,7 +324,7 @@ if __name__ == '__main__':
log.info(f"{link}: {missing_actors[link]}")
actor_name = get_actor_name(link, log)
if actor_name and args.add_actor:
create_actor(link, actor_name, log)
create_actor(link, actor_name, log, api_data=api_data)
log.info("Sort missing actors by occurence count:")
sorted_missing = dict(sorted(missing_actors.items(), key=lambda item: item[1]))
for key in sorted_missing: