Vorbereitung Release 0.2.0
This commit is contained in:
@@ -0,0 +1,369 @@
|
||||
"""
|
||||
add actors
|
||||
"""
|
||||
import logging.config
|
||||
import requests
|
||||
import re
|
||||
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument('--verbose', '-v', action='count', default=0)
|
||||
args = parser.parse_args()
|
||||
|
||||
def get_logger(level: int) -> logging.Logger:
|
||||
logging.config.dictConfig({
|
||||
'version': 1,
|
||||
'disable_existing_loggers': False,
|
||||
'formatters': {
|
||||
'simple': {
|
||||
'format': '[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s',
|
||||
'datefmt': '%Y-%m-%d %H:%M:%S',
|
||||
},
|
||||
},
|
||||
'handlers': {
|
||||
'console': {
|
||||
'class': logging.StreamHandler,
|
||||
'level': logging.DEBUG,
|
||||
'formatter': 'simple',
|
||||
'stream': 'ext://sys.stdout'
|
||||
},
|
||||
},
|
||||
'loggers': {
|
||||
'urllib3.connectionpool': {
|
||||
'level': 'WARNING',
|
||||
'propagate': False,
|
||||
},
|
||||
'root': {
|
||||
'level': 'DEBUG',
|
||||
'handlers': ['console'],
|
||||
},
|
||||
},
|
||||
})
|
||||
logger = logging.getLogger(__file__)
|
||||
if level is not None:
|
||||
match level:
|
||||
case 0:
|
||||
logger.setLevel(logging.WARNING)
|
||||
case 1:
|
||||
logger.setLevel(logging.INFO)
|
||||
case 2:
|
||||
logger.setLevel(logging.DEBUG)
|
||||
case _:
|
||||
logger.setLevel(logging.CRITICAL)
|
||||
return logger
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
log = get_logger(args.verbose)
|
||||
log.warning('kontor.add_actors started')
|
||||
log.debug('get all actors')
|
||||
response = requests.get("http://127.0.0.1:8800/api/media/actors")
|
||||
data = response.json()
|
||||
actors = {}
|
||||
for item in data:
|
||||
actor = {}
|
||||
actor['id'] = item['id']
|
||||
actor['name'] = item['name']
|
||||
actor['url'] = item['url']
|
||||
actors[item['url']] = actor
|
||||
log.debug(f'all actors: {actors}')
|
||||
|
||||
new_actor_list = [
|
||||
{ 'name': 'Herschel Savage', 'url': 'https://ge.xhamster.com/pornstars/herschel-savage'},
|
||||
{ 'name': 'Janey Robbins', 'url': 'https://ge.xhamster.com/pornstars/janey-robbins'},
|
||||
{ 'name': 'Kimberly Carson', 'url': 'https://ge.xhamster.com/pornstars/kimberly-carson'},
|
||||
{ 'name': 'Paul Thomas', 'url': 'https://ge.xhamster.com/pornstars/paul-thomas'},
|
||||
{ 'name': 'Shauna Grant', 'url': 'https://ge.xhamster.com/pornstars/shauna-grant'},
|
||||
{ 'name': 'Tara Aire', 'url': 'https://ge.xhamster.com/pornstars/tara-aire'},
|
||||
{ 'name': 'Kari Milla', 'url': 'https://ge.xhamster.com/pornstars/kari-milla'},
|
||||
{ 'name': 'Drago Gvozdik', 'url': 'https://ge.xhamster.com/pornstars/drago-gvozdik'},
|
||||
{ 'name': 'Cheyne Collins', 'url': 'https://ge.xhamster.com/pornstars/cheyne-collins'},
|
||||
{ 'name': 'Christian XXX', 'url': 'https://ge.xhamster.com/pornstars/christian-xxx'},
|
||||
{ 'name': 'Derrick Pierce', 'url': 'https://ge.xhamster.com/pornstars/derrick-pierce'},
|
||||
{ 'name': 'Holly Halston', 'url': 'https://ge.xhamster.com/pornstars/holly-halston'},
|
||||
{ 'name': 'Holly West', 'url': 'https://ge.xhamster.com/pornstars/holly-west'},
|
||||
{ 'name': 'Jarod Diamond', 'url': 'https://ge.xhamster.com/pornstars/jarod-diamond'},
|
||||
{ 'name': 'Kristina Cross', 'url': 'https://ge.xhamster.com/pornstars/kristina-cross'},
|
||||
{ 'name': 'Lee Stone', 'url': 'https://ge.xhamster.com/pornstars/lee-stone'},
|
||||
{ 'name': 'Monica Mayhem', 'url': 'https://ge.xhamster.com/pornstars/monica-mayhem'},
|
||||
{ 'name': 'Sienna West', 'url': 'https://ge.xhamster.com/pornstars/sienna-west'},
|
||||
{ 'name': 'Aria Carson', 'url': 'https://ge.xhamster.com/pornstars/aria-carson'},
|
||||
{ 'name': 'Britney Amber', 'url': 'https://ge.xhamster.com/pornstars/britney-amber'},
|
||||
{ 'name': 'Kit Mercer', 'url': 'https://ge.xhamster.com/pornstars/kit-mercer'},
|
||||
{ 'name': 'Riley Reyes', 'url': 'https://ge.xhamster.com/pornstars/riley-reyes'},
|
||||
{ 'name': 'Ryan Keely', 'url': 'https://ge.xhamster.com/pornstars/ryan-keely'},
|
||||
{ 'name': 'Alexandra Diamond', 'url': 'https://ge.xhamster.com/pornstars/alexandra-diamond'},
|
||||
{ 'name': 'Amanda Twice', 'url': 'https://ge.xhamster.com/pornstars/amanda-twice'},
|
||||
{ 'name': 'Choky Ice', 'url': 'https://ge.xhamster.com/pornstars/choky-ice'},
|
||||
{ 'name': 'Cindy Cox', 'url': 'https://ge.xhamster.com/pornstars/cindy-cox'},
|
||||
{ 'name': 'Kelly White', 'url': 'https://ge.xhamster.com/pornstars/kelly-white'},
|
||||
{ 'name': 'Mike Foster', 'url': 'https://ge.xhamster.com/pornstars/mike-foster'},
|
||||
{ 'name': 'Susie Sorrento', 'url': 'https://ge.xhamster.com/pornstars/susie-sorrento'},
|
||||
{ 'name': 'Lara Sanchez', 'url': 'https://ge.xhamster.com/pornstars/lara-sanchez'},
|
||||
{ 'name': 'Alicia Williams', 'url': 'https://ge.xhamster.com/pornstars/alicia-williams'},
|
||||
{ 'name': 'Dirty Tina', 'url': 'https://ge.xhamster.com/pornstars/dirty-tina'},
|
||||
{ 'name': 'Andrew Andretti', 'url': 'https://ge.xhamster.com/pornstars/andrew-andretti'},
|
||||
{ 'name': 'Dick Nasty', 'url': 'https://ge.xhamster.com/pornstars/dick-nasty'},
|
||||
{ 'name': 'Vicky Vette', 'url': 'https://ge.xhamster.com/pornstars/vicky-vette'},
|
||||
{ 'name': 'Alex Gonz', 'url': 'https://ge.xhamster.com/pornstars/alex-gonz'},
|
||||
{ 'name': 'Audrey Hollander', 'url': 'https://ge.xhamster.com/pornstars/audrey-hollander'},
|
||||
{ 'name': 'Desiree Diamond', 'url': 'https://ge.xhamster.com/pornstars/desiree-diamond'},
|
||||
{ 'name': 'Lexi Bardot', 'url': 'https://ge.xhamster.com/pornstars/lexi-bardot'},
|
||||
{ 'name': 'Adam Ocelot', 'url': 'https://ge.xhamster.com/pornstars/adam-ocelot'},
|
||||
{ 'name': 'Scarlett Hampton', 'url': 'https://ge.xhamster.com/pornstars/scarlett-hampton'},
|
||||
{ 'name': 'Laetitia', 'url': 'https://ge.xhamster.com/pornstars/laetitia'},
|
||||
{ 'name': 'Raffaela Anderson', 'url': 'https://ge.xhamster.com/pornstars/raffaela-anderson'},
|
||||
{ 'name': 'Michael Swayze', 'url': 'https://ge.xhamster.com/pornstars/michael-swayze'},
|
||||
{ 'name': 'Winter Jade', 'url': 'https://ge.xhamster.com/pornstars/winter-jade'},
|
||||
{ 'name': 'Celine Noiret', 'url': 'https://ge.xhamster.com/pornstars/celine-noiret'},
|
||||
{ 'name': 'Aurora Snow', 'url': 'https://ge.xhamster.com/pornstars/aurora-snow'},
|
||||
{ 'name': 'Bree Brooks', 'url': 'https://ge.xhamster.com/pornstars/bree-brooks'},
|
||||
{ 'name': 'Flick Shagwell', 'url': 'https://ge.xhamster.com/pornstars/flick-shagwell'},
|
||||
{ 'name': 'Noname Jane', 'url': 'https://ge.xhamster.com/pornstars/noname-jane'},
|
||||
{ 'name': 'Alex Magni', 'url': 'https://ge.xhamster.com/pornstars/alex-magni'},
|
||||
{ 'name': 'Maeva Dream', 'url': 'https://ge.xhamster.com/pornstars/maeva-dream'},
|
||||
{ 'name': 'Squirty Alice', 'url': 'https://ge.xhamster.com/pornstars/squirty-alice'},
|
||||
{ 'name': 'Mandy Rhea', 'url': 'https://ge.xhamster.com/pornstars/mandy-rhea'},
|
||||
{ 'name': 'Alessia Donati', 'url': 'https://ge.xhamster.com/pornstars/alessia-donati'},
|
||||
{ 'name': 'Pierre DJ', 'url': 'https://ge.xhamster.com/pornstars/pierre-dj'},
|
||||
{ 'name': 'Veronica Belli', 'url': 'https://ge.xhamster.com/pornstars/veronica-belli'},
|
||||
{ 'name': 'Christiana Cinn', 'url': 'https://ge.xhamster.com/pornstars/christiana-cinn'},
|
||||
{ 'name': 'Jasmine Jae', 'url': 'https://ge.xhamster.com/pornstars/jasmine-jae'},
|
||||
{ 'name': 'Jay Smooth', 'url': 'https://ge.xhamster.com/pornstars/jay-smooth'},
|
||||
{ 'name': 'Natalie Knight', 'url': 'https://ge.xhamster.com/pornstars/natalie-knight'},
|
||||
{ 'name': 'Joshua Lewis', 'url': 'https://ge.xhamster.com/pornstars/joshua-lewis'},
|
||||
{ 'name': 'Lauren Phillips', 'url': 'https://ge.xhamster.com/pornstars/lauren-phillips'},
|
||||
{ 'name': 'Matt Cash', 'url': 'https://ge.xhamster.com/pornstars/matt-cash'},
|
||||
{ 'name': 'Nickey Huntsman', 'url': 'https://ge.xhamster.com/pornstars/nickey-huntsman'},
|
||||
{ 'name': 'Jade Sin', 'url': 'https://ge.xhamster.com/pornstars/jade-sin'},
|
||||
{ 'name': 'Wein Lewis', 'url': 'https://ge.xhamster.com/pornstars/wein-lewis'},
|
||||
{ 'name': 'Megan Murkovski', 'url': 'https://ge.xhamster.com/pornstars/megan-murkovski'},
|
||||
{ 'name': 'Tara Wild', 'url': 'https://ge.xhamster.com/pornstars/tara-wild'},
|
||||
{ 'name': 'Lana Roy', 'url': 'https://ge.xhamster.com/pornstars/lana-roy'},
|
||||
{ 'name': 'Nick Moreno', 'url': 'https://ge.xhamster.com/pornstars/nick-moreno'},
|
||||
{ 'name': 'Brad Armstrong', 'url': 'https://ge.xhamster.com/pornstars/brad-armstrong'},
|
||||
{ 'name': 'Kaylani Lei', 'url': 'https://ge.xhamster.com/pornstars/kaylani-lei'},
|
||||
{ 'name': 'Deborah Wells', 'url': 'https://ge.xhamster.com/pornstars/deborah-wells'},
|
||||
{ 'name': 'Judith Ramirez', 'url': 'https://ge.xhamster.com/pornstars/judith-ramirez'},
|
||||
{ 'name': 'Richard Langin', 'url': 'https://ge.xhamster.com/pornstars/richard-langin'},
|
||||
{ 'name': 'Simona Valli', 'url': 'https://ge.xhamster.com/pornstars/simona-valli'},
|
||||
{ 'name': 'Mandy Dee', 'url': 'https://ge.xhamster.com/pornstars/mandy-dee'},
|
||||
{ 'name': 'Allison Wyte', 'url': 'https://ge.xhamster.com/pornstars/allison-wyte'},
|
||||
{ 'name': 'Ben English', 'url': 'https://ge.xhamster.com/pornstars/ben-english'},
|
||||
{ 'name': 'Chuck Martino', 'url': 'https://ge.xhamster.com/pornstars/chuck-martino'},
|
||||
{ 'name': 'Erika Kole', 'url': 'https://ge.xhamster.com/pornstars/erika-kole'},
|
||||
{ 'name': 'Katie Morgan', 'url': 'https://ge.xhamster.com/pornstars/katie-morgan'},
|
||||
{ 'name': 'Misty Parks', 'url': 'https://ge.xhamster.com/pornstars/misty-parks'},
|
||||
{ 'name': 'Mr. Pete', 'url': 'https://ge.xhamster.com/pornstars/mr-pete'},
|
||||
{ 'name': 'Candie Luciani', 'url': 'https://ge.xhamster.com/pornstars/candie-luciani'},
|
||||
{ 'name': 'Clara Mia', 'url': 'https://ge.xhamster.com/pornstars/clara-mia'},
|
||||
{ 'name': 'Tommy Cabrio', 'url': 'https://ge.xhamster.com/pornstars/tommy-cabrio'},
|
||||
{ 'name': 'Nathan Bronson', 'url': 'https://ge.xhamster.com/pornstars/nathan-bronson'},
|
||||
{ 'name': 'Octavia Red', 'url': 'https://ge.xhamster.com/pornstars/octavia-red'},
|
||||
{ 'name': 'Big George', 'url': 'https://ge.xhamster.com/pornstars/big-george'},
|
||||
{ 'name': 'Cassy Young', 'url': 'https://ge.xhamster.com/pornstars/cassy-young'},
|
||||
{ 'name': 'Egon Kowalski', 'url': 'https://ge.xhamster.com/pornstars/egon-kowalski'},
|
||||
{ 'name': 'Mark Aurel', 'url': 'https://ge.xhamster.com/pornstars/mark-aurel'},
|
||||
{ 'name': 'Samy Fox', 'url': 'https://ge.xhamster.com/pornstars/samy-fox'},
|
||||
{ 'name': 'Valeria Jones', 'url': 'https://ge.xhamster.com/pornstars/valeria-jones'},
|
||||
{ 'name': 'Dieter Von Stein', 'url': 'https://ge.xhamster.com/pornstars/dieter-von-stein'},
|
||||
{ 'name': 'Donna Bell', 'url': 'https://ge.xhamster.com/pornstars/donna-bell'},
|
||||
{ 'name': 'Molly Jane', 'url': 'https://ge.xhamster.com/pornstars/molly-jane'},
|
||||
{ 'name': 'Mark Zane', 'url': 'https://ge.xhamster.com/pornstars/mark-zane'},
|
||||
{ 'name': 'Titus Steel', 'url': 'https://ge.xhamster.com/pornstars/titus-steel'},
|
||||
{ 'name': 'Mackenzie Mace', 'url': 'https://ge.xhamster.com/pornstars/mackenzie-mace'},
|
||||
{ 'name': 'Christina Lang', 'url': 'https://ge.xhamster.com/pornstars/christina-lang'},
|
||||
{ 'name': 'Jens Modena', 'url': 'https://ge.xhamster.com/pornstars/jens-modena'},
|
||||
{ 'name': 'Karma Rosenberg', 'url': 'https://ge.xhamster.com/pornstars/karma-rosenberg'},
|
||||
{ 'name': 'Dani Jensen', 'url': 'https://ge.xhamster.com/pornstars/dani-jensen'},
|
||||
{ 'name': 'Crystal Ray', 'url': 'https://ge.xhamster.com/pornstars/crystal-ray'},
|
||||
{ 'name': 'Andrea Szabo', 'url': 'https://ge.xhamster.com/pornstars/andrea-szabo'},
|
||||
{ 'name': 'Desiree Barclay', 'url': 'https://ge.xhamster.com/pornstars/desiree-barclay'},
|
||||
{ 'name': 'Jada Stevens', 'url': 'https://ge.xhamster.com/pornstars/jada-stevens'},
|
||||
{ 'name': 'Syren Demer', 'url': 'https://ge.xhamster.com/pornstars/syren-demer'},
|
||||
{ 'name': 'Ara Mix', 'url': 'https://ge.xhamster.com/pornstars/ara-mix'},
|
||||
{ 'name': 'Redneck John', 'url': 'https://ge.xhamster.com/pornstars/redneck-john'},
|
||||
{ 'name': 'Allysin Chaynes', 'url': 'https://ge.xhamster.com/pornstars/allysin-chaynes'},
|
||||
{ 'name': 'Aspen Brock', 'url': 'https://ge.xhamster.com/pornstars/aspen-brock'},
|
||||
{ 'name': 'Kaylynn', 'url': 'https://ge.xhamster.com/pornstars/kaylynn'},
|
||||
{ 'name': 'Monica Cameron', 'url': 'https://ge.xhamster.com/pornstars/monica-cameron'},
|
||||
{ 'name': 'Sheila Rossi', 'url': 'https://ge.xhamster.com/pornstars/sheila-rossi'},
|
||||
{ 'name': 'Alex Ginger', 'url': 'https://ge.xhamster.com/pornstars/alex-ginger'},
|
||||
{ 'name': 'Ally Style', 'url': 'https://ge.xhamster.com/pornstars/ally-style'},
|
||||
{ 'name': 'Paola Mike', 'url': 'https://ge.xhamster.com/pornstars/paola-mike'},
|
||||
{ 'name': 'Jeanette Littledove', 'url': 'https://ge.xhamster.com/pornstars/jeanette-littledove'},
|
||||
{ 'name': 'Melissa Melendez', 'url': 'https://ge.xhamster.com/pornstars/melissa-melendez'},
|
||||
{ 'name': 'Peter North', 'url': 'https://ge.xhamster.com/pornstars/peter-north'},
|
||||
{ 'name': 'Siobhan Hunter', 'url': 'https://ge.xhamster.com/pornstars/siobhan-hunter'},
|
||||
{ 'name': 'Tami White', 'url': 'https://ge.xhamster.com/pornstars/tami-white'},
|
||||
{ 'name': 'Tracey Adams', 'url': 'https://ge.xhamster.com/pornstars/tracey-adams'},
|
||||
{ 'name': 'Ashley Haze', 'url': 'https://ge.xhamster.com/pornstars/ashley-haze'},
|
||||
{ 'name': 'Cailey Taylor', 'url': 'https://ge.xhamster.com/pornstars/cailey-taylor'},
|
||||
{ 'name': 'Eve Laurence', 'url': 'https://ge.xhamster.com/pornstars/eve-laurence'},
|
||||
{ 'name': 'Naudia Nyce', 'url': 'https://ge.xhamster.com/pornstars/naudia-nyce'},
|
||||
{ 'name': 'Candy Apples', 'url': 'https://ge.xhamster.com/pornstars/candy-apples'},
|
||||
{ 'name': 'Kate Rich', 'url': 'https://ge.xhamster.com/pornstars/kate-rich'},
|
||||
{ 'name': 'Aniko Kaposi', 'url': 'https://ge.xhamster.com/pornstars/aniko-kaposi'},
|
||||
{ 'name': 'Beatrice Poggi', 'url': 'https://ge.xhamster.com/pornstars/beatrice-poggi'},
|
||||
{ 'name': 'Erika Bella', 'url': 'https://ge.xhamster.com/pornstars/erika-bella'},
|
||||
{ 'name': 'Nikita Gross', 'url': 'https://ge.xhamster.com/pornstars/nikita-gross'},
|
||||
{ 'name': 'Ursula Moore', 'url': 'https://ge.xhamster.com/pornstars/ursula-moore'},
|
||||
{ 'name': 'Caty Kiss', 'url': 'https://ge.xhamster.com/pornstars/caty-kiss'},
|
||||
{ 'name': 'Light Fairy', 'url': 'https://ge.xhamster.com/pornstars/light-fairy'},
|
||||
{ 'name': 'Flame', 'url': 'https://ge.xhamster.com/pornstars/flame'},
|
||||
{ 'name': 'Tiffany Tatum', 'url': 'https://ge.xhamster.com/pornstars/tiffany-tatum'},
|
||||
{ 'name': 'Alyson Sykes', 'url': 'https://ge.xhamster.com/pornstars/alyson-sykes'},
|
||||
{ 'name': 'Jenifer Stone', 'url': 'https://ge.xhamster.com/pornstars/jenifer-stone'},
|
||||
{ 'name': 'Lucy Love', 'url': 'https://ge.xhamster.com/pornstars/lucy-love'},
|
||||
{ 'name': 'Thomas Stone', 'url': 'https://ge.xhamster.com/pornstars/thomas-stone'},
|
||||
{ 'name': 'Nikki Babe', 'url': 'https://ge.xhamster.com/pornstars/nikki-babe'},
|
||||
{ 'name': 'Tyra Misoux', 'url': 'https://ge.xhamster.com/pornstars/tyra-misoux'},
|
||||
{ 'name': 'Kenzie Reeves', 'url': 'https://ge.xhamster.com/pornstars/kenzie-reeves'},
|
||||
{ 'name': 'Anissa Kate', 'url': 'https://ge.xhamster.com/pornstars/anissa-kate'},
|
||||
{ 'name': 'Anna Polina', 'url': 'https://ge.xhamster.com/pornstars/anna-polina'},
|
||||
{ 'name': 'Kimber Delice', 'url': 'https://ge.xhamster.com/pornstars/kimber-delice'},
|
||||
{ 'name': 'Lucy Heart', 'url': 'https://ge.xhamster.com/pornstars/lucy-heart'},
|
||||
{ 'name': 'John Strong', 'url': 'https://ge.xhamster.com/pornstars/john-strong'},
|
||||
{ 'name': 'Markus Dupree', 'url': 'https://ge.xhamster.com/pornstars/markus-dupree'},
|
||||
{ 'name': 'Mick Blue', 'url': 'https://ge.xhamster.com/pornstars/mick-blue'},
|
||||
{ 'name': 'Natasha Nice', 'url': 'https://ge.xhamster.com/pornstars/natasha-nice'},
|
||||
{ 'name': 'Lena Reif', 'url': 'https://ge.xhamster.com/pornstars/lena-reif'},
|
||||
{ 'name': 'Sonia Paganini', 'url': 'https://ge.xhamster.com/pornstars/sonia-paganini'},
|
||||
{ 'name': 'Demi Hawks', 'url': 'https://ge.xhamster.com/pornstars/demi-hawks'},
|
||||
{ 'name': 'Juan El Caballo Loco', 'url': 'https://ge.xhamster.com/pornstars/juan-el-caballo-loco'},
|
||||
{ 'name': 'Mike Mancini', 'url': 'https://ge.xhamster.com/pornstars/mike-mancini'},
|
||||
{ 'name': 'Millie Morgan', 'url': 'https://ge.xhamster.com/pornstars/millie-morgan'},
|
||||
{ 'name': 'Richard Allan', 'url': 'https://ge.xhamster.com/pornstars/richard-allan'},
|
||||
{ 'name': 'Damon Dice', 'url': 'https://ge.xhamster.com/pornstars/damon-dice'},
|
||||
{ 'name': 'Sera Ryder', 'url': 'https://ge.xhamster.com/pornstars/sera-ryder'},
|
||||
{ 'name': 'Zoltan Kaabai', 'url': 'https://ge.xhamster.com/pornstars/zoltan-kabai'},
|
||||
{ 'name': 'Cathy Heaven', 'url': 'https://ge.xhamster.com/pornstars/cathy-heaven'},
|
||||
{ 'name': 'Coco Lovelock', 'url': 'https://ge.xhamster.com/pornstars/coco-lovelock'},
|
||||
{ 'name': 'Percy Sires', 'url': 'https://ge.xhamster.com/pornstars/percy-sires'},
|
||||
{ 'name': 'Meridian', 'url': 'https://ge.xhamster.com/pornstars/meridian'},
|
||||
{ 'name': 'Pascal St. James', 'url': 'https://ge.xhamster.com/pornstars/pascal-st-james'},
|
||||
{ 'name': 'Red Fox', 'url': 'https://ge.xhamster.com/pornstars/red-fox'},
|
||||
{ 'name': 'Tony Art', 'url': 'https://ge.xhamster.com/pornstars/tony-art'},
|
||||
{ 'name': 'Addison Lee', 'url': 'https://ge.xhamster.com/pornstars/addison-lee'},
|
||||
{ 'name': 'Daria Glover', 'url': 'https://ge.xhamster.com/pornstars/daria-glover'},
|
||||
{ 'name': 'Mandy Bright', 'url': 'https://ge.xhamster.com/pornstars/mandy-bright'},
|
||||
{ 'name': 'Antonia Sainz', 'url': 'https://ge.xhamster.com/pornstars/antonia-sainz'},
|
||||
{ 'name': 'Nicole Love', 'url': 'https://ge.xhamster.com/pornstars/nicole-love'},
|
||||
{ 'name': 'Sarah Kay', 'url': 'https://ge.xhamster.com/pornstars/sarah-kay'},
|
||||
{ 'name': 'Judith Kostner', 'url': 'https://ge.xhamster.com/pornstars/judith-kostner'},
|
||||
{ 'name': 'Maria Bellucci', 'url': 'https://ge.xhamster.com/pornstars/maria-bellucci'},
|
||||
{ 'name': 'Melissa Monet', 'url': 'https://ge.xhamster.com/pornstars/melissa-monet'},
|
||||
{ 'name': 'Stephanie Cane', 'url': 'https://ge.xhamster.com/pornstars/stephanie-cane'},
|
||||
{ 'name': 'Will Steiger', 'url': 'https://ge.xhamster.com/pornstars/will-steiger'},
|
||||
{ 'name': 'Katty West', 'url': 'https://ge.xhamster.com/pornstars/katty-west'},
|
||||
{ 'name': 'Jean Pallett', 'url': 'https://ge.xhamster.com/pornstars/jean-pallett'},
|
||||
{ 'name': 'Conny Dachs', 'url': 'https://ge.xhamster.com/pornstars/conny-dachs'},
|
||||
{ 'name': 'Juicy Leyla', 'url': 'https://ge.xhamster.com/pornstars/juicy-leyla'},
|
||||
{ 'name': 'Mandy Mystery', 'url': 'https://ge.xhamster.com/pornstars/mandy-mystery'},
|
||||
{ 'name': 'Jack Vegas', 'url': 'https://ge.xhamster.com/pornstars/jack-vegas'},
|
||||
{ 'name': 'Jessica Ryan', 'url': 'https://ge.xhamster.com/pornstars/jessica-ryan'},
|
||||
{ 'name': 'Nathan Bronson', 'url': 'https://ge.xhamster.com/pornstars/nathan-bronson'},
|
||||
{ 'name': 'Lexi Lore', 'url': 'https://ge.xhamster.com/pornstars/lexi-lore'},
|
||||
{ 'name': 'Molly Little', 'url': 'https://ge.xhamster.com/pornstars/molly-little'},
|
||||
{ 'name': 'Dolly Leigh', 'url': 'https://ge.xhamster.com/pornstars/dolly-leigh'},
|
||||
{ 'name': 'Marcus London', 'url': 'https://ge.xhamster.com/pornstars/marcus-london'},
|
||||
{ 'name': 'Kendra Sunderland', 'url': 'https://ge.xhamster.com/pornstars/kendra-sunderland'},
|
||||
{ 'name': 'Manuel Ferrara', 'url': 'https://ge.xhamster.com/pornstars/manuel-ferrara'},
|
||||
{ 'name': 'Kelly Trump', 'url': 'https://ge.xhamster.com/pornstars/kelly-trump'},
|
||||
{ 'name': 'Angelica Heaven', 'url': 'https://ge.xhamster.com/pornstars/angelica-heaven'},
|
||||
{ 'name': 'Luna Lynx', 'url': 'https://ge.xhamster.com/pornstars/luna-lynx'},
|
||||
{ 'name': 'Princess Lili', 'url': 'https://ge.xhamster.com/pornstars/princess-lili'},
|
||||
{ 'name': 'Alexa Wild', 'url': 'https://ge.xhamster.com/pornstars/alexa-wild'},
|
||||
{ 'name': 'Jessyka Swan', 'url': 'https://ge.xhamster.com/pornstars/jessyka-swan'},
|
||||
{ 'name': 'Nikky Thorne', 'url': 'https://ge.xhamster.com/pornstars/nikky-thorne'},
|
||||
{ 'name': 'Tristan Summers', 'url': 'https://ge.xhamster.com/pornstars/tristan-summers'},
|
||||
{ 'name': 'Rhaya Shyne', 'url': 'https://ge.xhamster.com/pornstars/rhaya-shyne'},
|
||||
{ 'name': 'Desiree West', 'url': 'https://ge.xhamster.com/pornstars/desiree-west'},
|
||||
{ 'name': 'Joan Devlon', 'url': 'https://ge.xhamster.com/pornstars/joan-devlon'},
|
||||
{ 'name': 'Jodi Thorpe', 'url': 'https://ge.xhamster.com/pornstars/jodi-thorpe'},
|
||||
{ 'name': 'Laurien Dominique', 'url': 'https://ge.xhamster.com/pornstars/laurien-dominique'},
|
||||
{ 'name': 'Paul Scharf', 'url': 'https://ge.xhamster.com/pornstars/paul-scharf'},
|
||||
{ 'name': 'Ray Wells', 'url': 'https://ge.xhamster.com/pornstars/ray-wells'},
|
||||
{ 'name': 'Sandy Carey', 'url': 'https://ge.xhamster.com/pornstars/sandy-carey'},
|
||||
{ 'name': 'Spender Travis', 'url': 'https://ge.xhamster.com/pornstars/spender-travis'},
|
||||
{ 'name': 'Starlyn Simone', 'url': 'https://ge.xhamster.com/pornstars/starlyn-simone'},
|
||||
{ 'name': 'Uschi Digard', 'url': 'https://ge.xhamster.com/pornstars/uschi-digard'},
|
||||
{ 'name': 'Katarina Muti', 'url': 'https://ge.xhamster.com/pornstars/katarina-muti'},
|
||||
{ 'name': 'Julia Power', 'url': 'https://ge.xhamster.com/pornstars/julia-power'},
|
||||
{ 'name': 'Salma De Nora', 'url': 'https://ge.xhamster.com/pornstars/salma-de-nora'},
|
||||
{ 'name': 'Valeria Jones', 'url': 'https://ge.xhamster.com/pornstars/valeria-jones'},
|
||||
{ 'name': 'Brandy Canyon', 'url': 'https://ge.xhamster.com/pornstars/brandy-canyon'},
|
||||
{ 'name': 'Marie Berger', 'url': 'https://ge.xhamster.com/pornstars/marie-berger'},
|
||||
{ 'name': 'Luna Rishi', 'url': 'https://ge.xhamster.com/pornstars/luna-rishi'},
|
||||
{ 'name': 'Colleen Brennan', 'url': 'https://ge.xhamster.com/pornstars/colleen-brennan'},
|
||||
{ 'name': 'Roxanne Brewer', 'url': 'https://ge.xhamster.com/pornstars/roxanne-brewer'},
|
||||
{ 'name': 'Elle Denay', 'url': 'https://ge.xhamster.com/pornstars/elle-denay'},
|
||||
{ 'name': 'Juan Largo', 'url': 'https://ge.xhamster.com/pornstars/juan-largo'},
|
||||
{ 'name': 'Codey Steele', 'url': 'https://ge.xhamster.com/pornstars/codey-steele'},
|
||||
{ 'name': 'Laney Grey', 'url': 'https://ge.xhamster.com/pornstars/laney-grey'},
|
||||
{ 'name': 'Stirling Cooper', 'url': 'https://ge.xhamster.com/pornstars/stirling-cooper'},
|
||||
{ 'name': 'Lana Smalls', 'url': 'https://ge.xhamster.com/pornstars/lana-smalls'},
|
||||
{ 'name': 'Alex Sanders', 'url':'https://ge.xhamster.com/pornstars/alex-sanders'},
|
||||
{ 'name': 'Felecia Danay', 'url':'https://ge.xhamster.com/pornstars/felecia-danay'},
|
||||
{ 'name': 'Lita Chase', 'url':'https://ge.xhamster.com/pornstars/lita-chase'},
|
||||
{ 'name': 'Mark Ashley', 'url':'https://ge.xhamster.com/pornstars/mark-ashley'},
|
||||
{ 'name': 'Phyllisha Anne', 'url':'https://ge.xhamster.com/pornstars/phyllisha-anne'},
|
||||
{ 'name': 'Ryan Conner', 'url':'https://ge.xhamster.com/pornstars/ryan-conner'},
|
||||
{ 'name': 'Tanya Danielle', 'url':'https://ge.xhamster.com/pornstars/tanya-danielle'},
|
||||
{ 'name': 'Jessica Moore', 'url':'https://ge.xhamster.com/pornstars/jessica-moore'},
|
||||
{ 'name': 'Mike Angelo', 'url':'https://ge.xhamster.com/pornstars/mike-angelo'},
|
||||
{ 'name': 'Morgan Moon', 'url':'https://ge.xhamster.com/pornstars/morgan-moon'},
|
||||
{ 'name': 'Tyler Steel', 'url':'https://ge.xhamster.com/pornstars/tyler-steel'},
|
||||
{ 'name': 'Abella Danger', 'url':'https://ge.xhamster.com/pornstars/abella-danger'},
|
||||
{ 'name': 'Alex Jett', 'url':'https://ge.xhamster.com/pornstars/alex-jett'},
|
||||
{ 'name': 'Alyson Queen', 'url':'https://ge.xhamster.com/pornstars/alyson-queen'},
|
||||
{ 'name': 'Antynia Rouge', 'url':'https://ge.xhamster.com/pornstars/antynia-rouge'},
|
||||
{ 'name': 'Bea Dumas', 'url':'https://ge.xhamster.com/pornstars/bea-dumas'},
|
||||
{ 'name': 'Callie Black', 'url':'https://ge.xhamster.com/pornstars/callie-black'},
|
||||
{ 'name': 'Caroline Cage', 'url':'https://ge.xhamster.com/pornstars/caroline-cage'},
|
||||
{ 'name': 'Cindy Dollar', 'url':'https://ge.xhamster.com/pornstars/cindy-dollar'},
|
||||
{ 'name': 'Crystal Frost', 'url':'https://ge.xhamster.com/pornstars/crystal-frost'},
|
||||
{ 'name': 'Fanny Steel', 'url':'https://ge.xhamster.com/pornstars/fanny-steel'},
|
||||
{ 'name': 'Gia Derza', 'url':'https://ge.xhamster.com/pornstars/gia-derza'},
|
||||
{ 'name': 'Horst Baron', 'url':'https://ge.xhamster.com/pornstars/horst-baron'},
|
||||
{ 'name': 'Jasmine Rouge', 'url':'https://ge.xhamster.com/pornstars/jasmine-rouge'},
|
||||
{ 'name': 'Jean-Pierre Armand', 'url':'https://ge.xhamster.com/pornstars/jean-pierre-armand'},
|
||||
{ 'name': 'Jessa Rhodes', 'url':'https://ge.xhamster.com/pornstars/jessa-rhodes'},
|
||||
{ 'name': 'Leonie Saint', 'url':'https://ge.xhamster.com/pornstars/leonie-saint'},
|
||||
{ 'name': 'Linda Ray', 'url':'https://ge.xhamster.com/pornstars/linda-ray'},
|
||||
{ 'name': 'Luca Ferrero', 'url':'https://ge.xhamster.com/pornstars/luca-ferrero'},
|
||||
{ 'name': 'Paris Pink', 'url':'https://ge.xhamster.com/pornstars/paris-pink'},
|
||||
{ 'name': 'Pavlina Stejskalova', 'url':'https://ge.xhamster.com/pornstars/pavlina-stejskalova'},
|
||||
{ 'name': 'Phoenix Marie', 'url':'https://ge.xhamster.com/pornstars/phoenix-marie'},
|
||||
{ 'name': 'Ricky Spanish', 'url':'https://ge.xhamster.com/pornstars/ricky-spanish'},
|
||||
{ 'name': 'Rumika Powers', 'url':'https://ge.xhamster.com/pornstars/rumika-powers'},
|
||||
{ 'name': 'Sara Blonde', 'url':'https://ge.xhamster.com/pornstars/sara-blonde'},
|
||||
{ 'name': 'Sean Lawless', 'url':'https://ge.xhamster.com/pornstars/sean-lawless'},
|
||||
{ 'name': 'Seth Gamble', 'url':'https://ge.xhamster.com/pornstars/seth-gamble'},
|
||||
{ 'name': 'Siri Dahl', 'url':'https://ge.xhamster.com/pornstars/siri-dahl'},
|
||||
{ 'name': 'Stephie Staar', 'url':'https://ge.xhamster.com/pornstars/stephie-staar'},
|
||||
{ 'name': 'Steve Holmes', 'url':'https://ge.xhamster.com/pornstars/steve-holmes'},
|
||||
{ 'name': 'Suzette Dale', 'url':'https://ge.xhamster.com/pornstars/suzette-dale'},
|
||||
{ 'name': 'Uncle George', 'url':'https://ge.xhamster.com/pornstars/uncle-george'},
|
||||
{ 'name': 'Winnie', 'url':'https://ge.xhamster.com/pornstars/winnie'},
|
||||
{ 'name': 'Zenza Raggi', 'url':'https://ge.xhamster.com/pornstars/zenza-raggi'},
|
||||
{ 'name': 'Zorah White', 'url':'https://ge.xhamster.com/pornstars/zorah-white'},
|
||||
{ 'name': 'Marilyn Jess', 'url':'https://ge.xhamster.com/pornstars/marilyn-jess'},
|
||||
{ 'name': 'Alexis Capri', 'url':'https://ge.xhamster.com/pornstars/alexis-capri'},
|
||||
]
|
||||
|
||||
|
||||
for new_actor in new_actor_list:
|
||||
if new_actor['url'] in actors:
|
||||
log.warning(f"Actor {new_actor['url']} already persisted")
|
||||
continue
|
||||
actor_response = requests.post(f"http://127.0.0.1:8800/api/media/actors", json=new_actor)
|
||||
log.warning(f"add status: {actor_response.status_code}")
|
||||
if actor_response.status_code == 201:
|
||||
log.warning(f"add Actor {new_actor['url']} to existing actor list")
|
||||
actors[new_actor['url']] = new_actor
|
||||
actor_data = actor_response.json()
|
||||
log.warning(f"Actor {actor_data} persisted")
|
||||
log.warning('kontor.add_actors finished')
|
||||
@@ -0,0 +1,216 @@
|
||||
"""
|
||||
read file with links and store it in DB
|
||||
"""
|
||||
from datetime import datetime
|
||||
import logging.config
|
||||
import re
|
||||
from typing import Dict, List
|
||||
import uuid
|
||||
from bs4 import BeautifulSoup
|
||||
import requests
|
||||
import yaml
|
||||
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
|
||||
from pathlib import Path
|
||||
from platformdirs import PlatformDirs
|
||||
from pathlib import Path
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker, Session
|
||||
from db.models.base import Base
|
||||
import os
|
||||
|
||||
from db.models.media import MediaActor, MediaActorFile, MediaFile
|
||||
|
||||
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument('--file', '-f', help='file with links', default='~/.sync/media/list.txt')
|
||||
parser.add_argument('--video', help='store Url as VideoFile', action="store_true")
|
||||
parser.add_argument('--config', '-c', default='kontor-docker')
|
||||
parser.add_argument('--verbose', '-v', action='count', default=0)
|
||||
parser.add_argument('--limit', '-l', type=int, help='maximum number of links to check')
|
||||
parser.add_argument('--dry-run', '-m', help='excute script without storing', action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
DB_USER: str = os.getenv("DB_USER", "kontor")
|
||||
DB_PASSWORD: str = os.getenv("DB_PASSWORD", "kontor")
|
||||
DB_SERVER: str = os.getenv("DB_SERVER", "127.0.0.1")
|
||||
DB_PORT: int = int(os.getenv("DB_PORT", 5432))
|
||||
DB_DBNAME: str = os.getenv("DB_DBNAME", "kontor")
|
||||
DATABASE_URL: str = f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_SERVER}:{DB_PORT}/{DB_DBNAME}"
|
||||
|
||||
def get_logger(level, config: str):
|
||||
dirs = PlatformDirs(config)
|
||||
logging_config = Path(dirs.user_config_dir, 'logging-config.yaml')
|
||||
with open(logging_config, 'rt') as f:
|
||||
log_config = yaml.safe_load(f.read())
|
||||
logging.config.dictConfig(log_config)
|
||||
logger = logging.getLogger('development')
|
||||
if level is not None:
|
||||
match level:
|
||||
case 0:
|
||||
logger.setLevel(logging.CRITICAL)
|
||||
case 1:
|
||||
logger.setLevel(logging.INFO)
|
||||
case 2:
|
||||
logger.setLevel(logging.DEBUG)
|
||||
case _:
|
||||
logger.setLevel(logging.INFO)
|
||||
return logger
|
||||
|
||||
def get_session() -> Session:
|
||||
engine = create_engine(DATABASE_URL)
|
||||
Base.metadata.create_all(bind=engine, checkfirst=True)
|
||||
SessionLocal = sessionmaker(bind=engine)
|
||||
return SessionLocal()
|
||||
|
||||
def load_data(filename: str, log) -> List[str]:
|
||||
links: List[str] = []
|
||||
log.debug("load_data")
|
||||
import_file = Path(filename)
|
||||
if not import_file.exists():
|
||||
log.info(f"File {filename} does not exist. Do nothing.")
|
||||
raise FileNotFoundError()
|
||||
log.info("read txt file")
|
||||
with open(filename, 'r') as txt_file:
|
||||
while line := txt_file.readline():
|
||||
# log.info(line.rstrip())
|
||||
links.append(line.rstrip())
|
||||
return links
|
||||
|
||||
def get_actors_mapping(actor_list: List[MediaActor]) -> Dict[str, MediaActor]:
|
||||
mapping: Dict[str, MediaActor] = {}
|
||||
for actor in actor_list:
|
||||
mapping[str(actor.url)] = actor
|
||||
return mapping
|
||||
|
||||
def get_actornames_mapping(actor_list: List[MediaActor]) -> Dict[str, MediaActor]:
|
||||
mapping: Dict[str, MediaActor] = {}
|
||||
for actor in actor_list:
|
||||
mapping[str(actor.name)] = actor
|
||||
return mapping
|
||||
|
||||
def get_meta_info(media_file: MediaFile, log) -> List[str]:
|
||||
actor_links: List[str] = []
|
||||
try:
|
||||
r = requests.get(media_file.url)
|
||||
soup = BeautifulSoup(r.content, "html.parser")
|
||||
error404 = soup.css.select_one('.error404-title')
|
||||
if error404 and error404.get_text() == "Video nicht gefunden":
|
||||
log.warning(f"{error404.get_text()}")
|
||||
media_file.url = None
|
||||
media_file.review = False
|
||||
return actor_links
|
||||
title_tag = soup.find('title')
|
||||
if title_tag:
|
||||
media_file.title = title_tag.get_text()
|
||||
media_file.review = False
|
||||
anchors = soup.find_all('a', attrs={'href': re.compile("^https://.*pornstars/.*")})
|
||||
for anchor in anchors:
|
||||
link_url = str(anchor.get("href")) # type: ignore
|
||||
if link_url.endswith('all/countries'):
|
||||
continue
|
||||
if link_url in actor_links:
|
||||
continue
|
||||
actor_links.append(link_url)
|
||||
except Exception as error:
|
||||
log.info(f"something went wrong: {error}")
|
||||
media_file.title = None
|
||||
media_file.review = True
|
||||
log.info(f"update MediaFile with MetaInfos to {repr(media_file)}")
|
||||
log.info(f"links({len(actor_links)}): {actor_links}")
|
||||
return actor_links
|
||||
|
||||
def get_actor_name(actor_url: str, log: logging.Logger) -> str | None:
|
||||
try:
|
||||
r = requests.get(actor_url)
|
||||
soup = BeautifulSoup(r.content, "html.parser")
|
||||
titles = soup.find_all('h1')
|
||||
for title in titles:
|
||||
log.info(f"title: {title.get_text()}")
|
||||
return title.get_text()
|
||||
except Exception as error:
|
||||
log.warning(f"something went wrong: {error}")
|
||||
return None
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
logger = get_logger(args.verbose, "kontor")
|
||||
logger.info('kontor.add_links started')
|
||||
if args.limit:
|
||||
logger.warning(f"check the first {args.limit} links")
|
||||
session = get_session()
|
||||
links_index = 1
|
||||
with session as db:
|
||||
links = load_data(args.file, logger)
|
||||
for link in links:
|
||||
logger.debug(f"process {link}")
|
||||
media_files = db.query(MediaFile).filter(MediaFile.url == link).all()
|
||||
media_actors = db.query(MediaActor).all()
|
||||
actor_mapping = get_actors_mapping(media_actors)
|
||||
actorname_mapping = get_actornames_mapping(media_actors)
|
||||
if len(media_files) == 0:
|
||||
logger.info(f"MediaFile for link {link} not found")
|
||||
media_file = MediaFile()
|
||||
media_file.id = str(uuid.uuid4())
|
||||
media_file.created_date = datetime.now()
|
||||
media_file.last_modified_date = datetime.now()
|
||||
media_file.version = 0
|
||||
media_file.url = link
|
||||
media_file.review = True
|
||||
media_file.should_download = True
|
||||
media_file.path = None
|
||||
media_file.cloud_link = None
|
||||
media_file.file_name = None
|
||||
actor_urls: List[str] = get_meta_info(media_file, logger)
|
||||
if not args.dry_run:
|
||||
db.add(media_file)
|
||||
db.commit()
|
||||
db.refresh(media_file)
|
||||
for actor_url in actor_urls:
|
||||
if actor_url in actor_mapping:
|
||||
media_actor: MediaActor = actor_mapping[actor_url]
|
||||
# logger.info(f"create mapping for {repr(media_actor)}")
|
||||
media_actor_file = MediaActorFile()
|
||||
media_actor_file.id = str(uuid.uuid4())
|
||||
media_actor_file.created_date = datetime.now()
|
||||
media_actor_file.last_modified_date = datetime.now()
|
||||
media_actor_file.version = 0
|
||||
media_actor_file.media_file_id = media_file.id
|
||||
media_actor_file.media_actor_id = media_actor.id
|
||||
logger.info(f"create mapping with {media_actor_file}")
|
||||
if not args.dry_run:
|
||||
db.add(media_actor_file)
|
||||
db.commit()
|
||||
else:
|
||||
media_actor: MediaActor = None # type: ignore
|
||||
actor_name = get_actor_name(actor_url, logger)
|
||||
if actor_name in actorname_mapping:
|
||||
media_actor = actorname_mapping[actor_name]
|
||||
else:
|
||||
media_actor = MediaActor()
|
||||
media_actor.id = str(uuid.uuid4())
|
||||
media_actor.created_date = datetime.now()
|
||||
media_actor.last_modified_date = datetime.now()
|
||||
media_actor.version = 0
|
||||
media_actor.name = get_actor_name(actor_url, logger)
|
||||
media_actor.url = actor_url
|
||||
logger.info(f"update MediaActor with {repr(media_actor)}")
|
||||
if not args.dry_run:
|
||||
db.add(media_actor)
|
||||
db.commit()
|
||||
media_actor_file = MediaActorFile()
|
||||
media_actor_file.id = str(uuid.uuid4())
|
||||
media_actor_file.created_date = datetime.now()
|
||||
media_actor_file.last_modified_date = datetime.now()
|
||||
media_actor_file.version = 0
|
||||
media_actor_file.media_file_id = media_file.id
|
||||
media_actor_file.media_actor_id = media_actor.id
|
||||
logger.info(f"create mapping with {media_actor_file}")
|
||||
if not args.dry_run:
|
||||
db.add(media_actor_file)
|
||||
db.commit()
|
||||
else:
|
||||
for media_file in media_files:
|
||||
logger.debug(f"MediaFile with {media_file.id} is found")
|
||||
links_index += 1
|
||||
if args.limit and args.limit < links_index:
|
||||
break
|
||||
logger.info('kontor.add_link finished')
|
||||
@@ -4,15 +4,21 @@ Setup database connections
|
||||
import sqlite3
|
||||
import psycopg2
|
||||
import logging.config
|
||||
from platformdirs import PlatformDirs
|
||||
import sqlite3
|
||||
from logging import Logger
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
|
||||
import psycopg2
|
||||
import requests
|
||||
import yaml
|
||||
from platformdirs import PlatformDirs
|
||||
|
||||
|
||||
def get_database_cursors(log, config: str):
|
||||
dirs = PlatformDirs(config)
|
||||
database_config = Path(dirs.user_config_dir, 'database-config.yaml')
|
||||
with open(database_config, 'rt') as f:
|
||||
database_config = Path(dirs.user_config_dir, "database-config.yaml")
|
||||
with open(database_config, "rt") as f:
|
||||
db_config = yaml.safe_load(f.read())
|
||||
sqlite_db = db_config["sqlite"]["file"]
|
||||
log.info('using SQLite3 database {}'.format(sqlite_db))
|
||||
@@ -23,10 +29,10 @@ def get_database_cursors(log, config: str):
|
||||
|
||||
|
||||
def create_tables(sqlite_conn, logger, recreate_db, scripts):
|
||||
logger.info('create_tables')
|
||||
logger.info("create_tables")
|
||||
for table_id in scripts:
|
||||
create_statement = scripts[table_id]['create']
|
||||
drop_statement = scripts[table_id]['drop']
|
||||
create_statement = scripts[table_id]["create"]
|
||||
drop_statement = scripts[table_id]["drop"]
|
||||
logger.debug(create_statement)
|
||||
cursor = sqlite_conn.cursor()
|
||||
if recreate_db:
|
||||
@@ -37,11 +43,12 @@ def create_tables(sqlite_conn, logger, recreate_db, scripts):
|
||||
|
||||
def get_logger(level, config: str):
|
||||
dirs = PlatformDirs(config)
|
||||
logging_config = Path(dirs.user_config_dir, 'logging-config.yaml')
|
||||
with open(logging_config, 'rt') as f:
|
||||
logging_config = Path(dirs.user_config_dir, "logging-config.yaml")
|
||||
log_config = None
|
||||
with open(logging_config, "rt") as f:
|
||||
log_config = yaml.safe_load(f.read())
|
||||
logging.config.dictConfig(log_config)
|
||||
logger = logging.getLogger('development')
|
||||
logger = logging.getLogger("development")
|
||||
if level is not None:
|
||||
match level:
|
||||
case 0:
|
||||
|
||||
+81
-37
@@ -1,72 +1,99 @@
|
||||
"""
|
||||
download files with URLs from DB
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
|
||||
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
|
||||
from datetime import datetime
|
||||
from enum import Enum, auto
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from typing import Any, Dict
|
||||
from uuid import UUID
|
||||
|
||||
from platformdirs import PlatformDirs
|
||||
import requests
|
||||
from config import get_logger
|
||||
import yaml
|
||||
|
||||
from config import get_api_config, get_logger
|
||||
|
||||
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument('--verbose', '-v', action='count', default=0)
|
||||
parser.add_argument('--config', '-c', default='kontor-docker')
|
||||
parser.add_argument('--dir', '-d', default='/data/media')
|
||||
parser.add_argument('--tool', '-t', default='yt-dlp')
|
||||
parser.add_argument('--dry-run', '-m', action='store_true')
|
||||
parser.add_argument("--verbose", "-v", action="count", default=0)
|
||||
parser.add_argument("--config", "-c", default="kontor-docker")
|
||||
parser.add_argument("--dir", "-d", default="/data/media")
|
||||
parser.add_argument("--limit", "-l", type=int, help="maximum number of links to check")
|
||||
parser.add_argument("--tool", "-t", default="yt-dlp")
|
||||
parser.add_argument("--dry-run", "-m", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
class FileStatus(Enum):
|
||||
DOWNLOADED = auto()
|
||||
RENAMED = auto()
|
||||
UNKNOWN = auto()
|
||||
|
||||
def download_file(url: str, file_info: dict, download_dir: str = "/data/media", dl_tool: str = "yt-dlp") -> dict:
|
||||
|
||||
def download_file(
|
||||
url: str,
|
||||
file_info: dict,
|
||||
download_dir: str = "/data/media",
|
||||
dl_tool: str = "yt-dlp",
|
||||
) -> dict:
|
||||
print(f"download file for {url} to {download_dir}")
|
||||
result = subprocess.run([dl_tool, url], cwd=download_dir, capture_output=True, text=True)
|
||||
result = subprocess.run(
|
||||
[dl_tool, url], cwd=download_dir, capture_output=True, text=True
|
||||
)
|
||||
if result.returncode == 0:
|
||||
output = result.stdout
|
||||
output = re.sub(' +', ' ', output)
|
||||
output = re.sub(" +", " ", output)
|
||||
lines_list = output.splitlines()
|
||||
file_name = __parse_output__(lines_list)
|
||||
if file_name is None:
|
||||
file_info['review'] = True
|
||||
file_info['should_download'] = True
|
||||
file_info['file_name'] = None
|
||||
log.info(f"found file: {file_name}")
|
||||
if file_name is None or not file_name.strip():
|
||||
file_info["review"] = True
|
||||
file_info["should_download"] = True
|
||||
file_info["file_name"] = None
|
||||
else:
|
||||
download_file_name = Path(download_dir, file_name)
|
||||
file_info['should_download'] = False
|
||||
file_info['file_name'] = download_file_name.name
|
||||
file_info['cloud_link'] = str(download_file_name.absolute())
|
||||
file_info['last_modified_date'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
file_info["should_download"] = False
|
||||
file_info["review"] = False
|
||||
file_info["file_name"] = download_file_name.name
|
||||
file_info["cloud_link"] = str(download_file_name.absolute())
|
||||
file_info["last_modified_date"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
return file_info
|
||||
|
||||
|
||||
def __parse_output__(lines_list: list[str]) -> str | None:
|
||||
file_name = None
|
||||
for line in lines_list:
|
||||
if 'has already been downloaded' in line:
|
||||
end_len = len(' has already been downloaded')
|
||||
log.debug(f"parse line: {line}")
|
||||
if "has already been downloaded" in line:
|
||||
end_len = len(" has already been downloaded")
|
||||
file_name = line[11:-end_len]
|
||||
if 'Destination' in line:
|
||||
log.info(f"file_name: {file_name}")
|
||||
break
|
||||
if "Destination" in line:
|
||||
line_len = len(line)
|
||||
start_len = len('[download] Destination: ')
|
||||
start_len = len("[download] Destination: ")
|
||||
file_len = line_len - start_len
|
||||
file_name = line[-file_len:]
|
||||
break
|
||||
else:
|
||||
file_name = None
|
||||
return file_name
|
||||
|
||||
|
||||
def is_file_downloaded(media_file: dict, dir: Path) -> FileStatus:
|
||||
file_name_as_title = f"{media_file['file_name']}"
|
||||
if not file_name_as_title:
|
||||
log.info("title has not been set - start download")
|
||||
return FileStatus.UNKNOWN
|
||||
file_title = Path(dir, f"{file_name_as_title}.mp4")
|
||||
if file_title.exists():
|
||||
log.info(f"{file_name_as_title} has been downloaded")
|
||||
media_file['should_download'] = False
|
||||
media_file["should_download"] = False
|
||||
return FileStatus.DOWNLOADED
|
||||
file_name_as_id = f"{media_file['id']}"
|
||||
file_with_id_as_name = Path(dir, f"{file_name_as_id}.mp4")
|
||||
@@ -86,37 +113,54 @@ def update_status(item_id: UUID, file_info: dict):
|
||||
|
||||
|
||||
def rename_file(file_info: dict):
|
||||
item_id = file_info['id']
|
||||
file = Path(args.dir, file_info['file_name'])
|
||||
item_id = file_info["id"]
|
||||
file_name = file_info["file_name"]
|
||||
if file_name is None or not file_name.strip():
|
||||
log.info("file_name is not set, rename is not executed")
|
||||
file_info["review"] = True
|
||||
file_info["should_download"] = True
|
||||
return
|
||||
file = Path(args.dir, file_name)
|
||||
new_file_path = file.with_name(f"{item_id}{file.suffix}")
|
||||
log.info(f"rename {file} to {new_file_path}")
|
||||
file.rename(Path(new_file_path))
|
||||
file_info['cloud_link'] = str(new_file_path)
|
||||
file_info["cloud_link"] = str(new_file_path)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
log = get_logger(args.verbose, args.config)
|
||||
log.info('kontor.download started')
|
||||
response = requests.get("http://127.0.0.1:8800/api/media/files?download=true")
|
||||
log.info(f"Status: {response.status_code}")
|
||||
data = response.json()
|
||||
log.info(f"data: {len(data)}")
|
||||
entries_count = len(data)
|
||||
log.info(f"data: {entries_count}")
|
||||
mediafile_index = 1
|
||||
log.debug(f"data: {data}")
|
||||
missing_actors = {}
|
||||
if args.dry_run:
|
||||
sys.exit(0)
|
||||
if args.limit:
|
||||
log.warning(f"check the first {args.limit} links")
|
||||
for item in data:
|
||||
link = item['url']
|
||||
file_id = item['id']
|
||||
link = item["url"]
|
||||
file_id = item["id"]
|
||||
log.info(f"{file_id} - {link}")
|
||||
download_status: FileStatus = is_file_downloaded(item, args.dir)
|
||||
match download_status:
|
||||
case FileStatus.DOWNLOADED:
|
||||
rename_file(item)
|
||||
update_status(file_id, item)
|
||||
update_status(file_id, item, api_data)
|
||||
case FileStatus.RENAMED:
|
||||
log.info("update status")
|
||||
update_status(file_id, item)
|
||||
update_status(file_id, item, api_data)
|
||||
case FileStatus.UNKNOWN:
|
||||
download_file(link, item)
|
||||
download_file(link, item, args.dir)
|
||||
rename_file(item)
|
||||
log.info(f'{item}')
|
||||
update_status(file_id, item)
|
||||
log.info('kontor.download finished')
|
||||
|
||||
log.info(f"{item}")
|
||||
update_status(file_id, item, api_data)
|
||||
log.warning(f"processed {mediafile_index}/{entries_count}")
|
||||
if args.limit and args.limit <= mediafile_index:
|
||||
break
|
||||
mediafile_index += 1
|
||||
log.info("kontor.download finished")
|
||||
|
||||
@@ -0,0 +1,332 @@
|
||||
"""
|
||||
download files with URLs from DB
|
||||
"""
|
||||
import logging.config
|
||||
import sys
|
||||
from typing import Any, Dict
|
||||
import requests
|
||||
import re
|
||||
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from config import get_api_config
|
||||
|
||||
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument('--verbose', '-v', action='count', default=0)
|
||||
parser.add_argument("--config", "-c", default="kontor-docker")
|
||||
parser.add_argument('--all', '-a', action='store_true')
|
||||
parser.add_argument('--limit', '-l', type=int, help='maximum number of links to check')
|
||||
parser.add_argument('--add-actor', action='store_true', help='add missing actors')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
def get_logger(level: int) -> logging.Logger:
|
||||
logging.config.dictConfig({
|
||||
'version': 1,
|
||||
'disable_existing_loggers': False,
|
||||
'formatters': {
|
||||
'simple': {
|
||||
'format': '[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s',
|
||||
'datefmt': '%Y-%m-%d %H:%M:%S',
|
||||
},
|
||||
},
|
||||
'handlers': {
|
||||
'console': {
|
||||
'class': logging.StreamHandler,
|
||||
'level': logging.DEBUG,
|
||||
'formatter': 'simple',
|
||||
'stream': 'ext://sys.stdout'
|
||||
},
|
||||
},
|
||||
'loggers': {
|
||||
'urllib3.connectionpool': {
|
||||
'level': 'WARNING',
|
||||
'propagate': False,
|
||||
},
|
||||
'root': {
|
||||
'level': 'DEBUG',
|
||||
'handlers': ['console'],
|
||||
},
|
||||
},
|
||||
})
|
||||
logger = logging.getLogger(__file__)
|
||||
if level is not None:
|
||||
match level:
|
||||
case 0:
|
||||
logger.setLevel(logging.WARNING)
|
||||
case 1:
|
||||
logger.setLevel(logging.INFO)
|
||||
case 2:
|
||||
logger.setLevel(logging.DEBUG)
|
||||
case _:
|
||||
logger.setLevel(logging.CRITICAL)
|
||||
return logger
|
||||
|
||||
def update_file(log: logging.Logger, media_file, api_data: Dict[str, Any]):
|
||||
host = api_data["host"]
|
||||
port = api_data["port"]
|
||||
token = api_data['token']
|
||||
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
|
||||
url = f"http://{host}:{port}/api/media/files/{media_file['id']}"
|
||||
update = requests.put(url, headers=headers, json=media_file)
|
||||
log.debug(f"update status: {update.status_code}")
|
||||
log.debug(f"update result: {update.json()}")
|
||||
|
||||
def get_actor_links(log: logging.Logger, media_file_url: str, api_data: Dict[str, Any]) -> list[str]:
|
||||
try:
|
||||
r = requests.get(media_file_url)
|
||||
soup = BeautifulSoup(r.content, "html.parser")
|
||||
error404 = soup.css.select_one('.error404-title')
|
||||
if error404 and error404.get_text() == "Video nicht gefunden":
|
||||
log.warning(f"{error404.get_text()}")
|
||||
media_file['url'] = None
|
||||
media_file['review'] = False
|
||||
update_file(log, media_file, api_data=api_data)
|
||||
return []
|
||||
anchors = soup.find_all('a', attrs={'href': re.compile("^https://.*pornstars/.*")})
|
||||
actor_links = []
|
||||
for anchor in anchors:
|
||||
link_url = str(anchor.get("href")) # type: ignore
|
||||
if link_url.endswith('all/countries'):
|
||||
continue
|
||||
if link_url in actor_links:
|
||||
continue
|
||||
actor_links.append(link_url)
|
||||
log.debug(f"links({len(actor_links)}): {actor_links}")
|
||||
return actor_links
|
||||
except Exception as error:
|
||||
log.warning(f"something went wrong: {error}")
|
||||
return []
|
||||
|
||||
def get_media_files(all_files: bool, api_data: Dict[str, Any])-> Any:
|
||||
files_url = ""
|
||||
host = api_data["host"]
|
||||
port = api_data["port"]
|
||||
token = api_data['token']
|
||||
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
|
||||
if all_files:
|
||||
files_url= f"http://{host}:{port}/api/media/files"
|
||||
else:
|
||||
files_url = f"http://{host}:{port}/api/media/files?review=true"
|
||||
response = requests.get(files_url, headers=headers)
|
||||
log.debug(f"Status: {response.status_code}")
|
||||
data = response.json()
|
||||
return data
|
||||
|
||||
def update_media_file(item, log: logging.Logger, api_data: Dict[str, Any]) -> Any:
|
||||
host = api_data["host"]
|
||||
port = api_data["port"]
|
||||
token = api_data['token']
|
||||
url: str = f"http://{host}:{port}/api/media/files/{item['id']}"
|
||||
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
|
||||
update = requests.put(url, headers=headers, json=item)
|
||||
log.debug(f"update status: {update.status_code}")
|
||||
log.debug(f"update result: {update.json()}")
|
||||
return update.json()
|
||||
|
||||
def update_media_file_actors(mediafile: dict,
|
||||
actor_id_list: list[dict[str, str]],
|
||||
actor_links: list[str],
|
||||
map_ids_actor: dict[str, str],
|
||||
log: logging.Logger,
|
||||
api_data: Dict[str, Any]):
|
||||
media_file_id = mediafile['id']
|
||||
host = api_data["host"]
|
||||
port = api_data["port"]
|
||||
token = api_data['token']
|
||||
url: str = f"http://{host}:{port}/api/media/files/{media_file_id}/actors"
|
||||
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
|
||||
actor_response = requests.put(url, headers=headers, json=actor_id_list)
|
||||
files_actor_list = actor_response.json()
|
||||
persisted_actor_links_count: int = len(files_actor_list)
|
||||
found_actor_links_count: int = len(actor_links)
|
||||
if persisted_actor_links_count < found_actor_links_count:
|
||||
log.warning(f"{persisted_actor_links_count} links persisted, but {found_actor_links_count} links are available")
|
||||
mediafile['review'] = True
|
||||
elif persisted_actor_links_count > found_actor_links_count:
|
||||
log.warning("more persisted links than found actors")
|
||||
for file_actor in files_actor_list:
|
||||
actor_id = file_actor['actor_id']
|
||||
actor_url = map_ids_actor[actor_id]['url'] # type: ignore
|
||||
log.debug(f"check if actor({actor_id}) with {actor_url} in list")
|
||||
if actor_url not in actor_links:
|
||||
log.info(f"actor not found in links, delete relation {file_actor['id']}")
|
||||
delete_media_file_actor(file_actor['id'], log, api_data)
|
||||
mediafile['review'] = True
|
||||
else:
|
||||
mediafile['review'] = False
|
||||
log.debug(f"found {persisted_actor_links_count} actors")
|
||||
log.debug(f"found actors: {files_actor_list}")
|
||||
|
||||
def delete_media_file_actor(media_actor_file_id: str, log: logging.Logger, api_data: Dict[str, Any]):
|
||||
host = api_data["host"]
|
||||
port = api_data["port"]
|
||||
token = api_data['token']
|
||||
url: str = f"http://{host}:{port}/api/media/actorfiles/{media_actor_file_id}"
|
||||
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
|
||||
delete_response = requests.delete(url, headers=headers)
|
||||
if delete_response.status_code == 204:
|
||||
log.info(f"actor file relation with id {media_actor_file_id} successfully deleted")
|
||||
|
||||
def get_actor_ids(link_list: list[str],
|
||||
map_url_actor: dict[str, str],
|
||||
map_ids_actor: dict[str, str],
|
||||
map_path_actor: dict[str, str],
|
||||
missing_actors: dict[str, int],
|
||||
log: logging.Logger) -> list[dict[str, str]]:
|
||||
found_actors: list[dict[str, str]] = []
|
||||
for link in link_list:
|
||||
actor = get_persisted_actor(link, map_url_actor, map_ids_actor, map_path_actor, log)
|
||||
if actor:
|
||||
found_actors.append(actor)
|
||||
else:
|
||||
if link in missing_actors:
|
||||
count = missing_actors[link]
|
||||
missing_actors[link] = count +1
|
||||
else:
|
||||
missing_actors.update({link: 1})
|
||||
return found_actors
|
||||
|
||||
def get_persisted_actor(actor_url: str,
|
||||
map_url_actor: dict[str, str],
|
||||
map_ids_actor: dict[str, str],
|
||||
map_path_actor: dict[str, str],
|
||||
log: logging.Logger) -> dict[str, str] | None:
|
||||
alternate_url_actor: dict[str, dict[str, str]] = {
|
||||
'https://ge.xhamster2.com/pornstars/jean-yves-lecastel':
|
||||
{'id': 'e354b866-717c-4a66-ad38-bc7c23d97e36', 'name': 'Jean-Yves Le Castel', 'url': 'https://ge.xhamster.com/pornstars/jean-yves-le-castel'},
|
||||
'https://ge.xhamster.com/pornstars/jean-yves-lecastel':
|
||||
{'id': 'e354b866-717c-4a66-ad38-bc7c23d97e36', 'name': 'Jean-Yves Le Castel', 'url': 'https://ge.xhamster.com/pornstars/jean-yves-le-castel'},
|
||||
'https://ge.xhamster.com/pornstars/gracie-green':
|
||||
{'id': 'cbec2e0d-869c-40f1-923f-21958d938d9f', 'name': 'Gracie May Green', 'url':'https://ge.xhamster.com/pornstars/gracie-may-green'},
|
||||
'https://ge.xhamster.com/pornstars/thomas-hyka':
|
||||
{'id': '1d814b45-ea98-4acc-88a2-227d3ed36959', 'name': 'Thomas Crown', 'url':'https://ge.xhamster.com/pornstars/thomas-crown'},
|
||||
'https://ge.xhamster.com/pornstars/chloe-couture':
|
||||
{'id': 'e22003a5-60a9-4d86-a1df-ae09ecbe5200', 'name': 'Chloe Cherry', 'url':'https://ge.xhamster.com/pornstars/chloe-cherry'},
|
||||
'https://ge.xhamster.com/pornstars/dava-fox':
|
||||
{'id': 'd913b778-4507-421b-88e0-9da73bb80a63', 'name': 'Dava Foxx', 'url':'https://ge.xhamster.com/pornstars/dava-foxx'},
|
||||
'https://ge.xhamster.com/pornstars/john-dough':
|
||||
{'id': 'a2ecd50f-09b2-4d31-9fcf-1a1438700f51', 'name': 'Jon Dough', 'url':'https://ge.xhamster.com/pornstars/jon-dough'},
|
||||
'https://ge.xhamster.com/pornstars/erica-mori':
|
||||
{'id': '5379dab9-63da-44ed-baf1-929d74ac60b1', 'name': 'Polly Yangs', 'url':'https://ge.xhamster.com/pornstars/polly-yangs'},
|
||||
'https://ge.xhamster.com/pornstars/elnara-cat':
|
||||
{'id': '543952d7-59a9-4492-a70f-e384b5f8eb57', 'name': 'Renata Fox', 'url':'https://ge.xhamster.com/pornstars/renata-fox'},
|
||||
'https://ge.xhamster.com/pornstars/melissa-grand':
|
||||
{'id': '5d025bea-4af6-4197-b38d-3b3afa9d30b9', 'name': 'Melissa Benz', 'url':'https://ge.xhamster.com/pornstars/melissa-benz'},
|
||||
'https://ge.xhamster.com/pornstars/sindy-dollar':
|
||||
{'id': 'fa97769c-9e53-4613-b3c3-4cc1a2672d4b', 'name': 'Cindy Dollar', 'url':'https://ge.xhamster.com/pornstars/cindy-dollar'},
|
||||
} # type: ignore
|
||||
if actor_url in map_url_actor:
|
||||
actor_id: str = map_url_actor[actor_url]['id'] # type: ignore
|
||||
log.debug(f"found actor with id: {actor_id}")
|
||||
return map_ids_actor[actor_id] # type: ignore
|
||||
path = actor_url.split('/')[-1]
|
||||
if path in map_path_actor:
|
||||
actor_id: str = map_path_actor[path]['id'] # type: ignore
|
||||
log.debug(f"found actor with id: {actor_id} by path {path}")
|
||||
return map_ids_actor[actor_id] # type: ignore
|
||||
if actor_url in alternate_url_actor:
|
||||
actor_id: str = alternate_url_actor[actor_url]['id']
|
||||
log.info(f"found actor with id: {actor_id} by alternative {path}")
|
||||
return alternate_url_actor[actor_url]
|
||||
log.info(f"found actor {actor_url} missing")
|
||||
return None
|
||||
|
||||
def get_actors(log: logging.Logger, api_data: Dict[str, Any]):
|
||||
actors_url = {}
|
||||
actors_id = {}
|
||||
actors_path = {}
|
||||
host = api_data["host"]
|
||||
port = api_data["port"]
|
||||
token = api_data['token']
|
||||
url: str = f"http://{host}:{port}/api/media/actors"
|
||||
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
|
||||
response = requests.get(url, headers=headers)
|
||||
data = response.json()
|
||||
for media_actor in data:
|
||||
actor_id = media_actor['id']
|
||||
actor_name = media_actor['name']
|
||||
actor_url = media_actor['url']
|
||||
actor = {}
|
||||
actor['id'] = actor_id
|
||||
actor['name'] = actor_name
|
||||
actor['url'] = actor_url
|
||||
actors_url[actor_url] = actor
|
||||
actors_id[actor_id] = actor
|
||||
actors_path[actor_url.split('/')[-1]] = actor
|
||||
log.debug(f'all actors: {actors_url}')
|
||||
log.debug(f'all actors: {actors_path}')
|
||||
return (actors_url, actors_id, actors_path)
|
||||
|
||||
def get_actor_name(actor_url: str, log: logging.Logger) -> str | None:
|
||||
try:
|
||||
r = requests.get(actor_url)
|
||||
soup = BeautifulSoup(r.content, "html.parser")
|
||||
titles = soup.find_all('h1')
|
||||
for title in titles:
|
||||
log.info(f"title: {title.get_text()}")
|
||||
return title.get_text()
|
||||
except Exception as error:
|
||||
log.warning(f"something went wrong: {error}")
|
||||
return None
|
||||
|
||||
def create_actor(actor_url: str, actor_name: str, log: logging.Logger, api_data: Dict[str, Any]):
|
||||
new_actor = { 'name': actor_name, 'url': actor_url}
|
||||
host = api_data["host"]
|
||||
port = api_data["port"]
|
||||
token = api_data['token']
|
||||
url: str = f"http://{host}:{port}/api/media/actors"
|
||||
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
|
||||
actor_response = requests.post(url, headers=headers, json=new_actor)
|
||||
log.warning(f"add status: {actor_response.status_code}")
|
||||
if actor_response.status_code == 201:
|
||||
actor_data = actor_response.json()
|
||||
log.warning(f"Actor {actor_data} persisted")
|
||||
else:
|
||||
log.info(f"Actor with {actor_url} not persisted")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
log = get_logger(args.verbose)
|
||||
log.warning('kontor.find_links started')
|
||||
log.debug('get all actors')
|
||||
api_data = get_api_config(log, args.config)
|
||||
host = api_data["host"]
|
||||
port = api_data["port"]
|
||||
token = api_data['token']
|
||||
|
||||
(actors_url, actors_id, actors_path) = get_actors(log, api_data=api_data)
|
||||
data = get_media_files(args.all, api_data)
|
||||
entries_count = len(data)
|
||||
mediafile_index = 1
|
||||
log.debug(f"data: {len(data)}")
|
||||
missing_actors = {}
|
||||
if args.limit:
|
||||
log.warning(f"check the first {args.limit} links")
|
||||
for media_file in data:
|
||||
link = media_file['url']
|
||||
media_file_id = media_file['id']
|
||||
if not link:
|
||||
continue
|
||||
if str(link) == "None":
|
||||
continue
|
||||
log.warning(f"{media_file['id']} - {str(link)}")
|
||||
actor_links: list[str] = get_actor_links(log, link, api_data=api_data)
|
||||
actor_id_list = get_actor_ids(actor_links, actors_url, actors_id, actors_path, missing_actors, log)
|
||||
update_media_file_actors(media_file, actor_id_list, actor_links, actors_id, log, api_data=api_data)
|
||||
result = update_media_file(media_file, log, api_data=api_data)
|
||||
log.warning(f"processed {mediafile_index}/{entries_count}")
|
||||
if args.limit and args.limit <= mediafile_index:
|
||||
break
|
||||
mediafile_index += 1
|
||||
for link in missing_actors:
|
||||
log.info(f"{link}: {missing_actors[link]}")
|
||||
actor_name = get_actor_name(link, log)
|
||||
if actor_name and args.add_actor:
|
||||
create_actor(link, actor_name, log, api_data=api_data)
|
||||
log.info("Sort missing actors by occurence count:")
|
||||
sorted_missing = dict(sorted(missing_actors.items(), key=lambda item: item[1]))
|
||||
for key in sorted_missing:
|
||||
log.info(f"{key} : {sorted_missing[key]}")
|
||||
log.warning('kontor.find_links finished')
|
||||
+56
-33
@@ -1,12 +1,12 @@
|
||||
"""
|
||||
read file with URLs and store in DB
|
||||
"""
|
||||
import uuid
|
||||
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
|
||||
import datetime
|
||||
|
||||
import mariadb
|
||||
from setup import get_database_cursors, get_logger, get_scripts, get_meta_data
|
||||
import logging
|
||||
import json
|
||||
from proton import Message, Event
|
||||
from proton.handlers import MessagingHandler
|
||||
from proton.reactor import Container
|
||||
|
||||
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument('-f', '--links', help='file with links')
|
||||
@@ -20,38 +20,61 @@ def read_links_file(links_file):
|
||||
return lines
|
||||
|
||||
|
||||
def add_link_to_db(statement, connection, video_url, log):
|
||||
entry_id = str(uuid.uuid4())
|
||||
current_date_time = datetime.datetime.now()
|
||||
try:
|
||||
cur = connection.cursor()
|
||||
cur.execute(statement, (entry_id, current_date_time, current_date_time, 0, video_url, True, True, None, None, None, None))
|
||||
connection.commit()
|
||||
log.info(f'link {video_url} added to db')
|
||||
except mariadb.Error as insert_error:
|
||||
log.debug("insert failed with %s", insert_error)
|
||||
entry_id = None
|
||||
return entry_id
|
||||
class AddLinkMessage(MessagingHandler):
|
||||
def __init__(self, server, url, log):
|
||||
super(AddLinkMessage, self).__init__()
|
||||
log.info("create AddLinkMessage")
|
||||
self.server = server
|
||||
self.address = "KontorMediaFile::add_link_file"
|
||||
self.url = url
|
||||
self.log = log
|
||||
|
||||
def on_start(self, event: Event):
|
||||
self.log.info("Connection...")
|
||||
conn = event.container.connect(self.server, user="artemis", password="artemis")
|
||||
event.container.create_sender(conn, self.address)
|
||||
|
||||
def on_connection_opened(self, event: Event) -> None:
|
||||
self.log.debug("connection open")
|
||||
|
||||
def on_connection_error(self, event: Event) -> None:
|
||||
self.log.info(f"error: {event}")
|
||||
|
||||
def on_disconnected(self, event: Event) -> None:
|
||||
self.log.debug(f"disconnected: {repr(event)}")
|
||||
|
||||
|
||||
def on_sendable(self, event: Event):
|
||||
self.log.info("send message")
|
||||
message = Message(body=self.url, address=self.address, content_type="application/json", durable=True)
|
||||
delivery = event.sender.send(message)
|
||||
self.log.info(f"Delivery {delivery} sent")
|
||||
event.connection.close()
|
||||
|
||||
def on_accepted(self, event: Event) -> None:
|
||||
self.log.info(f"accepted Delivery: {event.delivery.remote_state}")
|
||||
|
||||
|
||||
def on_rejected(self, event: Event) -> None:
|
||||
self.log.info(f"rejected Delivery: {event.delivery}")
|
||||
|
||||
if __name__ == '__main__':
|
||||
logger = get_logger(args.verbose)
|
||||
logger.info('kontor.read_list started')
|
||||
s_conn, m_conn = get_database_cursors(logger)
|
||||
meta_data_tables = get_meta_data(m_conn)
|
||||
scripts = get_scripts(meta_data_tables, logger)
|
||||
tables = {}
|
||||
for table_id in scripts:
|
||||
tables[scripts[table_id]['name']] = table_id
|
||||
media_file_id = tables['media_file']
|
||||
insert_statement = scripts[tables['media_file']]['insert_mariadb']
|
||||
logging.basicConfig(level=logging.INFO, format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s')
|
||||
logging.info('kontor.read_list started')
|
||||
#conn = stomp.Connection([('127.0.0.1', '61616')])
|
||||
#conn.connect('artemis', 'artemis', wait=True)
|
||||
if args.links:
|
||||
logger.info("read links from file")
|
||||
logging.info("read links from file")
|
||||
links = read_links_file(args.links)
|
||||
for link in links:
|
||||
logger.info("add link to db")
|
||||
add_link_to_db(insert_statement, m_conn, link.strip(), logger)
|
||||
else:
|
||||
logger.info('script used: {}'.format(insert_statement))
|
||||
logger.info('kontor.read_list finished')
|
||||
data_dict = {'url': link.strip()}
|
||||
data = json.dumps(data_dict)
|
||||
logging.info("send link message")
|
||||
handler = AddLinkMessage("amqp://127.0.0.1:5672", data, logging)
|
||||
container = Container(handler)
|
||||
container.container_id = "process_add_links"
|
||||
container.run()
|
||||
# conn.send(body=data, destination='KontorMediaFile::add_link_file', headers={'content-type': 'application/json'})
|
||||
#conn.disconnect()
|
||||
logging.info('kontor.read_list finished')
|
||||
|
||||
|
||||
Generated
+165
@@ -46,6 +46,28 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/38/fc/bce832fd4fd99766c04d1ee0eead6b0ec6486fb100ae5e74c1d91292b982/certifi-2025.1.31-py3-none-any.whl", hash = "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe", size = 166393, upload-time = "2025-01-31T02:16:45.015Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cffi"
|
||||
version = "1.17.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "pycparser" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621, upload_time = "2024-09-04T20:45:21.852Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/8d/f8/dd6c246b148639254dad4d6803eb6a54e8c85c6e11ec9df2cffa87571dbe/cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e", size = 182989, upload_time = "2024-09-04T20:44:28.956Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8b/f1/672d303ddf17c24fc83afd712316fda78dc6fce1cd53011b839483e1ecc8/cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2", size = 178802, upload_time = "2024-09-04T20:44:30.289Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0e/2d/eab2e858a91fdff70533cab61dcff4a1f55ec60425832ddfdc9cd36bc8af/cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3", size = 454792, upload_time = "2024-09-04T20:44:32.01Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/75/b2/fbaec7c4455c604e29388d55599b99ebcc250a60050610fadde58932b7ee/cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683", size = 478893, upload_time = "2024-09-04T20:44:33.606Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4f/b7/6e4a2162178bf1935c336d4da8a9352cccab4d3a5d7914065490f08c0690/cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5", size = 485810, upload_time = "2024-09-04T20:44:35.191Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c7/8a/1d0e4a9c26e54746dc08c2c6c037889124d4f59dffd853a659fa545f1b40/cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4", size = 471200, upload_time = "2024-09-04T20:44:36.743Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/26/9f/1aab65a6c0db35f43c4d1b4f580e8df53914310afc10ae0397d29d697af4/cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd", size = 479447, upload_time = "2024-09-04T20:44:38.492Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5f/e4/fb8b3dd8dc0e98edf1135ff067ae070bb32ef9d509d6cb0f538cd6f7483f/cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed", size = 484358, upload_time = "2024-09-04T20:44:40.046Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f1/47/d7145bf2dc04684935d57d67dff9d6d795b2ba2796806bb109864be3a151/cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9", size = 488469, upload_time = "2024-09-04T20:44:41.616Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bf/ee/f94057fa6426481d663b88637a9a10e859e492c73d0384514a17d78ee205/cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d", size = 172475, upload_time = "2024-09-04T20:44:43.733Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009, upload_time = "2024-09-04T20:44:45.309Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "charset-normalizer"
|
||||
version = "3.4.1"
|
||||
@@ -118,6 +140,35 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/59/f1/4da7717f0063a222db253e7121bd6a56f6fb1ba439dcc36659088793347c/coverage-7.8.0-py3-none-any.whl", hash = "sha256:dbf364b4c5e7bae9250528167dfe40219b62e2d573c854d74be213e1e52069f7", size = 203435, upload-time = "2025-03-30T20:36:43.61Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "coverage"
|
||||
version = "7.8.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/19/4f/2251e65033ed2ce1e68f00f91a0294e0f80c80ae8c3ebbe2f12828c4cd53/coverage-7.8.0.tar.gz", hash = "sha256:7a3d62b3b03b4b6fd41a085f3574874cf946cb4604d2b4d3e8dca8cd570ca501", size = 811872, upload_time = "2025-03-30T20:36:45.376Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f3/21/87e9b97b568e223f3438d93072479c2f36cc9b3f6b9f7094b9d50232acc0/coverage-7.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ac46d0c2dd5820ce93943a501ac5f6548ea81594777ca585bf002aa8854cacd", size = 211708, upload_time = "2025-03-30T20:35:47.417Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/75/be/882d08b28a0d19c9c4c2e8a1c6ebe1f79c9c839eb46d4fca3bd3b34562b9/coverage-7.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:771eb7587a0563ca5bb6f622b9ed7f9d07bd08900f7589b4febff05f469bea00", size = 211981, upload_time = "2025-03-30T20:35:49.002Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7a/1d/ce99612ebd58082fbe3f8c66f6d8d5694976c76a0d474503fa70633ec77f/coverage-7.8.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42421e04069fb2cbcbca5a696c4050b84a43b05392679d4068acbe65449b5c64", size = 245495, upload_time = "2025-03-30T20:35:51.073Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/dc/8d/6115abe97df98db6b2bd76aae395fcc941d039a7acd25f741312ced9a78f/coverage-7.8.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:554fec1199d93ab30adaa751db68acec2b41c5602ac944bb19187cb9a41a8067", size = 242538, upload_time = "2025-03-30T20:35:52.941Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cb/74/2f8cc196643b15bc096d60e073691dadb3dca48418f08bc78dd6e899383e/coverage-7.8.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aaeb00761f985007b38cf463b1d160a14a22c34eb3f6a39d9ad6fc27cb73008", size = 244561, upload_time = "2025-03-30T20:35:54.658Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/22/70/c10c77cd77970ac965734fe3419f2c98665f6e982744a9bfb0e749d298f4/coverage-7.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:581a40c7b94921fffd6457ffe532259813fc68eb2bdda60fa8cc343414ce3733", size = 244633, upload_time = "2025-03-30T20:35:56.221Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/38/5a/4f7569d946a07c952688debee18c2bb9ab24f88027e3d71fd25dbc2f9dca/coverage-7.8.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f319bae0321bc838e205bf9e5bc28f0a3165f30c203b610f17ab5552cff90323", size = 242712, upload_time = "2025-03-30T20:35:57.801Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bb/a1/03a43b33f50475a632a91ea8c127f7e35e53786dbe6781c25f19fd5a65f8/coverage-7.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:04bfec25a8ef1c5f41f5e7e5c842f6b615599ca8ba8391ec33a9290d9d2db3a3", size = 244000, upload_time = "2025-03-30T20:35:59.378Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6a/89/ab6c43b1788a3128e4d1b7b54214548dcad75a621f9d277b14d16a80d8a1/coverage-7.8.0-cp313-cp313-win32.whl", hash = "sha256:dd19608788b50eed889e13a5d71d832edc34fc9dfce606f66e8f9f917eef910d", size = 214195, upload_time = "2025-03-30T20:36:01.005Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/12/12/6bf5f9a8b063d116bac536a7fb594fc35cb04981654cccb4bbfea5dcdfa0/coverage-7.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:a9abbccd778d98e9c7e85038e35e91e67f5b520776781d9a1e2ee9d400869487", size = 214998, upload_time = "2025-03-30T20:36:03.006Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2a/e6/1e9df74ef7a1c983a9c7443dac8aac37a46f1939ae3499424622e72a6f78/coverage-7.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:18c5ae6d061ad5b3e7eef4363fb27a0576012a7447af48be6c75b88494c6cf25", size = 212541, upload_time = "2025-03-30T20:36:04.638Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/04/51/c32174edb7ee49744e2e81c4b1414ac9df3dacfcb5b5f273b7f285ad43f6/coverage-7.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:95aa6ae391a22bbbce1b77ddac846c98c5473de0372ba5c463480043a07bff42", size = 212767, upload_time = "2025-03-30T20:36:06.503Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e9/8f/f454cbdb5212f13f29d4a7983db69169f1937e869a5142bce983ded52162/coverage-7.8.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e013b07ba1c748dacc2a80e69a46286ff145935f260eb8c72df7185bf048f502", size = 256997, upload_time = "2025-03-30T20:36:08.137Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e6/74/2bf9e78b321216d6ee90a81e5c22f912fc428442c830c4077b4a071db66f/coverage-7.8.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d766a4f0e5aa1ba056ec3496243150698dc0481902e2b8559314368717be82b1", size = 252708, upload_time = "2025-03-30T20:36:09.781Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/92/4d/50d7eb1e9a6062bee6e2f92e78b0998848a972e9afad349b6cdde6fa9e32/coverage-7.8.0-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad80e6b4a0c3cb6f10f29ae4c60e991f424e6b14219d46f1e7d442b938ee68a4", size = 255046, upload_time = "2025-03-30T20:36:11.409Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/40/9e/71fb4e7402a07c4198ab44fc564d09d7d0ffca46a9fb7b0a7b929e7641bd/coverage-7.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:b87eb6fc9e1bb8f98892a2458781348fa37e6925f35bb6ceb9d4afd54ba36c73", size = 256139, upload_time = "2025-03-30T20:36:13.86Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/49/1a/78d37f7a42b5beff027e807c2843185961fdae7fe23aad5a4837c93f9d25/coverage-7.8.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:d1ba00ae33be84066cfbe7361d4e04dec78445b2b88bdb734d0d1cbab916025a", size = 254307, upload_time = "2025-03-30T20:36:16.074Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/58/e9/8fb8e0ff6bef5e170ee19d59ca694f9001b2ec085dc99b4f65c128bb3f9a/coverage-7.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f3c38e4e5ccbdc9198aecc766cedbb134b2d89bf64533973678dfcf07effd883", size = 255116, upload_time = "2025-03-30T20:36:18.033Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/56/b0/d968ecdbe6fe0a863de7169bbe9e8a476868959f3af24981f6a10d2b6924/coverage-7.8.0-cp313-cp313t-win32.whl", hash = "sha256:379fe315e206b14e21db5240f89dc0774bdd3e25c3c58c2c733c99eca96f1ada", size = 214909, upload_time = "2025-03-30T20:36:19.644Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/87/e9/d6b7ef9fecf42dfb418d93544af47c940aa83056c49e6021a564aafbc91f/coverage-7.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:2e4b6b87bb0c846a9315e3ab4be2d52fac905100565f4b92f02c445c8799e257", size = 216068, upload_time = "2025-03-30T20:36:21.282Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/59/f1/4da7717f0063a222db253e7121bd6a56f6fb1ba439dcc36659088793347c/coverage-7.8.0-py3-none-any.whl", hash = "sha256:dbf364b4c5e7bae9250528167dfe40219b62e2d573c854d74be213e1e52069f7", size = 203435, upload_time = "2025-03-30T20:36:43.61Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dnspython"
|
||||
version = "2.7.0"
|
||||
@@ -284,6 +335,15 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iniconfig"
|
||||
version = "2.1.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload_time = "2025-03-19T20:09:59.721Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload_time = "2025-03-19T20:10:01.071Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jinja2"
|
||||
version = "3.1.6"
|
||||
@@ -435,6 +495,49 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/08/50/d13ea0a054189ae1bc21af1d85b6f8bb9bbc5572991055d70ad9006fe2d6/psycopg2_binary-2.9.10-cp313-cp313-win_amd64.whl", hash = "sha256:27422aa5f11fbcd9b18da48373eb67081243662f9b46e6fd07c3eb46e4535142", size = 2569224, upload-time = "2025-01-04T20:09:19.234Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pluggy"
|
||||
version = "1.5.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/96/2d/02d4312c973c6050a18b314a5ad0b3210edb65a906f868e31c111dede4a6/pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1", size = 67955, upload_time = "2024-04-20T21:34:42.531Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/88/5f/e351af9a41f866ac3f1fac4ca0613908d9a41741cfcf2228f4ad853b697d/pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669", size = 20556, upload_time = "2024-04-20T21:34:40.434Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proton"
|
||||
version = "0.9.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/6a/2f/512c06db681c1cace7773b5615fab88b507b06b287c061d4b56fdeeafb4e/proton-0.9.1.tar.gz", hash = "sha256:aaea5dcbd3f57b4ef59207b92bc34c43e84be256822b4ea66daf55286c9f256f", size = 16983, upload_time = "2021-08-08T22:32:03.958Z" }
|
||||
|
||||
[[package]]
|
||||
name = "psycopg2-binary"
|
||||
version = "2.9.10"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/cb/0e/bdc8274dc0585090b4e3432267d7be4dfbfd8971c0fa59167c711105a6bf/psycopg2-binary-2.9.10.tar.gz", hash = "sha256:4b3df0e6990aa98acda57d983942eff13d824135fe2250e6522edaa782a06de2", size = 385764, upload_time = "2024-10-16T11:24:58.126Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/3e/30/d41d3ba765609c0763505d565c4d12d8f3c79793f0d0f044ff5a28bf395b/psycopg2_binary-2.9.10-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:26540d4a9a4e2b096f1ff9cce51253d0504dca5a85872c7f7be23be5a53eb18d", size = 3044699, upload_time = "2024-10-16T11:21:42.841Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/35/44/257ddadec7ef04536ba71af6bc6a75ec05c5343004a7ec93006bee66c0bc/psycopg2_binary-2.9.10-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:e217ce4d37667df0bc1c397fdcd8de5e81018ef305aed9415c3b093faaeb10fb", size = 3275245, upload_time = "2024-10-16T11:21:51.989Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1b/11/48ea1cd11de67f9efd7262085588790a95d9dfcd9b8a687d46caf7305c1a/psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:245159e7ab20a71d989da00f280ca57da7641fa2cdcf71749c193cea540a74f7", size = 2851631, upload_time = "2024-10-16T11:21:57.584Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/62/e0/62ce5ee650e6c86719d621a761fe4bc846ab9eff8c1f12b1ed5741bf1c9b/psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c4ded1a24b20021ebe677b7b08ad10bf09aac197d6943bfe6fec70ac4e4690d", size = 3082140, upload_time = "2024-10-16T11:22:02.005Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/27/ce/63f946c098611f7be234c0dd7cb1ad68b0b5744d34f68062bb3c5aa510c8/psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3abb691ff9e57d4a93355f60d4f4c1dd2d68326c968e7db17ea96df3c023ef73", size = 3264762, upload_time = "2024-10-16T11:22:06.412Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/43/25/c603cd81402e69edf7daa59b1602bd41eb9859e2824b8c0855d748366ac9/psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8608c078134f0b3cbd9f89b34bd60a943b23fd33cc5f065e8d5f840061bd0673", size = 3020967, upload_time = "2024-10-16T11:22:11.583Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5f/d6/8708d8c6fca531057fa170cdde8df870e8b6a9b136e82b361c65e42b841e/psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:230eeae2d71594103cd5b93fd29d1ace6420d0b86f4778739cb1a5a32f607d1f", size = 2872326, upload_time = "2024-10-16T11:22:16.406Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ce/ac/5b1ea50fc08a9df82de7e1771537557f07c2632231bbab652c7e22597908/psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bb89f0a835bcfc1d42ccd5f41f04870c1b936d8507c6df12b7737febc40f0909", size = 2822712, upload_time = "2024-10-16T11:22:21.366Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/fc/504d4503b2abc4570fac3ca56eb8fed5e437bf9c9ef13f36b6621db8ef00/psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f0c2d907a1e102526dd2986df638343388b94c33860ff3bbe1384130828714b1", size = 2920155, upload_time = "2024-10-16T11:22:25.684Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b2/d1/323581e9273ad2c0dbd1902f3fb50c441da86e894b6e25a73c3fda32c57e/psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f8157bed2f51db683f31306aa497311b560f2265998122abe1dce6428bd86567", size = 2959356, upload_time = "2024-10-16T11:22:30.562Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/08/50/d13ea0a054189ae1bc21af1d85b6f8bb9bbc5572991055d70ad9006fe2d6/psycopg2_binary-2.9.10-cp313-cp313-win_amd64.whl", hash = "sha256:27422aa5f11fbcd9b18da48373eb67081243662f9b46e6fd07c3eb46e4535142", size = 2569224, upload_time = "2025-01-04T20:09:19.234Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pycparser"
|
||||
version = "2.22"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/1d/b2/31537cf4b1ca988837256c910a668b553fceb8f069bedc4b1c826024b52c/pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", size = 172736, upload_time = "2024-03-30T13:22:22.564Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552, upload_time = "2024-03-30T13:22:20.476Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pydantic"
|
||||
version = "2.11.3"
|
||||
@@ -515,6 +618,34 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/28/d0/def53b4a790cfb21483016430ed828f64830dd981ebe1089971cd10cab25/pytest_cov-6.1.1-py3-none-any.whl", hash = "sha256:bddf29ed2d0ab6f4df17b4c55b0a657287db8684af9c42ea546b21b1041b3dde", size = 23841, upload-time = "2025-04-05T14:07:49.641Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pytest"
|
||||
version = "8.3.5"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "colorama", marker = "sys_platform == 'win32'" },
|
||||
{ name = "iniconfig" },
|
||||
{ name = "packaging" },
|
||||
{ name = "pluggy" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ae/3c/c9d525a414d506893f0cd8a8d0de7706446213181570cdbd766691164e40/pytest-8.3.5.tar.gz", hash = "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845", size = 1450891, upload_time = "2025-03-02T12:54:54.503Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/30/3d/64ad57c803f1fa1e963a7946b6e0fea4a70df53c1a7fed304586539c2bac/pytest-8.3.5-py3-none-any.whl", hash = "sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820", size = 343634, upload_time = "2025-03-02T12:54:52.069Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pytest-cov"
|
||||
version = "6.1.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "coverage" },
|
||||
{ name = "pytest" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/25/69/5f1e57f6c5a39f81411b550027bf72842c4567ff5fd572bed1edc9e4b5d9/pytest_cov-6.1.1.tar.gz", hash = "sha256:46935f7aaefba760e716c2ebfbe1c216240b9592966e7da99ea8292d4d3e2a0a", size = 66857, upload_time = "2025-04-05T14:07:51.592Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/28/d0/def53b4a790cfb21483016430ed828f64830dd981ebe1089971cd10cab25/pytest_cov-6.1.1-py3-none-any.whl", hash = "sha256:bddf29ed2d0ab6f4df17b4c55b0a657287db8684af9c42ea546b21b1041b3dde", size = 23841, upload_time = "2025-04-05T14:07:49.641Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "python-dotenv"
|
||||
version = "1.1.0"
|
||||
@@ -533,6 +664,18 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/45/58/38b5afbc1a800eeea951b9285d3912613f2603bdf897a4ab0f4bd7f405fc/python_multipart-0.0.20-py3-none-any.whl", hash = "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104", size = 24546, upload-time = "2024-12-16T19:45:44.423Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "python-qpid-proton"
|
||||
version = "0.40.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "cffi" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/d5/dd/e9e5066009517bdfee92374264a2b6794fa0987bfeddcbf4d2a08dccaf36/python_qpid_proton-0.40.0.tar.gz", hash = "sha256:7680d607cf6e9684f97bf5b2ba16cda7d8512aab9e4ff78f98d44a4644fc819a", size = 354215, upload_time = "2025-05-19T18:45:37.932Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/7b/dd/a82c1e377f08d62d83898c1aa9b39aef890e910f683fca6dc5242a123f6b/python_qpid_proton-0.40.0-cp313-cp313-win_amd64.whl", hash = "sha256:a19d8c71c908700ceb38f6cbc1eb4a039428570f96bfc2caeeafdfec804fb94f", size = 277376, upload_time = "2025-05-19T19:39:31.201Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyyaml"
|
||||
version = "6.0.2"
|
||||
@@ -678,6 +821,19 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/7b/b6/ebfd6daef0c19a5ca3ac1fb2fc092331d67af5a30c868f106fcc2504c287/stomp_py-8.2.0-py3-none-any.whl", hash = "sha256:fad24e51b505996015a39ca1632df4e0225c1c552980955e21f2aebfc0d9d85c", size = 42751, upload-time = "2024-10-31T21:59:36.658Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "stomp-py"
|
||||
version = "8.2.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "docopt" },
|
||||
{ name = "websocket-client" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/bc/66/c07f01feb5fbc669c4333c76eb02fb8149c653c25ba9769477f8427d5e55/stomp_py-8.2.0.tar.gz", hash = "sha256:9908689361e263bf198e6acfb3c4386759fb7df7d141f4384d7414771c68d7fc", size = 39286, upload_time = "2024-10-31T21:59:38.465Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/7b/b6/ebfd6daef0c19a5ca3ac1fb2fc092331d67af5a30c868f106fcc2504c287/stomp_py-8.2.0-py3-none-any.whl", hash = "sha256:fad24e51b505996015a39ca1632df4e0225c1c552980955e21f2aebfc0d9d85c", size = 42751, upload_time = "2024-10-31T21:59:36.658Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typer"
|
||||
version = "0.15.2"
|
||||
@@ -793,6 +949,15 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/5a/84/44687a29792a70e111c5c477230a72c4b957d88d16141199bf9acb7537a3/websocket_client-1.8.0-py3-none-any.whl", hash = "sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526", size = 58826, upload-time = "2024-04-23T22:16:14.422Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "websocket-client"
|
||||
version = "1.8.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e6/30/fba0d96b4b5fbf5948ed3f4681f7da2f9f64512e1d303f94b4cc174c24a5/websocket_client-1.8.0.tar.gz", hash = "sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da", size = 54648, upload_time = "2024-04-23T22:16:16.976Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/5a/84/44687a29792a70e111c5c477230a72c4b957d88d16141199bf9acb7537a3/websocket_client-1.8.0-py3-none-any.whl", hash = "sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526", size = 58826, upload_time = "2024-04-23T22:16:14.422Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "websockets"
|
||||
version = "15.0.1"
|
||||
|
||||
Reference in New Issue
Block a user