tpeetz/kontor
1
0
Fork 0
Code Issues 6 Pull Requests 2 Actions Packages Projects Releases 2 Wiki Activity

Vorbereitung Release 0.2.0

Browse Source
This commit is contained in:
Thomas Peetz
2026-01-29 23:50:41 +01:00
parent 58f80b3e76
commit b26b5ecc9c
571 changed files with 35728 additions and 5022 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kontor-scripts/add_actors.py
+369
View File
@@ -0,0 +1,369 @@
"""
add actors
"""
import logging.config
import requests
import re
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from bs4 import BeautifulSoup
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--verbose', '-v', action='count', default=0)
args = parser.parse_args()
def get_logger(level: int) -> logging.Logger:
logging.config.dictConfig({
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'simple': {
'format': '[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s',
'datefmt': '%Y-%m-%d %H:%M:%S',
},
},
'handlers': {
'console': {
'class': logging.StreamHandler,
'level': logging.DEBUG,
'formatter': 'simple',
'stream': 'ext://sys.stdout'
},
},
'loggers': {
'urllib3.connectionpool': {
'level': 'WARNING',
'propagate': False,
},
'root': {
'level': 'DEBUG',
'handlers': ['console'],
},
},
})
logger = logging.getLogger(__file__)
if level is not None:
match level:
case 0:
logger.setLevel(logging.WARNING)
case 1:
logger.setLevel(logging.INFO)
case 2:
logger.setLevel(logging.DEBUG)
case _:
logger.setLevel(logging.CRITICAL)
return logger
if __name__ == '__main__':
log = get_logger(args.verbose)
log.warning('kontor.add_actors started')
log.debug('get all actors')
response = requests.get("http://127.0.0.1:8800/api/media/actors")
data = response.json()
actors = {}
for item in data:
actor = {}
actor['id'] = item['id']
actor['name'] = item['name']
actor['url'] = item['url']
actors[item['url']] = actor
log.debug(f'all actors: {actors}')
new_actor_list = [
{ 'name': 'Herschel Savage', 'url': 'https://ge.xhamster.com/pornstars/herschel-savage'},
{ 'name': 'Janey Robbins', 'url': 'https://ge.xhamster.com/pornstars/janey-robbins'},
{ 'name': 'Kimberly Carson', 'url': 'https://ge.xhamster.com/pornstars/kimberly-carson'},
{ 'name': 'Paul Thomas', 'url': 'https://ge.xhamster.com/pornstars/paul-thomas'},
{ 'name': 'Shauna Grant', 'url': 'https://ge.xhamster.com/pornstars/shauna-grant'},
{ 'name': 'Tara Aire', 'url': 'https://ge.xhamster.com/pornstars/tara-aire'},
{ 'name': 'Kari Milla', 'url': 'https://ge.xhamster.com/pornstars/kari-milla'},
{ 'name': 'Drago Gvozdik', 'url': 'https://ge.xhamster.com/pornstars/drago-gvozdik'},
{ 'name': 'Cheyne Collins', 'url': 'https://ge.xhamster.com/pornstars/cheyne-collins'},
{ 'name': 'Christian XXX', 'url': 'https://ge.xhamster.com/pornstars/christian-xxx'},
{ 'name': 'Derrick Pierce', 'url': 'https://ge.xhamster.com/pornstars/derrick-pierce'},
{ 'name': 'Holly Halston', 'url': 'https://ge.xhamster.com/pornstars/holly-halston'},
{ 'name': 'Holly West', 'url': 'https://ge.xhamster.com/pornstars/holly-west'},
{ 'name': 'Jarod Diamond', 'url': 'https://ge.xhamster.com/pornstars/jarod-diamond'},
{ 'name': 'Kristina Cross', 'url': 'https://ge.xhamster.com/pornstars/kristina-cross'},
{ 'name': 'Lee Stone', 'url': 'https://ge.xhamster.com/pornstars/lee-stone'},
{ 'name': 'Monica Mayhem', 'url': 'https://ge.xhamster.com/pornstars/monica-mayhem'},
{ 'name': 'Sienna West', 'url': 'https://ge.xhamster.com/pornstars/sienna-west'},
{ 'name': 'Aria Carson', 'url': 'https://ge.xhamster.com/pornstars/aria-carson'},
{ 'name': 'Britney Amber', 'url': 'https://ge.xhamster.com/pornstars/britney-amber'},
{ 'name': 'Kit Mercer', 'url': 'https://ge.xhamster.com/pornstars/kit-mercer'},
{ 'name': 'Riley Reyes', 'url': 'https://ge.xhamster.com/pornstars/riley-reyes'},
{ 'name': 'Ryan Keely', 'url': 'https://ge.xhamster.com/pornstars/ryan-keely'},
{ 'name': 'Alexandra Diamond', 'url': 'https://ge.xhamster.com/pornstars/alexandra-diamond'},
{ 'name': 'Amanda Twice', 'url': 'https://ge.xhamster.com/pornstars/amanda-twice'},
{ 'name': 'Choky Ice', 'url': 'https://ge.xhamster.com/pornstars/choky-ice'},
{ 'name': 'Cindy Cox', 'url': 'https://ge.xhamster.com/pornstars/cindy-cox'},
{ 'name': 'Kelly White', 'url': 'https://ge.xhamster.com/pornstars/kelly-white'},
{ 'name': 'Mike Foster', 'url': 'https://ge.xhamster.com/pornstars/mike-foster'},
{ 'name': 'Susie Sorrento', 'url': 'https://ge.xhamster.com/pornstars/susie-sorrento'},
{ 'name': 'Lara Sanchez', 'url': 'https://ge.xhamster.com/pornstars/lara-sanchez'},
{ 'name': 'Alicia Williams', 'url': 'https://ge.xhamster.com/pornstars/alicia-williams'},
{ 'name': 'Dirty Tina', 'url': 'https://ge.xhamster.com/pornstars/dirty-tina'},
{ 'name': 'Andrew Andretti', 'url': 'https://ge.xhamster.com/pornstars/andrew-andretti'},
{ 'name': 'Dick Nasty', 'url': 'https://ge.xhamster.com/pornstars/dick-nasty'},
{ 'name': 'Vicky Vette', 'url': 'https://ge.xhamster.com/pornstars/vicky-vette'},
{ 'name': 'Alex Gonz', 'url': 'https://ge.xhamster.com/pornstars/alex-gonz'},
{ 'name': 'Audrey Hollander', 'url': 'https://ge.xhamster.com/pornstars/audrey-hollander'},
{ 'name': 'Desiree Diamond', 'url': 'https://ge.xhamster.com/pornstars/desiree-diamond'},
{ 'name': 'Lexi Bardot', 'url': 'https://ge.xhamster.com/pornstars/lexi-bardot'},
{ 'name': 'Adam Ocelot', 'url': 'https://ge.xhamster.com/pornstars/adam-ocelot'},
{ 'name': 'Scarlett Hampton', 'url': 'https://ge.xhamster.com/pornstars/scarlett-hampton'},
{ 'name': 'Laetitia', 'url': 'https://ge.xhamster.com/pornstars/laetitia'},
{ 'name': 'Raffaela Anderson', 'url': 'https://ge.xhamster.com/pornstars/raffaela-anderson'},
{ 'name': 'Michael Swayze', 'url': 'https://ge.xhamster.com/pornstars/michael-swayze'},
{ 'name': 'Winter Jade', 'url': 'https://ge.xhamster.com/pornstars/winter-jade'},
{ 'name': 'Celine Noiret', 'url': 'https://ge.xhamster.com/pornstars/celine-noiret'},
{ 'name': 'Aurora Snow', 'url': 'https://ge.xhamster.com/pornstars/aurora-snow'},
{ 'name': 'Bree Brooks', 'url': 'https://ge.xhamster.com/pornstars/bree-brooks'},
{ 'name': 'Flick Shagwell', 'url': 'https://ge.xhamster.com/pornstars/flick-shagwell'},
{ 'name': 'Noname Jane', 'url': 'https://ge.xhamster.com/pornstars/noname-jane'},
{ 'name': 'Alex Magni', 'url': 'https://ge.xhamster.com/pornstars/alex-magni'},
{ 'name': 'Maeva Dream', 'url': 'https://ge.xhamster.com/pornstars/maeva-dream'},
{ 'name': 'Squirty Alice', 'url': 'https://ge.xhamster.com/pornstars/squirty-alice'},
{ 'name': 'Mandy Rhea', 'url': 'https://ge.xhamster.com/pornstars/mandy-rhea'},
{ 'name': 'Alessia Donati', 'url': 'https://ge.xhamster.com/pornstars/alessia-donati'},
{ 'name': 'Pierre DJ', 'url': 'https://ge.xhamster.com/pornstars/pierre-dj'},
{ 'name': 'Veronica Belli', 'url': 'https://ge.xhamster.com/pornstars/veronica-belli'},
{ 'name': 'Christiana Cinn', 'url': 'https://ge.xhamster.com/pornstars/christiana-cinn'},
{ 'name': 'Jasmine Jae', 'url': 'https://ge.xhamster.com/pornstars/jasmine-jae'},
{ 'name': 'Jay Smooth', 'url': 'https://ge.xhamster.com/pornstars/jay-smooth'},
{ 'name': 'Natalie Knight', 'url': 'https://ge.xhamster.com/pornstars/natalie-knight'},
{ 'name': 'Joshua Lewis', 'url': 'https://ge.xhamster.com/pornstars/joshua-lewis'},
{ 'name': 'Lauren Phillips', 'url': 'https://ge.xhamster.com/pornstars/lauren-phillips'},
{ 'name': 'Matt Cash', 'url': 'https://ge.xhamster.com/pornstars/matt-cash'},
{ 'name': 'Nickey Huntsman', 'url': 'https://ge.xhamster.com/pornstars/nickey-huntsman'},
{ 'name': 'Jade Sin', 'url': 'https://ge.xhamster.com/pornstars/jade-sin'},
{ 'name': 'Wein Lewis', 'url': 'https://ge.xhamster.com/pornstars/wein-lewis'},
{ 'name': 'Megan Murkovski', 'url': 'https://ge.xhamster.com/pornstars/megan-murkovski'},
{ 'name': 'Tara Wild', 'url': 'https://ge.xhamster.com/pornstars/tara-wild'},
{ 'name': 'Lana Roy', 'url': 'https://ge.xhamster.com/pornstars/lana-roy'},
{ 'name': 'Nick Moreno', 'url': 'https://ge.xhamster.com/pornstars/nick-moreno'},
{ 'name': 'Brad Armstrong', 'url': 'https://ge.xhamster.com/pornstars/brad-armstrong'},
{ 'name': 'Kaylani Lei', 'url': 'https://ge.xhamster.com/pornstars/kaylani-lei'},
{ 'name': 'Deborah Wells', 'url': 'https://ge.xhamster.com/pornstars/deborah-wells'},
{ 'name': 'Judith Ramirez', 'url': 'https://ge.xhamster.com/pornstars/judith-ramirez'},
{ 'name': 'Richard Langin', 'url': 'https://ge.xhamster.com/pornstars/richard-langin'},
{ 'name': 'Simona Valli', 'url': 'https://ge.xhamster.com/pornstars/simona-valli'},
{ 'name': 'Mandy Dee', 'url': 'https://ge.xhamster.com/pornstars/mandy-dee'},
{ 'name': 'Allison Wyte', 'url': 'https://ge.xhamster.com/pornstars/allison-wyte'},
{ 'name': 'Ben English', 'url': 'https://ge.xhamster.com/pornstars/ben-english'},
{ 'name': 'Chuck Martino', 'url': 'https://ge.xhamster.com/pornstars/chuck-martino'},
{ 'name': 'Erika Kole', 'url': 'https://ge.xhamster.com/pornstars/erika-kole'},
{ 'name': 'Katie Morgan', 'url': 'https://ge.xhamster.com/pornstars/katie-morgan'},
{ 'name': 'Misty Parks', 'url': 'https://ge.xhamster.com/pornstars/misty-parks'},
{ 'name': 'Mr. Pete', 'url': 'https://ge.xhamster.com/pornstars/mr-pete'},
{ 'name': 'Candie Luciani', 'url': 'https://ge.xhamster.com/pornstars/candie-luciani'},
{ 'name': 'Clara Mia', 'url': 'https://ge.xhamster.com/pornstars/clara-mia'},
{ 'name': 'Tommy Cabrio', 'url': 'https://ge.xhamster.com/pornstars/tommy-cabrio'},
{ 'name': 'Nathan Bronson', 'url': 'https://ge.xhamster.com/pornstars/nathan-bronson'},
{ 'name': 'Octavia Red', 'url': 'https://ge.xhamster.com/pornstars/octavia-red'},
{ 'name': 'Big George', 'url': 'https://ge.xhamster.com/pornstars/big-george'},
{ 'name': 'Cassy Young', 'url': 'https://ge.xhamster.com/pornstars/cassy-young'},
{ 'name': 'Egon Kowalski', 'url': 'https://ge.xhamster.com/pornstars/egon-kowalski'},
{ 'name': 'Mark Aurel', 'url': 'https://ge.xhamster.com/pornstars/mark-aurel'},
{ 'name': 'Samy Fox', 'url': 'https://ge.xhamster.com/pornstars/samy-fox'},
{ 'name': 'Valeria Jones', 'url': 'https://ge.xhamster.com/pornstars/valeria-jones'},
{ 'name': 'Dieter Von Stein', 'url': 'https://ge.xhamster.com/pornstars/dieter-von-stein'},
{ 'name': 'Donna Bell', 'url': 'https://ge.xhamster.com/pornstars/donna-bell'},
{ 'name': 'Molly Jane', 'url': 'https://ge.xhamster.com/pornstars/molly-jane'},
{ 'name': 'Mark Zane', 'url': 'https://ge.xhamster.com/pornstars/mark-zane'},
{ 'name': 'Titus Steel', 'url': 'https://ge.xhamster.com/pornstars/titus-steel'},
{ 'name': 'Mackenzie Mace', 'url': 'https://ge.xhamster.com/pornstars/mackenzie-mace'},
{ 'name': 'Christina Lang', 'url': 'https://ge.xhamster.com/pornstars/christina-lang'},
{ 'name': 'Jens Modena', 'url': 'https://ge.xhamster.com/pornstars/jens-modena'},
{ 'name': 'Karma Rosenberg', 'url': 'https://ge.xhamster.com/pornstars/karma-rosenberg'},
{ 'name': 'Dani Jensen', 'url': 'https://ge.xhamster.com/pornstars/dani-jensen'},
{ 'name': 'Crystal Ray', 'url': 'https://ge.xhamster.com/pornstars/crystal-ray'},
{ 'name': 'Andrea Szabo', 'url': 'https://ge.xhamster.com/pornstars/andrea-szabo'},
{ 'name': 'Desiree Barclay', 'url': 'https://ge.xhamster.com/pornstars/desiree-barclay'},
{ 'name': 'Jada Stevens', 'url': 'https://ge.xhamster.com/pornstars/jada-stevens'},
{ 'name': 'Syren Demer', 'url': 'https://ge.xhamster.com/pornstars/syren-demer'},
{ 'name': 'Ara Mix', 'url': 'https://ge.xhamster.com/pornstars/ara-mix'},
{ 'name': 'Redneck John', 'url': 'https://ge.xhamster.com/pornstars/redneck-john'},
{ 'name': 'Allysin Chaynes', 'url': 'https://ge.xhamster.com/pornstars/allysin-chaynes'},
{ 'name': 'Aspen Brock', 'url': 'https://ge.xhamster.com/pornstars/aspen-brock'},
{ 'name': 'Kaylynn', 'url': 'https://ge.xhamster.com/pornstars/kaylynn'},
{ 'name': 'Monica Cameron', 'url': 'https://ge.xhamster.com/pornstars/monica-cameron'},
{ 'name': 'Sheila Rossi', 'url': 'https://ge.xhamster.com/pornstars/sheila-rossi'},
{ 'name': 'Alex Ginger', 'url': 'https://ge.xhamster.com/pornstars/alex-ginger'},
{ 'name': 'Ally Style', 'url': 'https://ge.xhamster.com/pornstars/ally-style'},
{ 'name': 'Paola Mike', 'url': 'https://ge.xhamster.com/pornstars/paola-mike'},
{ 'name': 'Jeanette Littledove', 'url': 'https://ge.xhamster.com/pornstars/jeanette-littledove'},
{ 'name': 'Melissa Melendez', 'url': 'https://ge.xhamster.com/pornstars/melissa-melendez'},
{ 'name': 'Peter North', 'url': 'https://ge.xhamster.com/pornstars/peter-north'},
{ 'name': 'Siobhan Hunter', 'url': 'https://ge.xhamster.com/pornstars/siobhan-hunter'},
{ 'name': 'Tami White', 'url': 'https://ge.xhamster.com/pornstars/tami-white'},
{ 'name': 'Tracey Adams', 'url': 'https://ge.xhamster.com/pornstars/tracey-adams'},
{ 'name': 'Ashley Haze', 'url': 'https://ge.xhamster.com/pornstars/ashley-haze'},
{ 'name': 'Cailey Taylor', 'url': 'https://ge.xhamster.com/pornstars/cailey-taylor'},
{ 'name': 'Eve Laurence', 'url': 'https://ge.xhamster.com/pornstars/eve-laurence'},
{ 'name': 'Naudia Nyce', 'url': 'https://ge.xhamster.com/pornstars/naudia-nyce'},
{ 'name': 'Candy Apples', 'url': 'https://ge.xhamster.com/pornstars/candy-apples'},
{ 'name': 'Kate Rich', 'url': 'https://ge.xhamster.com/pornstars/kate-rich'},
{ 'name': 'Aniko Kaposi', 'url': 'https://ge.xhamster.com/pornstars/aniko-kaposi'},
{ 'name': 'Beatrice Poggi', 'url': 'https://ge.xhamster.com/pornstars/beatrice-poggi'},
{ 'name': 'Erika Bella', 'url': 'https://ge.xhamster.com/pornstars/erika-bella'},
{ 'name': 'Nikita Gross', 'url': 'https://ge.xhamster.com/pornstars/nikita-gross'},
{ 'name': 'Ursula Moore', 'url': 'https://ge.xhamster.com/pornstars/ursula-moore'},
{ 'name': 'Caty Kiss', 'url': 'https://ge.xhamster.com/pornstars/caty-kiss'},
{ 'name': 'Light Fairy', 'url': 'https://ge.xhamster.com/pornstars/light-fairy'},
{ 'name': 'Flame', 'url': 'https://ge.xhamster.com/pornstars/flame'},
{ 'name': 'Tiffany Tatum', 'url': 'https://ge.xhamster.com/pornstars/tiffany-tatum'},
{ 'name': 'Alyson Sykes', 'url': 'https://ge.xhamster.com/pornstars/alyson-sykes'},
{ 'name': 'Jenifer Stone', 'url': 'https://ge.xhamster.com/pornstars/jenifer-stone'},
{ 'name': 'Lucy Love', 'url': 'https://ge.xhamster.com/pornstars/lucy-love'},
{ 'name': 'Thomas Stone', 'url': 'https://ge.xhamster.com/pornstars/thomas-stone'},
{ 'name': 'Nikki Babe', 'url': 'https://ge.xhamster.com/pornstars/nikki-babe'},
{ 'name': 'Tyra Misoux', 'url': 'https://ge.xhamster.com/pornstars/tyra-misoux'},
{ 'name': 'Kenzie Reeves', 'url': 'https://ge.xhamster.com/pornstars/kenzie-reeves'},
{ 'name': 'Anissa Kate', 'url': 'https://ge.xhamster.com/pornstars/anissa-kate'},
{ 'name': 'Anna Polina', 'url': 'https://ge.xhamster.com/pornstars/anna-polina'},
{ 'name': 'Kimber Delice', 'url': 'https://ge.xhamster.com/pornstars/kimber-delice'},
{ 'name': 'Lucy Heart', 'url': 'https://ge.xhamster.com/pornstars/lucy-heart'},
{ 'name': 'John Strong', 'url': 'https://ge.xhamster.com/pornstars/john-strong'},
{ 'name': 'Markus Dupree', 'url': 'https://ge.xhamster.com/pornstars/markus-dupree'},
{ 'name': 'Mick Blue', 'url': 'https://ge.xhamster.com/pornstars/mick-blue'},
{ 'name': 'Natasha Nice', 'url': 'https://ge.xhamster.com/pornstars/natasha-nice'},
{ 'name': 'Lena Reif', 'url': 'https://ge.xhamster.com/pornstars/lena-reif'},
{ 'name': 'Sonia Paganini', 'url': 'https://ge.xhamster.com/pornstars/sonia-paganini'},
{ 'name': 'Demi Hawks', 'url': 'https://ge.xhamster.com/pornstars/demi-hawks'},
{ 'name': 'Juan El Caballo Loco', 'url': 'https://ge.xhamster.com/pornstars/juan-el-caballo-loco'},
{ 'name': 'Mike Mancini', 'url': 'https://ge.xhamster.com/pornstars/mike-mancini'},
{ 'name': 'Millie Morgan', 'url': 'https://ge.xhamster.com/pornstars/millie-morgan'},
{ 'name': 'Richard Allan', 'url': 'https://ge.xhamster.com/pornstars/richard-allan'},
{ 'name': 'Damon Dice', 'url': 'https://ge.xhamster.com/pornstars/damon-dice'},
{ 'name': 'Sera Ryder', 'url': 'https://ge.xhamster.com/pornstars/sera-ryder'},
{ 'name': 'Zoltan Kaabai', 'url': 'https://ge.xhamster.com/pornstars/zoltan-kabai'},
{ 'name': 'Cathy Heaven', 'url': 'https://ge.xhamster.com/pornstars/cathy-heaven'},
{ 'name': 'Coco Lovelock', 'url': 'https://ge.xhamster.com/pornstars/coco-lovelock'},
{ 'name': 'Percy Sires', 'url': 'https://ge.xhamster.com/pornstars/percy-sires'},
{ 'name': 'Meridian', 'url': 'https://ge.xhamster.com/pornstars/meridian'},
{ 'name': 'Pascal St. James', 'url': 'https://ge.xhamster.com/pornstars/pascal-st-james'},
{ 'name': 'Red Fox', 'url': 'https://ge.xhamster.com/pornstars/red-fox'},
{ 'name': 'Tony Art', 'url': 'https://ge.xhamster.com/pornstars/tony-art'},
{ 'name': 'Addison Lee', 'url': 'https://ge.xhamster.com/pornstars/addison-lee'},
{ 'name': 'Daria Glover', 'url': 'https://ge.xhamster.com/pornstars/daria-glover'},
{ 'name': 'Mandy Bright', 'url': 'https://ge.xhamster.com/pornstars/mandy-bright'},
{ 'name': 'Antonia Sainz', 'url': 'https://ge.xhamster.com/pornstars/antonia-sainz'},
{ 'name': 'Nicole Love', 'url': 'https://ge.xhamster.com/pornstars/nicole-love'},
{ 'name': 'Sarah Kay', 'url': 'https://ge.xhamster.com/pornstars/sarah-kay'},
{ 'name': 'Judith Kostner', 'url': 'https://ge.xhamster.com/pornstars/judith-kostner'},
{ 'name': 'Maria Bellucci', 'url': 'https://ge.xhamster.com/pornstars/maria-bellucci'},
{ 'name': 'Melissa Monet', 'url': 'https://ge.xhamster.com/pornstars/melissa-monet'},
{ 'name': 'Stephanie Cane', 'url': 'https://ge.xhamster.com/pornstars/stephanie-cane'},
{ 'name': 'Will Steiger', 'url': 'https://ge.xhamster.com/pornstars/will-steiger'},
{ 'name': 'Katty West', 'url': 'https://ge.xhamster.com/pornstars/katty-west'},
{ 'name': 'Jean Pallett', 'url': 'https://ge.xhamster.com/pornstars/jean-pallett'},
{ 'name': 'Conny Dachs', 'url': 'https://ge.xhamster.com/pornstars/conny-dachs'},
{ 'name': 'Juicy Leyla', 'url': 'https://ge.xhamster.com/pornstars/juicy-leyla'},
{ 'name': 'Mandy Mystery', 'url': 'https://ge.xhamster.com/pornstars/mandy-mystery'},
{ 'name': 'Jack Vegas', 'url': 'https://ge.xhamster.com/pornstars/jack-vegas'},
{ 'name': 'Jessica Ryan', 'url': 'https://ge.xhamster.com/pornstars/jessica-ryan'},
{ 'name': 'Nathan Bronson', 'url': 'https://ge.xhamster.com/pornstars/nathan-bronson'},
{ 'name': 'Lexi Lore', 'url': 'https://ge.xhamster.com/pornstars/lexi-lore'},
{ 'name': 'Molly Little', 'url': 'https://ge.xhamster.com/pornstars/molly-little'},
{ 'name': 'Dolly Leigh', 'url': 'https://ge.xhamster.com/pornstars/dolly-leigh'},
{ 'name': 'Marcus London', 'url': 'https://ge.xhamster.com/pornstars/marcus-london'},
{ 'name': 'Kendra Sunderland', 'url': 'https://ge.xhamster.com/pornstars/kendra-sunderland'},
{ 'name': 'Manuel Ferrara', 'url': 'https://ge.xhamster.com/pornstars/manuel-ferrara'},
{ 'name': 'Kelly Trump', 'url': 'https://ge.xhamster.com/pornstars/kelly-trump'},
{ 'name': 'Angelica Heaven', 'url': 'https://ge.xhamster.com/pornstars/angelica-heaven'},
{ 'name': 'Luna Lynx', 'url': 'https://ge.xhamster.com/pornstars/luna-lynx'},
{ 'name': 'Princess Lili', 'url': 'https://ge.xhamster.com/pornstars/princess-lili'},
{ 'name': 'Alexa Wild', 'url': 'https://ge.xhamster.com/pornstars/alexa-wild'},
{ 'name': 'Jessyka Swan', 'url': 'https://ge.xhamster.com/pornstars/jessyka-swan'},
{ 'name': 'Nikky Thorne', 'url': 'https://ge.xhamster.com/pornstars/nikky-thorne'},
{ 'name': 'Tristan Summers', 'url': 'https://ge.xhamster.com/pornstars/tristan-summers'},
{ 'name': 'Rhaya Shyne', 'url': 'https://ge.xhamster.com/pornstars/rhaya-shyne'},
{ 'name': 'Desiree West', 'url': 'https://ge.xhamster.com/pornstars/desiree-west'},
{ 'name': 'Joan Devlon', 'url': 'https://ge.xhamster.com/pornstars/joan-devlon'},
{ 'name': 'Jodi Thorpe', 'url': 'https://ge.xhamster.com/pornstars/jodi-thorpe'},
{ 'name': 'Laurien Dominique', 'url': 'https://ge.xhamster.com/pornstars/laurien-dominique'},
{ 'name': 'Paul Scharf', 'url': 'https://ge.xhamster.com/pornstars/paul-scharf'},
{ 'name': 'Ray Wells', 'url': 'https://ge.xhamster.com/pornstars/ray-wells'},
{ 'name': 'Sandy Carey', 'url': 'https://ge.xhamster.com/pornstars/sandy-carey'},
{ 'name': 'Spender Travis', 'url': 'https://ge.xhamster.com/pornstars/spender-travis'},
{ 'name': 'Starlyn Simone', 'url': 'https://ge.xhamster.com/pornstars/starlyn-simone'},
{ 'name': 'Uschi Digard', 'url': 'https://ge.xhamster.com/pornstars/uschi-digard'},
{ 'name': 'Katarina Muti', 'url': 'https://ge.xhamster.com/pornstars/katarina-muti'},
{ 'name': 'Julia Power', 'url': 'https://ge.xhamster.com/pornstars/julia-power'},
{ 'name': 'Salma De Nora', 'url': 'https://ge.xhamster.com/pornstars/salma-de-nora'},
{ 'name': 'Valeria Jones', 'url': 'https://ge.xhamster.com/pornstars/valeria-jones'},
{ 'name': 'Brandy Canyon', 'url': 'https://ge.xhamster.com/pornstars/brandy-canyon'},
{ 'name': 'Marie Berger', 'url': 'https://ge.xhamster.com/pornstars/marie-berger'},
{ 'name': 'Luna Rishi', 'url': 'https://ge.xhamster.com/pornstars/luna-rishi'},
{ 'name': 'Colleen Brennan', 'url': 'https://ge.xhamster.com/pornstars/colleen-brennan'},
{ 'name': 'Roxanne Brewer', 'url': 'https://ge.xhamster.com/pornstars/roxanne-brewer'},
{ 'name': 'Elle Denay', 'url': 'https://ge.xhamster.com/pornstars/elle-denay'},
{ 'name': 'Juan Largo', 'url': 'https://ge.xhamster.com/pornstars/juan-largo'},
{ 'name': 'Codey Steele', 'url': 'https://ge.xhamster.com/pornstars/codey-steele'},
{ 'name': 'Laney Grey', 'url': 'https://ge.xhamster.com/pornstars/laney-grey'},
{ 'name': 'Stirling Cooper', 'url': 'https://ge.xhamster.com/pornstars/stirling-cooper'},
{ 'name': 'Lana Smalls', 'url': 'https://ge.xhamster.com/pornstars/lana-smalls'},
{ 'name': 'Alex Sanders', 'url':'https://ge.xhamster.com/pornstars/alex-sanders'},
{ 'name': 'Felecia Danay', 'url':'https://ge.xhamster.com/pornstars/felecia-danay'},
{ 'name': 'Lita Chase', 'url':'https://ge.xhamster.com/pornstars/lita-chase'},
{ 'name': 'Mark Ashley', 'url':'https://ge.xhamster.com/pornstars/mark-ashley'},
{ 'name': 'Phyllisha Anne', 'url':'https://ge.xhamster.com/pornstars/phyllisha-anne'},
{ 'name': 'Ryan Conner', 'url':'https://ge.xhamster.com/pornstars/ryan-conner'},
{ 'name': 'Tanya Danielle', 'url':'https://ge.xhamster.com/pornstars/tanya-danielle'},
{ 'name': 'Jessica Moore', 'url':'https://ge.xhamster.com/pornstars/jessica-moore'},
{ 'name': 'Mike Angelo', 'url':'https://ge.xhamster.com/pornstars/mike-angelo'},
{ 'name': 'Morgan Moon', 'url':'https://ge.xhamster.com/pornstars/morgan-moon'},
{ 'name': 'Tyler Steel', 'url':'https://ge.xhamster.com/pornstars/tyler-steel'},
{ 'name': 'Abella Danger', 'url':'https://ge.xhamster.com/pornstars/abella-danger'},
{ 'name': 'Alex Jett', 'url':'https://ge.xhamster.com/pornstars/alex-jett'},
{ 'name': 'Alyson Queen', 'url':'https://ge.xhamster.com/pornstars/alyson-queen'},
{ 'name': 'Antynia Rouge', 'url':'https://ge.xhamster.com/pornstars/antynia-rouge'},
{ 'name': 'Bea Dumas', 'url':'https://ge.xhamster.com/pornstars/bea-dumas'},
{ 'name': 'Callie Black', 'url':'https://ge.xhamster.com/pornstars/callie-black'},
{ 'name': 'Caroline Cage', 'url':'https://ge.xhamster.com/pornstars/caroline-cage'},
{ 'name': 'Cindy Dollar', 'url':'https://ge.xhamster.com/pornstars/cindy-dollar'},
{ 'name': 'Crystal Frost', 'url':'https://ge.xhamster.com/pornstars/crystal-frost'},
{ 'name': 'Fanny Steel', 'url':'https://ge.xhamster.com/pornstars/fanny-steel'},
{ 'name': 'Gia Derza', 'url':'https://ge.xhamster.com/pornstars/gia-derza'},
{ 'name': 'Horst Baron', 'url':'https://ge.xhamster.com/pornstars/horst-baron'},
{ 'name': 'Jasmine Rouge', 'url':'https://ge.xhamster.com/pornstars/jasmine-rouge'},
{ 'name': 'Jean-Pierre Armand', 'url':'https://ge.xhamster.com/pornstars/jean-pierre-armand'},
{ 'name': 'Jessa Rhodes', 'url':'https://ge.xhamster.com/pornstars/jessa-rhodes'},
{ 'name': 'Leonie Saint', 'url':'https://ge.xhamster.com/pornstars/leonie-saint'},
{ 'name': 'Linda Ray', 'url':'https://ge.xhamster.com/pornstars/linda-ray'},
{ 'name': 'Luca Ferrero', 'url':'https://ge.xhamster.com/pornstars/luca-ferrero'},
{ 'name': 'Paris Pink', 'url':'https://ge.xhamster.com/pornstars/paris-pink'},
{ 'name': 'Pavlina Stejskalova', 'url':'https://ge.xhamster.com/pornstars/pavlina-stejskalova'},
{ 'name': 'Phoenix Marie', 'url':'https://ge.xhamster.com/pornstars/phoenix-marie'},
{ 'name': 'Ricky Spanish', 'url':'https://ge.xhamster.com/pornstars/ricky-spanish'},
{ 'name': 'Rumika Powers', 'url':'https://ge.xhamster.com/pornstars/rumika-powers'},
{ 'name': 'Sara Blonde', 'url':'https://ge.xhamster.com/pornstars/sara-blonde'},
{ 'name': 'Sean Lawless', 'url':'https://ge.xhamster.com/pornstars/sean-lawless'},
{ 'name': 'Seth Gamble', 'url':'https://ge.xhamster.com/pornstars/seth-gamble'},
{ 'name': 'Siri Dahl', 'url':'https://ge.xhamster.com/pornstars/siri-dahl'},
{ 'name': 'Stephie Staar', 'url':'https://ge.xhamster.com/pornstars/stephie-staar'},
{ 'name': 'Steve Holmes', 'url':'https://ge.xhamster.com/pornstars/steve-holmes'},
{ 'name': 'Suzette Dale', 'url':'https://ge.xhamster.com/pornstars/suzette-dale'},
{ 'name': 'Uncle George', 'url':'https://ge.xhamster.com/pornstars/uncle-george'},
{ 'name': 'Winnie', 'url':'https://ge.xhamster.com/pornstars/winnie'},
{ 'name': 'Zenza Raggi', 'url':'https://ge.xhamster.com/pornstars/zenza-raggi'},
{ 'name': 'Zorah White', 'url':'https://ge.xhamster.com/pornstars/zorah-white'},
{ 'name': 'Marilyn Jess', 'url':'https://ge.xhamster.com/pornstars/marilyn-jess'},
{ 'name': 'Alexis Capri', 'url':'https://ge.xhamster.com/pornstars/alexis-capri'},
]
for new_actor in new_actor_list:
if new_actor['url'] in actors:
log.warning(f"Actor {new_actor['url']} already persisted")
continue
actor_response = requests.post(f"http://127.0.0.1:8800/api/media/actors", json=new_actor)
log.warning(f"add status: {actor_response.status_code}")
if actor_response.status_code == 201:
log.warning(f"add Actor {new_actor['url']} to existing actor list")
actors[new_actor['url']] = new_actor
actor_data = actor_response.json()
log.warning(f"Actor {actor_data} persisted")
log.warning('kontor.add_actors finished')
kontor-scripts/add_link.py
+81
View File
@@ -0,0 +1,81 @@
"""
read file with URLs and store in DB
"""
import logging.config
import requests
import yaml
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from pathlib import Path
from platformdirs import PlatformDirs
from proton import Message, Event
from proton.handlers import MessagingHandler
from proton.reactor import Container
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('-u', '--url', help='link')
parser.add_argument('--video', help='store Url as VideoFile', action="store_true")
parser.add_argument("--api", help="use Kontor API", action="store_true")
parser.add_argument('--config', '-c', default='kontor-docker')
parser.add_argument('--verbose', '-v', action='count', default=0)
args = parser.parse_args()
def get_logger(level: int, config: str):
dirs = PlatformDirs(config)
logging_config = Path(dirs.user_config_dir, 'logging-config.yaml')
with open(logging_config, 'rt') as f:
configDict = yaml.safe_load(f.read())
logging.config.dictConfig(configDict)
logger = logging.getLogger('development')
if level is not None:
match level:
case 0:
logger.setLevel(logging.INFO)
case 1:
logger.setLevel(logging.DEBUG)
case _:
logger.setLevel(logging.CRITICAL)
return logger
class AddLinkMessage(MessagingHandler):
def __init__(self, server, url, log):
super(AddLinkMessage, self).__init__()
log.info("create AddLinkMessage")
self.server = server
self.address = "add_link_file"
self.url = url
self.log = log
def on_start(self, event: Event):
self.log.info("Connection...")
conn = event.container.connect(self.server, user="artemis", password="artemis")
event.container.create_sender(conn, self.address)
def on_connection_error(self, event: Event) -> None:
self.log.info(f"error: {event}")
def on_sendable(self, event: Event):
self.log.info("send message")
event.sender.send(Message(body=self.url, address=self.address, content_type="text/json"))
event.connection.close()
event.sender.close()
def on_accepted(self, event: Event) -> None:
self.log.info(f"accepted: {event}")
if __name__ == '__main__':
logger = get_logger(args.verbose, args.config)
logger.info('kontor.add_link started')
link: str = args.url
data = {"url": link}
if args.api:
if args.video:
request: str = "http://127.0.0.1:8800/api/video/files"
else:
request: str = "http://127.0.0.1:8800/api/media/files"
response = requests.post(request, json=data)
logger.info(f"Status: {response.status_code}")
data = response.json()
else:
Container(AddLinkMessage("amqp://127.0.0.1:5672", data, logger)).run()
logger.info('kontor.add_link finished')
kontor-scripts/add_links.py
+216
View File
@@ -0,0 +1,216 @@
"""
read file with links and store it in DB
"""
from datetime import datetime
import logging.config
import re
from typing import Dict, List
import uuid
from bs4 import BeautifulSoup
import requests
import yaml
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from pathlib import Path
from platformdirs import PlatformDirs
from pathlib import Path
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, Session
from db.models.base import Base
import os
from db.models.media import MediaActor, MediaActorFile, MediaFile
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--file', '-f', help='file with links', default='~/.sync/media/list.txt')
parser.add_argument('--video', help='store Url as VideoFile', action="store_true")
parser.add_argument('--config', '-c', default='kontor-docker')
parser.add_argument('--verbose', '-v', action='count', default=0)
parser.add_argument('--limit', '-l', type=int, help='maximum number of links to check')
parser.add_argument('--dry-run', '-m', help='excute script without storing', action="store_true")
args = parser.parse_args()
DB_USER: str = os.getenv("DB_USER", "kontor")
DB_PASSWORD: str = os.getenv("DB_PASSWORD", "kontor")
DB_SERVER: str = os.getenv("DB_SERVER", "127.0.0.1")
DB_PORT: int = int(os.getenv("DB_PORT", 5432))
DB_DBNAME: str = os.getenv("DB_DBNAME", "kontor")
DATABASE_URL: str = f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_SERVER}:{DB_PORT}/{DB_DBNAME}"
def get_logger(level, config: str):
dirs = PlatformDirs(config)
logging_config = Path(dirs.user_config_dir, 'logging-config.yaml')
with open(logging_config, 'rt') as f:
log_config = yaml.safe_load(f.read())
logging.config.dictConfig(log_config)
logger = logging.getLogger('development')
if level is not None:
match level:
case 0:
logger.setLevel(logging.CRITICAL)
case 1:
logger.setLevel(logging.INFO)
case 2:
logger.setLevel(logging.DEBUG)
case _:
logger.setLevel(logging.INFO)
return logger
def get_session() -> Session:
engine = create_engine(DATABASE_URL)
Base.metadata.create_all(bind=engine, checkfirst=True)
SessionLocal = sessionmaker(bind=engine)
return SessionLocal()
def load_data(filename: str, log) -> List[str]:
links: List[str] = []
log.debug("load_data")
import_file = Path(filename)
if not import_file.exists():
log.info(f"File {filename} does not exist. Do nothing.")
raise FileNotFoundError()
log.info("read txt file")
with open(filename, 'r') as txt_file:
while line := txt_file.readline():
# log.info(line.rstrip())
links.append(line.rstrip())
return links
def get_actors_mapping(actor_list: List[MediaActor]) -> Dict[str, MediaActor]:
mapping: Dict[str, MediaActor] = {}
for actor in actor_list:
mapping[str(actor.url)] = actor
return mapping
def get_actornames_mapping(actor_list: List[MediaActor]) -> Dict[str, MediaActor]:
mapping: Dict[str, MediaActor] = {}
for actor in actor_list:
mapping[str(actor.name)] = actor
return mapping
def get_meta_info(media_file: MediaFile, log) -> List[str]:
actor_links: List[str] = []
try:
r = requests.get(media_file.url)
soup = BeautifulSoup(r.content, "html.parser")
error404 = soup.css.select_one('.error404-title')
if error404 and error404.get_text() == "Video nicht gefunden":
log.warning(f"{error404.get_text()}")
media_file.url = None
media_file.review = False
return actor_links
title_tag = soup.find('title')
if title_tag:
media_file.title = title_tag.get_text()
media_file.review = False
anchors = soup.find_all('a', attrs={'href': re.compile("^https://.*pornstars/.*")})
for anchor in anchors:
link_url = str(anchor.get("href")) # type: ignore
if link_url.endswith('all/countries'):
continue
if link_url in actor_links:
continue
actor_links.append(link_url)
except Exception as error:
log.info(f"something went wrong: {error}")
media_file.title = None
media_file.review = True
log.info(f"update MediaFile with MetaInfos to {repr(media_file)}")
log.info(f"links({len(actor_links)}): {actor_links}")
return actor_links
def get_actor_name(actor_url: str, log: logging.Logger) -> str | None:
try:
r = requests.get(actor_url)
soup = BeautifulSoup(r.content, "html.parser")
titles = soup.find_all('h1')
for title in titles:
log.info(f"title: {title.get_text()}")
return title.get_text()
except Exception as error:
log.warning(f"something went wrong: {error}")
return None
if __name__ == '__main__':
logger = get_logger(args.verbose, "kontor")
logger.info('kontor.add_links started')
if args.limit:
logger.warning(f"check the first {args.limit} links")
session = get_session()
links_index = 1
with session as db:
links = load_data(args.file, logger)
for link in links:
logger.debug(f"process {link}")
media_files = db.query(MediaFile).filter(MediaFile.url == link).all()
media_actors = db.query(MediaActor).all()
actor_mapping = get_actors_mapping(media_actors)
actorname_mapping = get_actornames_mapping(media_actors)
if len(media_files) == 0:
logger.info(f"MediaFile for link {link} not found")
media_file = MediaFile()
media_file.id = str(uuid.uuid4())
media_file.created_date = datetime.now()
media_file.last_modified_date = datetime.now()
media_file.version = 0
media_file.url = link
media_file.review = True
media_file.should_download = True
media_file.path = None
media_file.cloud_link = None
media_file.file_name = None
actor_urls: List[str] = get_meta_info(media_file, logger)
if not args.dry_run:
db.add(media_file)
db.commit()
db.refresh(media_file)
for actor_url in actor_urls:
if actor_url in actor_mapping:
media_actor: MediaActor = actor_mapping[actor_url]
# logger.info(f"create mapping for {repr(media_actor)}")
media_actor_file = MediaActorFile()
media_actor_file.id = str(uuid.uuid4())
media_actor_file.created_date = datetime.now()
media_actor_file.last_modified_date = datetime.now()
media_actor_file.version = 0
media_actor_file.media_file_id = media_file.id
media_actor_file.media_actor_id = media_actor.id
logger.info(f"create mapping with {media_actor_file}")
if not args.dry_run:
db.add(media_actor_file)
db.commit()
else:
media_actor: MediaActor = None # type: ignore
actor_name = get_actor_name(actor_url, logger)
if actor_name in actorname_mapping:
media_actor = actorname_mapping[actor_name]
else:
media_actor = MediaActor()
media_actor.id = str(uuid.uuid4())
media_actor.created_date = datetime.now()
media_actor.last_modified_date = datetime.now()
media_actor.version = 0
media_actor.name = get_actor_name(actor_url, logger)
media_actor.url = actor_url
logger.info(f"update MediaActor with {repr(media_actor)}")
if not args.dry_run:
db.add(media_actor)
db.commit()
media_actor_file = MediaActorFile()
media_actor_file.id = str(uuid.uuid4())
media_actor_file.created_date = datetime.now()
media_actor_file.last_modified_date = datetime.now()
media_actor_file.version = 0
media_actor_file.media_file_id = media_file.id
media_actor_file.media_actor_id = media_actor.id
logger.info(f"create mapping with {media_actor_file}")
if not args.dry_run:
db.add(media_actor_file)
db.commit()
else:
for media_file in media_files:
logger.debug(f"MediaFile with {media_file.id} is found")
links_index += 1
if args.limit and args.limit < links_index:
break
logger.info('kontor.add_link finished')
kontor-scripts/config.py
+60 -76
View File
@@ -1,37 +1,41 @@
"""
Setup database connections
"""
import sqlite3
import mariadb
import logging.config
from platformdirs import PlatformDirs
import sqlite3
from logging import Logger
from pathlib import Path
from typing import Any, Dict
import psycopg2
import requests
import yaml
from platformdirs import PlatformDirs
def get_database_cursors(log, config: str):
dirs = PlatformDirs(config)
database_config = Path(dirs.user_config_dir, 'database-config.yaml')
with open(database_config, 'rt') as f:
database_config = Path(dirs.user_config_dir, "database-config.yaml")
with open(database_config, "rt") as f:
db_config = yaml.safe_load(f.read())
sqlite_db = db_config["sqlite"]["file"]
log.info('using SQLite3 database {}'.format(sqlite_db))
sqlite_conn = sqlite3.connect(sqlite_db, detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES)
mariadb_conn = mariadb.connect(
host=db_config['mariadb']['host'],
port=db_config['mariadb']['port'],
user=db_config['mariadb']['user'],
password=db_config['mariadb']['password'],
database=db_config['mariadb']['database']
log.info("using SQLite3 database {}".format(sqlite_db))
sqlite_conn = sqlite3.connect(
sqlite_db, detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES
)
return sqlite_conn, mariadb_conn
mariadb_conn = None
postgres_conn = psycopg2.connect(
f"host={db_config['postgres']['host']} port={db_config['postgres']['port']} user={db_config['postgres']['user']} password={db_config['postgres']['password']} dbname={db_config['postgres']['database']}"
)
return sqlite_conn, mariadb_conn, postgres_conn
def create_tables(sqlite_conn, logger, recreate_db, scripts):
logger.info('create_tables')
logger.info("create_tables")
for table_id in scripts:
create_statement = scripts[table_id]['create']
drop_statement = scripts[table_id]['drop']
create_statement = scripts[table_id]["create"]
drop_statement = scripts[table_id]["drop"]
logger.debug(create_statement)
cursor = sqlite_conn.cursor()
if recreate_db:
@@ -42,11 +46,12 @@ def create_tables(sqlite_conn, logger, recreate_db, scripts):
def get_logger(level, config: str):
dirs = PlatformDirs(config)
logging_config = Path(dirs.user_config_dir, 'logging-config.yaml')
with open(logging_config, 'rt') as f:
logging_config = Path(dirs.user_config_dir, "logging-config.yaml")
log_config = None
with open(logging_config, "rt") as f:
log_config = yaml.safe_load(f.read())
logging.config.dictConfig(log_config)
logger = logging.getLogger('development')
logger = logging.getLogger("development")
if level is not None:
match level:
case 0:
@@ -60,59 +65,38 @@ def get_logger(level, config: str):
return logger
def get_meta_data(mariadb_conn):
mariadb_cursor = mariadb_conn.cursor()
select_statement = "SELECT id, table_name FROM meta_data_table"
mariadb_cursor.execute(select_statement)
rows = mariadb_cursor.fetchall()
meta_data = {}
for (identifier, table_name) in rows:
table_data = {"name": table_name}
mariadb_cursor.execute("SELECT column_name, column_sync_name, column_type, column_modifier, column_order FROM meta_data_column WHERE table_id=?", (identifier, ))
column_rows = mariadb_cursor.fetchall()
column_list = []
for (column_name, column_sync_name, column_type, column_modifier, column_order) in column_rows:
column_data = {"column_name": column_name, "column_sync_name": column_sync_name, "column_type": column_type,
"column_modifier": column_modifier, "column_order": column_order}
column_list.append(column_data)
# logger.info(column_list)
table_data["columns"] = column_list
meta_data[identifier] = table_data
return meta_data
def get_scripts(meta_data, logger):
scripts_map = {}
for table_id in meta_data:
table_scripts = {}
m_columns = []
s_columns = []
columns = []
for column_data in meta_data[table_id]["columns"]:
column_line = "{} {}".format(column_data["column_sync_name"], column_data["column_type"])
if column_data["column_modifier"]:
column_line += " " + column_data["column_modifier"]
columns.append(column_line)
m_columns.append(column_data['column_name'])
s_columns.append(column_data['column_sync_name'])
table_name = meta_data[table_id]["name"]
create_statement = "CREATE TABLE IF NOT EXISTS {} ({});".format(table_name, ", ".join(columns))
drop_statement = 'DROP TABLE IF EXISTS {}'.format(table_name)
select_mariadb_statement = 'SELECT {} FROM {}'.format(', '.join(m_columns), table_name)
select_sqlite_statement = 'SELECT {} FROM {}'.format(', '.join(s_columns), table_name)
insert_sqlite_statement = 'INSERT INTO {}({}) VALUES({})'.format(table_name, ', '.join(s_columns), ', '.join(['?']*len(s_columns)))
insert_mariadb_statement = 'INSERT INTO {}({}) VALUES({})'.format(table_name, ', '.join(m_columns), ', '.join(['?']*len(m_columns)))
truncate_mariadb_statement = 'TRUNCATE {}'.format(table_name)
#logger.debug(create_statement)
#logger.debug(select_mariadb_statement)
table_scripts["create"] = create_statement
table_scripts["drop"] = drop_statement
table_scripts["select_mariadb"] = select_mariadb_statement
table_scripts["select_sqlite"] = select_sqlite_statement
table_scripts["insert_sqlite"] = insert_sqlite_statement
table_scripts["insert_mariadb"] = insert_mariadb_statement
table_scripts["truncate_mariadb"] = truncate_mariadb_statement
table_scripts["count"] = "SELECT COUNT(*) FROM {}".format(table_name)
table_scripts["name"] = table_name
scripts_map[table_id] = table_scripts
return scripts_map
def get_api_config(log: Logger, config: str) -> Dict[str, Any]:
api_data: Dict[str, Any] = {}
token: str | None = None
host: str | None = None
port: int = 0
dirs = PlatformDirs(config)
api_config = Path(dirs.user_config_dir, "api.yaml")
with open(api_config, "rt") as f:
api_data = yaml.safe_load(f.read())
if not api_data:
log.fatal("API configuration is missing")
return api_data
host = api_data["host"]
port = api_data["login_port"]
if not token:
log.info("Call login first")
login_url = f"http://{host}:{port}/login"
login_data = {}
login_data["email"] = api_data["email"]
login_data["password"] = api_data["password"]
response = requests.post(login_url, json=login_data)
status = response.status_code
log.info(f"Status: {status}")
if status != 200:
log.fatal("authentication failed")
return api_data
data = response.json()
log.debug(f"got data: {data}")
token = data["access_token"]
token_type = data["token_type"]
api_data["token"] = token
api_data["token_type"] = token_type
with open(api_config, "w") as f:
yaml.dump(api_data, f)
return api_data
kontor-scripts/schema/__init__.py → kontor-scripts/db/__init__.py
View File
kontor-scripts/db/models/__init__.py
+84
View File
@@ -0,0 +1,84 @@
from typing import Any, Dict
from db.models.admin import (
Assignment,
Token,
Profile,
Permission,
MailAccount,
Mail,
)
from db.models.bookshelf import (
ArticleAuthor,
BookAuthor,
BookshelfPublisher,
Article,
Book,
Author,
)
from db.models.comic import (
Issue,
StoryArc,
TradePaperback,
Volume,
ComicWork,
IssueWork,
Artist,
Comic,
Publisher,
WorkType,
)
from db.models.media import (
MediaFile,
MediaActor,
MediaActorFile,
MediaArticle,
MediaVideo,
)
from db.models.tysc import (
Card,
CardSet,
Rooster,
Team,
FieldPosition,
Player,
Vendor,
Sport,
)
registry: Dict[str, Any] = {
Sport.__tablename__: Sport,
Player.__tablename__: Player,
Team.__tablename__: Team,
FieldPosition.__tablename__: FieldPosition,
Rooster.__tablename__: Rooster,
Vendor.__tablename__: Vendor,
CardSet.__tablename__: CardSet,
Card.__tablename__: Card,
Artist.__tablename__: Artist,
Publisher.__tablename__: Publisher,
WorkType.__tablename__: WorkType,
Comic.__tablename__: Comic,
Volume.__tablename__: Volume,
StoryArc.__tablename__: StoryArc,
Issue.__tablename__: Issue,
TradePaperback.__tablename__: TradePaperback,
ComicWork.__tablename__: ComicWork,
IssueWork.__tablename__: IssueWork,
Article.__tablename__: Article,
BookshelfPublisher.__tablename__: BookshelfPublisher,
Book.__tablename__: Book,
Author.__tablename__: Author,
ArticleAuthor.__tablename__: ArticleAuthor,
BookAuthor.__tablename__: BookAuthor,
MediaArticle.__tablename__: MediaArticle,
MediaVideo.__tablename__: MediaVideo,
MediaFile.__tablename__: MediaFile,
MediaActor.__tablename__: MediaActor,
MediaActorFile.__tablename__: MediaActorFile,
Profile.__tablename__: Profile,
Permission.__tablename__: Permission,
Assignment.__tablename__: Assignment,
Token.__tablename__: Token,
MailAccount.__tablename__: MailAccount,
Mail.__tablename__: Mail
}
kontor-scripts/db/models/admin.py
+205
View File
@@ -0,0 +1,205 @@
from datetime import datetime
from typing import Any, Dict
from sqlalchemy import Column, ForeignKey, Integer, String, Boolean
from sqlalchemy.orm import relationship, mapped_column, Mapped
from db.models.base import Base, BaseMixin
class Profile(Base, BaseMixin):
__tablename__ = 'profile'
first_name = Column(String)
last_name = Column(String)
user_name = Column(String, nullable=False)
email = Column(String)
password = Column(String)
enabled = Column(Boolean)
assignments = relationship("Assignment")
tokens = relationship("Token")
def get_full_name(self) -> str:
full_name: str = ""
if self.first_name is not None:
full_name += str(self.first_name)
if self.last_name is not None:
if len(full_name) > 0:
full_name += " "
full_name += str(self.last_name)
return full_name
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.first_name = import_data['first_name']
self.last_name = import_data['last_name']
self.user_name = import_data['user_name']
self.email = import_data['email']
self.password = import_data['password']
self.enabled = import_data['enabled']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['first_name'] = self.first_name
item['last_name'] = self.last_name
item['user_name'] = self.user_name
item['email'] = self.email
item['password'] = self.password
item['enabled'] = self.enabled
return item
class Token(Base, BaseMixin):
__tablename__ = "token"
token = Column(String, nullable=False, unique=True)
name = Column(String)
last_used_date: Mapped[datetime] = mapped_column()
enabled = Column(Boolean)
profile_id = Column(String, ForeignKey("profile.id"), nullable=False)
profile = relationship("Profile", back_populates="tokens")
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.token = import_data['token']
self.name = import_data['name']
self.last_used_date = import_data['last_used_date']
self.enabled = import_data['enabled']
self.profile_id = import_data['profile_id']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['token'] = self.token
item['name'] = self.name
item['last_used_date'] = self.last_used_date
item['enabled'] = self.enabled
item['profile_id'] = self.profile_id
return item
class Permission(Base, BaseMixin):
__tablename__ = "permission"
name = Column(String, nullable=False)
assignments = relationship("Assignment")
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.name = import_data['name']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['name'] = self.name
return item
class Assignment(Base, BaseMixin):
__tablename__ = "assignment"
profile_id = Column(String, ForeignKey("profile.id"), nullable=False)
profile = relationship("Profile", back_populates="assignments")
permission_id = Column(String, ForeignKey("permission.id"), nullable=False)
permission = relationship("Permission", back_populates="assignments")
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.profile_id = import_data['profile_id']
self.permission_id = import_data['permission_id']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['profile_id'] = self.profile_id
item['permission_id'] = self.permission_id
return item
class MailAccount(Base, BaseMixin):
__tablename__ = "mail_account"
host = Column(String)
port = Column(Integer)
protocol = Column(String)
user_name = Column(String)
password = Column(String)
start_tls = Column(Boolean)
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.host = import_data['host']
self.port = import_data['port']
self.protocol = import_data['protocol']
self.user_name = import_data['user_name']
self.password = import_data['password']
self.start_tls = import_data['start_tls']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['host'] = self.host
item['port'] = self.port
item['protocol'] = self.protocol
item['user_name'] = self.user_name
item['password'] = self.password
item['start_tls'] = self.start_tls
return item
class Mail(Base, BaseMixin):
__tablename__ = "mail"
folder: Mapped[str] = mapped_column()
subject: Mapped[str] = mapped_column()
body: Mapped[str] = mapped_column()
sent_date: Mapped[datetime] = mapped_column()
received_date: Mapped[datetime] = mapped_column()
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.folder = import_data['folder']
self.subject = import_data['subject']
self.body = import_data['body']
self.sent_date = import_data['sent_date']
self.received_date = import_data['received_date']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['folder'] = self.folder
item['subject'] = self.subject
item['body'] = self.body
item['sent_date'] = str(self.sent_date)
item['received_date'] = str(self.received_date)
return item
kontor-scripts/schema/base.py → kontor-scripts/db/models/base.py
+10 -11
View File
@@ -1,8 +1,7 @@
import uuid
from datetime import datetime
from sqlalchemy import func, Column, String
from sqlalchemy.dialects.mysql import BIT
from sqlalchemy import func, Column, String, Boolean
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
@@ -11,8 +10,8 @@ class Base(DeclarativeBase):
class BaseMixin:
id = Column(String(255), primary_key=True, default=uuid.uuid4())
# id: Mapped[str] = mapped_column(primary_key=True, default=uuid.uuid4())
#id = Column(String, primary_key=True, default=uuid.uuid4)
id: Mapped[str] = mapped_column(primary_key=True, default=uuid.uuid4)
# created_date = Column(DateTime)
created_date: Mapped[datetime] = mapped_column(default=func.now())
# last_modified_date = Column(DateTime)
@@ -22,10 +21,10 @@ class BaseMixin:
class BaseVideoMixin:
cloud_link = Column(String(255))
file_name = Column(String(255))
path = Column(String(255))
review = Column(BIT(1))
title = Column(String(255))
url = Column(String(255), unique=True)
should_download = Column(BIT(1))
cloud_link = Column(String, nullable=True)
file_name = Column(String, nullable=True)
path = Column(String)
review = Column(Boolean)
title = Column(String)
url = Column(String, nullable=True)
should_download = Column(Boolean)
kontor-scripts/db/models/bookshelf.py
+159
View File
@@ -0,0 +1,159 @@
from typing import Any, AnyStr, Dict
from sqlalchemy import Column, ForeignKey, Integer, String
from sqlalchemy.orm import relationship
from db.models.base import Base, BaseMixin
class Article(Base, BaseMixin):
__tablename__ = 'article'
title = Column(String, unique=True)
article_authors = relationship("ArticleAuthor")
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.title = import_data['title']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['title'] = self.title
return item
class Author(Base, BaseMixin):
__tablename__ = 'author'
first_name = Column(String)
last_name = Column(String)
article_authors = relationship("ArticleAuthor")
book_authors = relationship("BookAuthor")
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.first_name = import_data['first_name']
self.last_name = import_data['last_name']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['first_name'] = self.first_name
item['last_name'] = self.last_name
return item
class BookshelfPublisher(Base, BaseMixin):
__tablename__ = 'bookshelf_publisher'
name = Column(String, unique=True)
books = relationship("Book")
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.name = import_data['name']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['name'] = self.name
return item
class Book(Base, BaseMixin):
__tablename__ = 'book'
isbn = Column(String, unique=True)
title = Column(String)
year = Column(Integer, nullable=False)
publisher_id = Column(String, ForeignKey('bookshelf_publisher.id'), nullable=False)
publisher = relationship('BookshelfPublisher', back_populates="books")
book_authors = relationship("BookAuthor")
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.isbn = import_data['isbn']
self.title = import_data['title']
self.year = import_data['year']
self.publisher_id = import_data['publisher_id']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['isbn'] = self.isbn
item['title'] = self.title
item['year'] = self.year
item['publisher_id'] = self.publisher_id
return item
class ArticleAuthor(Base, BaseMixin):
__tablename__ = 'article_author'
article_id = Column(String, ForeignKey('article.id'), nullable=False)
article = relationship('Article', back_populates="article_authors")
author_id = Column(String, ForeignKey('author.id'), nullable=False)
author = relationship('Author', back_populates="article_authors")
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.article_id = import_data['article_id']
self.author_id = import_data['author_id']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['article_id'] = self.article_id
item['author_id'] = self.author_id
return item
class BookAuthor(Base, BaseMixin):
__tablename__ = 'book_author'
author_id = Column(String, ForeignKey('author.id'), nullable=False)
author = relationship('Author', back_populates="book_authors")
book_id = Column(String, ForeignKey('book.id'), nullable=False)
book = relationship('Book', back_populates="book_authors")
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.author_id = import_data['author_id']
self.book_id = import_data['book_id']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['author_id'] = self.author_id
item['book_id'] = self.book_id
return item
kontor-scripts/db/models/comic.py
+352
View File
@@ -0,0 +1,352 @@
import uuid
from datetime import datetime
from typing import AnyStr, Dict, List, Optional, Any
from sqlalchemy import Column, ForeignKey, Integer, String, Boolean, func
from sqlalchemy.orm import relationship, Mapped, mapped_column
from db.models.base import Base, BaseMixin
class Publisher(Base):
__tablename__ = "publisher"
id: Mapped[str] = mapped_column(primary_key=True, default=uuid.uuid4)
created_date: Mapped[datetime] = mapped_column(default=func.now())
last_modified_date: Mapped[datetime] = mapped_column(default=func.now())
version: Mapped[int] = mapped_column(default=0)
name = Column(String, unique=True)
weblink = Column(String, nullable=True)
parent_publisher_id: Mapped[Optional[str]] = mapped_column(ForeignKey('publisher.id'))
parent_publisher: Mapped[Optional['Publisher']] = relationship("Publisher", back_populates="imprints", remote_side=[id])
imprints: Mapped[List['Publisher']] = relationship('Publisher', back_populates="parent_publisher")
comics = relationship("Comic")
def __repr__(self):
return f'Publisher({self.id} {self.name})'
def __str__(self):
return self.__repr__()
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.name = import_data['name']
self.parent_publisher_id = import_data['parent_publisher_id']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {
'id': self.id,
'created_date': str(self.created_date),
'last_modified_date': str(self.last_modified_date),
'version': self.version,
'name': self.name,
'weblink': self.weblink,
'parent_publisher_id': self.parent_publisher_id
}
return item
class Comic(Base, BaseMixin):
__tablename__ = 'comic'
title = Column(String, unique=True)
publisher_id = Column(String, ForeignKey('publisher.id'), nullable=False)
publisher = relationship("Publisher", back_populates="comics")
current_order = Column(Boolean)
completed = Column(Boolean)
weblink = Column(String, nullable=True)
issues = relationship("Issue")
story_arcs = relationship("StoryArc")
trade_paperbacks = relationship("TradePaperback")
volumes = relationship("Volume")
comic_works = relationship("ComicWork")
def __repr__(self):
return f'Comic({self.id} {self.version} {self.title} {self.publisher.name})'
def __str__(self):
return f'{self.title}({self.id})'
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.title = import_data['title']
self.publisher_id = import_data['publisher_id']
self.current_order = import_data['current_order']
self.completed = import_data['completed']
if 'weblink' in import_data:
self.weblink = import_data['weblink']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {
'id': self.id,
'created_date': str(self.created_date),
'last_modified_date': str(self.last_modified_date),
'version': self.version,
'title': self.title,
'publisher_id': self.publisher_id,
'current_order': self.current_order,
'completed': self.completed,
'weblink': self.weblink
}
return item
class Volume(Base, BaseMixin):
__tablename__ = "volume"
name = Column(String, nullable=False)
comic_id = Column(String, ForeignKey("comic.id"), nullable=False)
comic = relationship("Comic", back_populates="volumes")
story_arcs = relationship("StoryArc")
issues = relationship("Issue")
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.name = import_data['name']
self.comic_id = import_data['comic_id']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['name'] = self.name
item['comic_id'] = self.comic_id
return item
class TradePaperback(Base, BaseMixin):
__tablename__ = "trade_paperback"
name = Column(String, nullable=False)
issue_start = Column(Integer)
issue_end = Column(Integer)
comic_id = Column(String, ForeignKey("comic.id"), nullable=False)
comic = relationship("Comic", back_populates="trade_paperbacks")
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.name = import_data['name']
self.issue_start = import_data['issue_start']
self.issue_end = import_data['issue_end']
self.comic_id = import_data['comic_id']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['name'] = self.name
item['issue_start'] = self.issue_start
item['issue_end'] = self.issue_end
item['comic_id'] = self.comic_id
return item
class StoryArc(Base, BaseMixin):
__tablename__ = "story_arc"
name = Column(String, nullable=False)
comic_id = Column(String, ForeignKey("comic.id"), nullable=False)
comic = relationship("Comic", back_populates="story_arcs")
volume_id = Column(String, ForeignKey("volume.id"), nullable=True)
volume = relationship("Volume", back_populates="story_arcs")
issues = relationship("Issue")
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.name = import_data['name']
self.comic_id = import_data['comic_id']
self.volume_id = import_data['volume_id']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {
'id': self.id,
'created_date': str(self.created_date),
'last_modified_date': str(self.last_modified_date),
'version': self.version,
'name': self.name,
'comic_id': self.comic_id,
'volume_id': self.volume_id
}
return item
class Issue(Base, BaseMixin):
__tablename__ = "issue"
issue_number = Column(String)
title = Column(String, nullable=True)
published_on: Mapped[datetime] = mapped_column(nullable=True)
in_stock = Column(Boolean)
is_read = Column(Boolean)
comic_id = Column(String, ForeignKey("comic.id"), nullable=False)
comic = relationship("Comic", back_populates="issues")
volume_id = Column(String, ForeignKey("volume.id"), nullable=True)
volume = relationship("Volume", back_populates="issues")
story_arc_id = Column(String, ForeignKey("story_arc.id"), nullable=True)
story_arc = relationship("StoryArc", back_populates="issues")
issue_works = relationship("IssueWork")
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.issue_number = import_data['issue_number']
self.title = import_data['title']
if import_data['published_on'] == 'None':
self.published_on = None # type: ignore
else:
self.published_on = import_data['published_on']
self.in_stock = import_data['in_stock']
self.is_read = import_data['is_read']
self.comic_id = import_data['comic_id']
self.volume_id = import_data['volume_id']
self.story_arc_id = import_data['story_arc_id']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {
'id': self.id,
'created_date': str(self.created_date),
'last_modified_date': str(self.last_modified_date),
'version': self.version,
'issue_number': self.issue_number,
'title': self.title,
'published_on': str(self.published_on),
'in_stock': self.in_stock,
'is_read': self.is_read,
'comic_id': self.comic_id,
'volume_id': self.volume_id,
'story_arc_id': self.story_arc_id
}
return item
class Artist(Base, BaseMixin):
__tablename__ = "artist"
name = Column(String, nullable=False)
weblink = Column(String, nullable=True)
comic_works = relationship("ComicWork")
issue_works = relationship("IssueWork")
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.name = import_data['name']
if 'weblink' in import_data:
self.weblink = import_data['weblink']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {
'id': self.id,
'created_date': str(self.created_date),
'last_modified_date': str(self.last_modified_date),
'version': self.version,
'name': self.name,
'weblink': self.weblink
}
return item
class WorkType(Base, BaseMixin):
__tablename__ = "worktype"
name = Column(String, nullable=False, unique=True)
comic_works = relationship("ComicWork")
issue_works = relationship("IssueWork")
def __repr__(self):
return f'Worktype({self.id} {self.version} {self.name} {len(self.comic_works)})'
def __str__(self):
return f'{self.name}({self.id})'
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.name = import_data['name']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {
'id': self.id,
'created_date': str(self.created_date),
'last_modified_date': str(self.last_modified_date),
'version': self.version,
'name': self.name
}
return item
class ComicWork(Base, BaseMixin):
__tablename__ = "comic_work"
comic_id = Column(String, ForeignKey("comic.id"), nullable=False)
comic = relationship("Comic", back_populates="comic_works")
artist_id = Column(String, ForeignKey("artist.id"), nullable=False)
artist = relationship("Artist", back_populates="comic_works")
work_type_id = Column(String, ForeignKey("worktype.id"), nullable=False)
work_type = relationship("WorkType", back_populates="comic_works")
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.comic_id = import_data['comic_id']
self.artist_id = import_data['artist_id']
self.work_type_id = import_data['work_type_id']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {
'id': self.id,
'created_date': str(self.created_date),
'last_modified_date': str(self.last_modified_date),
'version': self.version,
'comic_id': self.comic_id,
'artist_id': self.artist_id,
'work_type_id': self.work_type_id
}
return item
class IssueWork(Base, BaseMixin):
__tablename__ = "issue_work"
issue_id = Column(String, ForeignKey("issue.id"), nullable=False)
issue = relationship("Issue", back_populates="issue_works")
artist_id = Column(String, ForeignKey("artist.id"), nullable=False)
artist = relationship("Artist", back_populates="issue_works")
work_type_id = Column(String, ForeignKey("worktype.id"), nullable=False)
work_type = relationship("WorkType", back_populates="issue_works")
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.issue_id = import_data['issue_id']
self.artist_id = import_data['artist_id']
self.work_type_id = import_data['work_type_id']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {
'id': self.id,
'created_date': str(self.created_date),
'last_modified_date': str(self.last_modified_date),
'version': self.version,
'issue_id': self.issue_id,
'artist_id': self.artist_id,
'work_type_id': self.work_type_id
}
return item
kontor-scripts/db/models/database.py
+66
View File
@@ -0,0 +1,66 @@
from enum import Enum, auto
from logging import Logger
from typing import Any
from sqlalchemy import select
from sqlalchemy.orm import sessionmaker
from db.models import registry
class ColumnEntry(Enum):
COLUMN_NAME = 'column'
COLUMN_LABEL = 'label'
COLUMN_ORDER = 'order'
COLUMN_REF_COLUMN = 'ref_column'
COLUMN_TYPE = 'type'
COLUMN_WIDGET = 'widget'
class StatusType(Enum):
UNKNOWN = auto()
FILE_NAME = auto()
FILE_ID = auto()
DUPLICATE = auto()
CLOUD_LINK = auto()
CLOUD_LINK_ID = auto()
class ExportType(Enum):
JSON = "JSON"
YAML = "YAML"
SQLITE = "SQLite"
class KontorDB:
def __init__(self, db_engine: Any, log: Logger):
self.engine = db_engine
self.log = log
def data(self, table_name: str, columns: dict, filters: dict) -> list:
data = []
__session__ = sessionmaker(self.engine)
table = registry[table_name]
with __session__() as session:
entries = []
if len(filters) == 0:
entries = session.scalars(select(table)).all()
else:
entries = session.scalars(select(table).filter_by(**filters)).all()
for entry in entries:
# self.log.info("data: %s", entry)
row = []
for order in columns.keys():
column_name = columns[order][ColumnEntry.COLUMN_NAME]
ref_column = columns[order][ColumnEntry.COLUMN_REF_COLUMN]
if str(column_name).endswith("_id"):
ref_table = column_name[:-3]
ref = getattr(entry, ref_table)
value = getattr(ref, ref_column)
row.append(value)
else:
row.append(getattr(entry, column_name))
data.append(row)
# self.log.info("data: %s", data)
return data
kontor-scripts/db/models/media.py
+224
View File
@@ -0,0 +1,224 @@
import re
import subprocess
from datetime import datetime
from pathlib import Path
from typing import Any, Dict
import requests
from bs4 import BeautifulSoup
from sqlalchemy import Boolean, Column, String, ForeignKey
from sqlalchemy.orm import relationship
from db.models.base import Base, BaseMixin, BaseVideoMixin
class MediaFile(Base, BaseMixin, BaseVideoMixin):
__tablename__ = 'media_file'
media_actor_files = relationship("MediaActorFile")
def __repr__(self):
return f'MediaFile(\n\tID: {self.id}\n\tTitle: {self.title}\n\tURL: {self.url}\n\tReview: {self.review}\n\tDownload: {self.should_download}\n\tPath: {self.path}\n\tCloudlink: {self.cloud_link})'
def __str__(self):
return f'{self.title}({self.id})'
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.cloud_link = import_data['cloud_link']
self.file_name = import_data['file_name']
self.path = import_data['path']
self.review = import_data['review']
self.title = import_data['title']
self.url = import_data['url']
self.should_download = import_data['should_download']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['cloud_link'] = self.cloud_link
item['file_name'] = self.file_name
item['path'] = self.path
item['review'] = self.review
item['title'] = self.title
item['url'] = self.url
item['should_download'] = self.should_download
return item
def update_title(self) -> None:
print(f"update title for {self.url}")
try:
r = requests.get(self.url)
soup = BeautifulSoup(r.content, "html.parser")
title_tag = soup.find('title')
if title_tag:
self.title = title_tag.get_text()
self.review = False
except:
self.title = None
self.review = True
self.last_modified_date = datetime.now()
def download_file(self, download_dir: str, dl_tool: str):
print(f"download file for {self.url} to {download_dir}")
result = subprocess.run([dl_tool, self.url], cwd=download_dir, capture_output=True, text=True)
if result.returncode == 0:
output = result.stdout
output = re.sub(' +', ' ', output)
lines_list = output.splitlines()
file_name = self.__parse_output__(lines_list)
if file_name is None:
self.review = True
self.should_download = True
self.file_name = None
else:
download_file = Path(file_name)
self.should_download = False
self.file_name = download_file.name
self.cloud_link = str(download_file.absolute())
self.last_modified_date = datetime.now()
def __parse_output__(self, lines_list):
self.file_name = None
for line in lines_list:
if 'has already been downloaded' in line:
end_len = len(' has already been downloaded')
self.file_name = line[11:-end_len]
if 'Destination' in line:
line_len = len(line)
start_len = len('[download] Destination: ')
file_len = line_len - start_len
self.file_name = line[-file_len:]
return self.file_name
class MediaActor(Base, BaseMixin):
__tablename__ = 'media_actor'
name = Column(String)
url = Column(String, unique=True)
media_actor_files = relationship("MediaActorFile")
def __repr__(self):
return f'MediaActor(\n\tID: {self.id}\n\tName: {self.name}\n\tURL: {self.url})'
def __str__(self):
return f'{self.name}({self.id})'
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.name = import_data['name']
self.url = import_data['url']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['name'] = self.name
item['url'] = self.url
return item
class MediaActorFile(Base, BaseMixin):
__tablename__ = 'media_actor_file'
media_actor_id = Column(String, ForeignKey("media_actor.id"), nullable=False)
media_actor = relationship("MediaActor", back_populates="media_actor_files")
media_file_id = Column(String, ForeignKey("media_file.id"), nullable=True)
media_file = relationship("MediaFile", back_populates="media_actor_files")
def __repr__(self):
return f'MediaActorFile(\n\tID: {self.id}\n\tMediaActor: {self.media_actor_id}\n\tMediaFile: {self.media_file_id})'
def __str__(self):
return f'{self.id}: MediaActor: {self.media_actor_id} - MediaFile: {self.media_file_id}'
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.media_actor_id = import_data['media_actor_id']
self.media_file_id = import_data['media_file_id']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['media_actor_id'] = self.media_actor_id
item['media_file_id'] = self.media_file_id
return item
class MediaArticle(Base, BaseMixin):
__tablename__ = 'media_article'
review = Column(Boolean)
title = Column(String)
url = Column(String, unique=True)
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.review = import_data['review']
self.title = import_data['title']
self.url = import_data['url']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['review'] = self.review
item['title'] = self.title
item['url'] = self.url
return item
class MediaVideo(Base, BaseMixin):
__tablename__ = 'media_video'
cloud_link = Column(String)
file_name = Column(String)
path = Column(String)
review = Column(Boolean)
title = Column(String)
url = Column(String, unique=True)
should_download = Column(Boolean)
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.cloud_link = import_data['cloud_link']
self.file_name = import_data['file_name']
self.path = import_data['path']
self.review = import_data['review']
self.title = import_data['title']
self.url = import_data['url']
self.should_download = import_data['should_download']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['cloud_link'] = self.cloud_link
item['file_name'] = self.file_name
item['path'] = self.path
item['review'] = self.review
item['title'] = self.title
item['url'] = self.url
item['should_download'] = self.should_download
return item
kontor-scripts/db/models/tysc.py
+259
View File
@@ -0,0 +1,259 @@
from typing import Dict, Any
from sqlalchemy import Column, Integer, String, ForeignKey, UniqueConstraint, Boolean
from sqlalchemy.orm import relationship
from db.models.base import Base, BaseMixin
class Sport(Base, BaseMixin):
__tablename__ = "sport"
__table_args__ = (
UniqueConstraint("name"),
)
name = Column(String, nullable=False, index=True, unique=True)
teams = relationship("Team")
positions = relationship("FieldPosition")
def __repr__(self):
return f"Sport(id={self.id}, name={self.name}, created_date={self.created_date})"
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.name = import_data['name']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['name'] = self.name
return item
class Team(Base, BaseMixin):
__tablename__ = "team"
name = Column(String, nullable=False, index=True, unique=True)
short_name = Column(String, nullable=False, )
sport_id = Column(String, ForeignKey("sport.id"), nullable=False)
sport = relationship("Sport", back_populates="teams")
roosters = relationship("Rooster")
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.name = import_data['name']
self.short_name = import_data['short_name']
self.sport_id = import_data['sport_id']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['name'] = self.name
item['short_name'] = self.short_name
item['sport_id'] = self.sport_id
return item
class FieldPosition(Base, BaseMixin):
__tablename__ = "field_position"
__table_args__ = (
UniqueConstraint("name", "sport_id"),
UniqueConstraint("short_name", "sport_id"),
)
name = Column(String, nullable=False, index=True)
short_name = Column(String, nullable=False)
sport_id = Column(String, ForeignKey("sport.id"), nullable=False, index=True)
sport = relationship("Sport", back_populates="positions")
roosters = relationship("Rooster")
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.name = import_data['name']
self.short_name = import_data['short_name']
self.sport_id = import_data['sport_id']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['name'] = self.name
item['short_name'] = self.short_name
item['sport_id'] = self.sport_id
return item
class Player(Base, BaseMixin):
__tablename__ = "player"
__table_args__ = (
UniqueConstraint("first_name", "last_name"),
)
first_name = Column(String, nullable=False, index=True)
last_name = Column(String, nullable=False, index=True)
roosters = relationship("Rooster")
def get_full_name(self) -> str:
return f"{self.last_name}, {self.first_name}"
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.first_name = import_data['first_name']
self.last_name = import_data['last_name']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['first_name'] = self.first_name
item['last_name'] = self.last_name
return item
class Rooster(Base, BaseMixin):
__tablename__ = "rooster"
__table_args__ = (
UniqueConstraint("year", "team_id", "player_id", "position_id"),
)
year = Column(Integer)
team_id = Column(String, ForeignKey("team.id"), nullable=False, index=True)
team = relationship("Team", back_populates="roosters")
player_id = Column(String, ForeignKey("player.id"), nullable=False, index=True)
player = relationship("Player", back_populates="roosters")
position_id = Column(String, ForeignKey("field_position.id"), nullable=False, index=True)
position = relationship("FieldPosition", back_populates="roosters")
cards = relationship("Card")
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.year = import_data['year']
self.team_id = import_data['team_id']
self.player_id = import_data['player_id']
self.position_id = import_data['position_id']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['year'] = self.year
item['team_id'] = self.team_id
item['player_id'] = self.player_id
item['position_id'] = self.position_id
return item
class Vendor(Base, BaseMixin):
__tablename__ = "vendor"
name = Column(String, nullable=False, unique=True, index=True)
card_sets = relationship("CardSet")
cards = relationship("Card")
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.name = import_data['name']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['name'] = self.name
return item
class CardSet(Base, BaseMixin):
__tablename__ = "card_set"
__table_args__ = (
UniqueConstraint("name", "vendor_id"),
)
name = Column(String, index=True)
parallel_set = Column(Boolean)
insert_set = Column(Boolean)
vendor_id = Column(String, ForeignKey("vendor.id"), nullable=False, index=True)
vendor = relationship("Vendor", back_populates="card_sets")
cards = relationship("Card")
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.name = import_data['name']
self.parallel_set = import_data['parallel_set']
self.insert_set = import_data['insert_set']
self.vendor_id = import_data['vendor_id']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['name'] = self.name
item['parallel_set'] = self.parallel_set
item['insert_set'] = self.insert_set
item['vendor_id'] = self.vendor_id
return item
class Card(Base, BaseMixin):
__tablename__ = "card"
__table_args__ = (
UniqueConstraint("card_number", "year", "vendor_id", "card_set_id"),
)
card_number = Column(Integer, index=True)
year = Column(Integer, index=True)
card_set_id = Column(String, ForeignKey("card_set.id"), nullable=False)
card_set = relationship("CardSet", back_populates="cards")
rooster_id = Column(String, ForeignKey("rooster.id"), nullable=False)
rooster = relationship("Rooster", back_populates="cards")
vendor_id = Column(String, ForeignKey("vendor.id"), nullable=False)
vendor = relationship("Vendor", back_populates="cards")
def import_dict(self, import_data: Dict[str, Any]):
self.id = import_data['id']
self.created_date = import_data['created_date']
self.last_modified_date = import_data['last_modified_date']
self.version = import_data['version']
self.card_number = import_data['card_number']
self.year = import_data['year']
self.card_set_id = import_data['card_set_id']
self.rooster_id = import_data['rooster_id']
self.vendor_id = import_data['vendor_id']
def export_dict(self) -> Dict[str, Any]:
item: Dict[str, Any] = {}
item['id'] = self.id
item['created_date'] = str(self.created_date)
item['last_modified_date'] = str(self.last_modified_date)
item['version'] = self.version
item['card_number'] = self.card_number
item['year'] = self.year
item['card_set_id'] = self.card_set_id
item['rooster_id'] = self.rooster_id
item['vendor_id'] = self.vendor_id
return item
kontor-scripts/db/repository/__init__.py
View File
kontor-scripts/download.py
+97 -43
View File
@@ -1,122 +1,176 @@
"""
download files with URLs from DB
"""
import os
import re
import subprocess
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
from datetime import datetime
from enum import Enum, auto
from pathlib import Path
import sys
from typing import Any, Dict
from uuid import UUID
from platformdirs import PlatformDirs
import requests
from config import get_logger
import yaml
from config import get_api_config, get_logger
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--verbose', '-v', action='count', default=0)
parser.add_argument('--config', '-c', default='kontor-docker')
parser.add_argument('--dir', '-d', default='/data/media')
parser.add_argument('--tool', '-t', default='yt-dlp')
parser.add_argument('--dry-run', '-m', action='store_true')
parser.add_argument("--verbose", "-v", action="count", default=0)
parser.add_argument("--config", "-c", default="kontor-docker")
parser.add_argument("--dir", "-d", default="/data/media")
parser.add_argument("--limit", "-l", type=int, help="maximum number of links to check")
parser.add_argument("--tool", "-t", default="yt-dlp")
parser.add_argument("--dry-run", "-m", action="store_true")
args = parser.parse_args()
class FileStatus(Enum):
DOWNLOADED = auto()
RENAMED = auto()
UNKNOWN = auto()
def download_file(url: str, file_info: dict, download_dir: str = "/data/media", dl_tool: str = "yt-dlp") -> dict:
def download_file(
url: str,
file_info: dict,
download_dir: str = "/data/media",
dl_tool: str = "yt-dlp",
) -> dict:
print(f"download file for {url} to {download_dir}")
result = subprocess.run([dl_tool, url], cwd=download_dir, capture_output=True, text=True)
result = subprocess.run(
[dl_tool, url], cwd=download_dir, capture_output=True, text=True
)
if result.returncode == 0:
output = result.stdout
output = re.sub(' +', ' ', output)
output = re.sub(" +", " ", output)
lines_list = output.splitlines()
file_name = __parse_output__(lines_list)
if file_name is None:
file_info['review'] = True
file_info['should_download'] = True
file_info['file_name'] = None
log.info(f"found file: {file_name}")
if file_name is None or not file_name.strip():
file_info["review"] = True
file_info["should_download"] = True
file_info["file_name"] = None
else:
download_file_name = Path(download_dir, file_name)
file_info['should_download'] = False
file_info['file_name'] = download_file_name.name
file_info['cloud_link'] = str(download_file_name.absolute())
file_info['last_modified_date'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
file_info["should_download"] = False
file_info["review"] = False
file_info["file_name"] = download_file_name.name
file_info["cloud_link"] = str(download_file_name.absolute())
file_info["last_modified_date"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
return file_info
def __parse_output__(lines_list: list[str]) -> str | None:
file_name = None
for line in lines_list:
if 'has already been downloaded' in line:
end_len = len(' has already been downloaded')
log.debug(f"parse line: {line}")
if "has already been downloaded" in line:
end_len = len(" has already been downloaded")
file_name = line[11:-end_len]
if 'Destination' in line:
log.info(f"file_name: {file_name}")
break
if "Destination" in line:
line_len = len(line)
start_len = len('[download] Destination: ')
start_len = len("[download] Destination: ")
file_len = line_len - start_len
file_name = line[-file_len:]
break
else:
file_name = None
return file_name
def is_file_downloaded(media_file: dict, dir: Path) -> FileStatus:
file_name_as_title = f"{media_file['file_name']}"
if not file_name_as_title:
log.info("title has not been set - start download")
return FileStatus.UNKNOWN
file_title = Path(dir, f"{file_name_as_title}.mp4")
if file_title.exists():
log.info(f"{file_name_as_title} has been downloaded")
media_file['should_download'] = False
media_file["should_download"] = False
return FileStatus.DOWNLOADED
file_name_as_id = f"{media_file['id']}"
file_with_id_as_name = Path(dir, f"{file_name_as_id}.mp4")
if file_with_id_as_name.exists():
log.info(f"{file_with_id_as_name} has been downloaded and renamed")
media_file['cloud_link'] = file_with_id_as_name
media_file['should_download'] = False
media_file["cloud_link"] = str(file_with_id_as_name)
media_file["should_download"] = False
return FileStatus.RENAMED
log.info("could not find file - start download")
return FileStatus.UNKNOWN
def update_status(item_id: UUID, file_info: dict):
update = requests.put(f"http://127.0.0.1:8800/media/files/{item_id}", json=file_info)
def update_status(item_id: UUID, file_info: dict, api_data: Dict[str, Any]):
host = api_data["host"]
token = api_data['token']
url: str = f"http://{host}:{port}/api/media/files/{item_id}"
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
update = requests.put(url, headers=headers, json=file_info)
log.info(f"update status: {update.status_code}")
log.info(f"update result: {update.json()}")
def rename_file(file_info: dict):
item_id = file_info['id']
file = Path(args.dir, file_info['file_name'])
item_id = file_info["id"]
file_name = file_info["file_name"]
if file_name is None or not file_name.strip():
log.info("file_name is not set, rename is not executed")
file_info["review"] = True
file_info["should_download"] = True
return
file = Path(args.dir, file_name)
new_file_path = file.with_name(f"{item_id}{file.suffix}")
log.info(f"rename {file} to {new_file_path}")
file.rename(Path(new_file_path))
file_info['cloud_link'] = str(new_file_path)
file_info["cloud_link"] = str(new_file_path)
if __name__ == '__main__':
if __name__ == "__main__":
log = get_logger(args.verbose, args.config)
log.info('kontor.download started')
response = requests.get("http://127.0.0.1:8800/media/files?download=true")
log.info("kontor.download started")
api_data = get_api_config(log, args.config)
host = api_data["host"]
port = api_data["port"]
token = api_data['token']
url: str = f"http://{host}:{port}/api/media/files?download=true"
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
response = requests.get(url, headers=headers)
log.info(f"Status: {response.status_code}")
data = response.json()
log.info(f"data: {len(data)}")
entries_count = len(data)
log.info(f"data: {entries_count}")
mediafile_index = 1
log.debug(f"data: {data}")
missing_actors = {}
if args.dry_run:
sys.exit(0)
if args.limit:
log.warning(f"check the first {args.limit} links")
for item in data:
link = item['url']
file_id = item['id']
link = item["url"]
file_id = item["id"]
log.info(f"{file_id} - {link}")
download_status: FileStatus = is_file_downloaded(item, args.dir)
match download_status:
case FileStatus.DOWNLOADED:
rename_file(item)
update_status(file_id, item)
update_status(file_id, item, api_data)
case FileStatus.RENAMED:
log.info("update status")
update_status(file_id, item)
update_status(file_id, item, api_data)
case FileStatus.UNKNOWN:
download_file(link, item)
download_file(link, item, args.dir)
rename_file(item)
log.info(f'{item}')
update_status(file_id, item)
log.info('kontor.download finished')
log.info(f"{item}")
update_status(file_id, item, api_data)
log.warning(f"processed {mediafile_index}/{entries_count}")
if args.limit and args.limit <= mediafile_index:
break
mediafile_index += 1
log.info("kontor.download finished")
kontor-scripts/export.py
+35 -23
View File
@@ -2,17 +2,13 @@
import data from json file to MariaDB
"""
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
import yaml
import json
import os
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from platformdirs import PlatformDirs
from pathlib import Path
from schema.base import Base
from schema.database import KontorDB
from db.models import registry
from db.models.base import Base
from config import get_logger
from schema.database import ExportType
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--verbose', '-v', action='count', default=0)
@@ -20,24 +16,40 @@ parser.add_argument('--config', '-c', default='kontor-docker')
parser.add_argument('--file', '-f', default='data.json')
args = parser.parse_args()
DB_USER: str = os.getenv("DB_USER", "kontor")
DB_PASSWORD: str = os.getenv("DB_PASSWORD", "kontor")
DB_SERVER: str = os.getenv("DB_SERVER", "127.0.0.1")
DB_PORT: int = int(os.getenv("DB_PORT", 5432))
DB_DBNAME: str = os.getenv("DB_DBNAME", "kontor")
DATABASE_URL: str = f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_SERVER}:{DB_PORT}/{DB_DBNAME}"
if __name__ == '__main__':
logger = get_logger(args.verbose, args.config)
logger.info('kontor.export started')
dirs = PlatformDirs(args.config)
database_config = Path(dirs.user_config_dir, 'database-config.yaml')
with open(database_config, 'rt') as f:
db_config = yaml.safe_load(f.read())
connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format(
db_config['mariadb']['user'],
db_config['mariadb']['password'],
db_config['mariadb']['host'],
db_config['mariadb']['port'],
db_config['mariadb']['database']
))
engine = create_engine(connect_string)
engine = create_engine(DATABASE_URL)
Base.metadata.create_all(bind=engine, checkfirst=True)
__session__ = sessionmaker(bind=engine)
kontor_db = KontorDB(engine, logger)
kontor_db.export_db(ExportType.JSON, args.file)
SessionLocal = sessionmaker(bind=engine)
with SessionLocal() as db:
data = {}
tables = registry.keys()
for table in tables:
# logger.info(f"Table {table.name} with {table.id}")
model = registry[table]
rows = db.query(model).all()
entries = []
for row in rows:
entry = row.export_dict()
entries.append(entry)
data[table] = entries
logger.info(f"{table}: {len(entries)} exported")
try:
json_dump = json.dumps(data, indent=4)
with open(args.file, "w") as dump_file:
dump_file.write(json_dump)
except TypeError as error:
logger.info(f"{error}")
logger.info(f"{len(data)} tables exported")
#kontor_db = KontorDB(engine, logger)
#kontor_db.export_db(ExportType.JSON, args.file)
logger.info('kontor.export finished')
kontor-scripts/find_links.py
+332
View File
@@ -0,0 +1,332 @@
"""
download files with URLs from DB
"""
import logging.config
import sys
from typing import Any, Dict
import requests
import re
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from bs4 import BeautifulSoup
from config import get_api_config
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--verbose', '-v', action='count', default=0)
parser.add_argument("--config", "-c", default="kontor-docker")
parser.add_argument('--all', '-a', action='store_true')
parser.add_argument('--limit', '-l', type=int, help='maximum number of links to check')
parser.add_argument('--add-actor', action='store_true', help='add missing actors')
args = parser.parse_args()
def get_logger(level: int) -> logging.Logger:
logging.config.dictConfig({
'version': 1,
'disable_existing_loggers': False,
'formatters': {
'simple': {
'format': '[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s',
'datefmt': '%Y-%m-%d %H:%M:%S',
},
},
'handlers': {
'console': {
'class': logging.StreamHandler,
'level': logging.DEBUG,
'formatter': 'simple',
'stream': 'ext://sys.stdout'
},
},
'loggers': {
'urllib3.connectionpool': {
'level': 'WARNING',
'propagate': False,
},
'root': {
'level': 'DEBUG',
'handlers': ['console'],
},
},
})
logger = logging.getLogger(__file__)
if level is not None:
match level:
case 0:
logger.setLevel(logging.WARNING)
case 1:
logger.setLevel(logging.INFO)
case 2:
logger.setLevel(logging.DEBUG)
case _:
logger.setLevel(logging.CRITICAL)
return logger
def update_file(log: logging.Logger, media_file, api_data: Dict[str, Any]):
host = api_data["host"]
port = api_data["port"]
token = api_data['token']
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
url = f"http://{host}:{port}/api/media/files/{media_file['id']}"
update = requests.put(url, headers=headers, json=media_file)
log.debug(f"update status: {update.status_code}")
log.debug(f"update result: {update.json()}")
def get_actor_links(log: logging.Logger, media_file_url: str, api_data: Dict[str, Any]) -> list[str]:
try:
r = requests.get(media_file_url)
soup = BeautifulSoup(r.content, "html.parser")
error404 = soup.css.select_one('.error404-title')
if error404 and error404.get_text() == "Video nicht gefunden":
log.warning(f"{error404.get_text()}")
media_file['url'] = None
media_file['review'] = False
update_file(log, media_file, api_data=api_data)
return []
anchors = soup.find_all('a', attrs={'href': re.compile("^https://.*pornstars/.*")})
actor_links = []
for anchor in anchors:
link_url = str(anchor.get("href")) # type: ignore
if link_url.endswith('all/countries'):
continue
if link_url in actor_links:
continue
actor_links.append(link_url)
log.debug(f"links({len(actor_links)}): {actor_links}")
return actor_links
except Exception as error:
log.warning(f"something went wrong: {error}")
return []
def get_media_files(all_files: bool, api_data: Dict[str, Any])-> Any:
files_url = ""
host = api_data["host"]
port = api_data["port"]
token = api_data['token']
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
if all_files:
files_url= f"http://{host}:{port}/api/media/files"
else:
files_url = f"http://{host}:{port}/api/media/files?review=true"
response = requests.get(files_url, headers=headers)
log.debug(f"Status: {response.status_code}")
data = response.json()
return data
def update_media_file(item, log: logging.Logger, api_data: Dict[str, Any]) -> Any:
host = api_data["host"]
port = api_data["port"]
token = api_data['token']
url: str = f"http://{host}:{port}/api/media/files/{item['id']}"
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
update = requests.put(url, headers=headers, json=item)
log.debug(f"update status: {update.status_code}")
log.debug(f"update result: {update.json()}")
return update.json()
def update_media_file_actors(mediafile: dict,
actor_id_list: list[dict[str, str]],
actor_links: list[str],
map_ids_actor: dict[str, str],
log: logging.Logger,
api_data: Dict[str, Any]):
media_file_id = mediafile['id']
host = api_data["host"]
port = api_data["port"]
token = api_data['token']
url: str = f"http://{host}:{port}/api/media/files/{media_file_id}/actors"
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
actor_response = requests.put(url, headers=headers, json=actor_id_list)
files_actor_list = actor_response.json()
persisted_actor_links_count: int = len(files_actor_list)
found_actor_links_count: int = len(actor_links)
if persisted_actor_links_count < found_actor_links_count:
log.warning(f"{persisted_actor_links_count} links persisted, but {found_actor_links_count} links are available")
mediafile['review'] = True
elif persisted_actor_links_count > found_actor_links_count:
log.warning("more persisted links than found actors")
for file_actor in files_actor_list:
actor_id = file_actor['actor_id']
actor_url = map_ids_actor[actor_id]['url'] # type: ignore
log.debug(f"check if actor({actor_id}) with {actor_url} in list")
if actor_url not in actor_links:
log.info(f"actor not found in links, delete relation {file_actor['id']}")
delete_media_file_actor(file_actor['id'], log, api_data)
mediafile['review'] = True
else:
mediafile['review'] = False
log.debug(f"found {persisted_actor_links_count} actors")
log.debug(f"found actors: {files_actor_list}")
def delete_media_file_actor(media_actor_file_id: str, log: logging.Logger, api_data: Dict[str, Any]):
host = api_data["host"]
port = api_data["port"]
token = api_data['token']
url: str = f"http://{host}:{port}/api/media/actorfiles/{media_actor_file_id}"
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
delete_response = requests.delete(url, headers=headers)
if delete_response.status_code == 204:
log.info(f"actor file relation with id {media_actor_file_id} successfully deleted")
def get_actor_ids(link_list: list[str],
map_url_actor: dict[str, str],
map_ids_actor: dict[str, str],
map_path_actor: dict[str, str],
missing_actors: dict[str, int],
log: logging.Logger) -> list[dict[str, str]]:
found_actors: list[dict[str, str]] = []
for link in link_list:
actor = get_persisted_actor(link, map_url_actor, map_ids_actor, map_path_actor, log)
if actor:
found_actors.append(actor)
else:
if link in missing_actors:
count = missing_actors[link]
missing_actors[link] = count +1
else:
missing_actors.update({link: 1})
return found_actors
def get_persisted_actor(actor_url: str,
map_url_actor: dict[str, str],
map_ids_actor: dict[str, str],
map_path_actor: dict[str, str],
log: logging.Logger) -> dict[str, str] | None:
alternate_url_actor: dict[str, dict[str, str]] = {
'https://ge.xhamster2.com/pornstars/jean-yves-lecastel':
{'id': 'e354b866-717c-4a66-ad38-bc7c23d97e36', 'name': 'Jean-Yves Le Castel', 'url': 'https://ge.xhamster.com/pornstars/jean-yves-le-castel'},
'https://ge.xhamster.com/pornstars/jean-yves-lecastel':
{'id': 'e354b866-717c-4a66-ad38-bc7c23d97e36', 'name': 'Jean-Yves Le Castel', 'url': 'https://ge.xhamster.com/pornstars/jean-yves-le-castel'},
'https://ge.xhamster.com/pornstars/gracie-green':
{'id': 'cbec2e0d-869c-40f1-923f-21958d938d9f', 'name': 'Gracie May Green', 'url':'https://ge.xhamster.com/pornstars/gracie-may-green'},
'https://ge.xhamster.com/pornstars/thomas-hyka':
{'id': '1d814b45-ea98-4acc-88a2-227d3ed36959', 'name': 'Thomas Crown', 'url':'https://ge.xhamster.com/pornstars/thomas-crown'},
'https://ge.xhamster.com/pornstars/chloe-couture':
{'id': 'e22003a5-60a9-4d86-a1df-ae09ecbe5200', 'name': 'Chloe Cherry', 'url':'https://ge.xhamster.com/pornstars/chloe-cherry'},
'https://ge.xhamster.com/pornstars/dava-fox':
{'id': 'd913b778-4507-421b-88e0-9da73bb80a63', 'name': 'Dava Foxx', 'url':'https://ge.xhamster.com/pornstars/dava-foxx'},
'https://ge.xhamster.com/pornstars/john-dough':
{'id': 'a2ecd50f-09b2-4d31-9fcf-1a1438700f51', 'name': 'Jon Dough', 'url':'https://ge.xhamster.com/pornstars/jon-dough'},
'https://ge.xhamster.com/pornstars/erica-mori':
{'id': '5379dab9-63da-44ed-baf1-929d74ac60b1', 'name': 'Polly Yangs', 'url':'https://ge.xhamster.com/pornstars/polly-yangs'},
'https://ge.xhamster.com/pornstars/elnara-cat':
{'id': '543952d7-59a9-4492-a70f-e384b5f8eb57', 'name': 'Renata Fox', 'url':'https://ge.xhamster.com/pornstars/renata-fox'},
'https://ge.xhamster.com/pornstars/melissa-grand':
{'id': '5d025bea-4af6-4197-b38d-3b3afa9d30b9', 'name': 'Melissa Benz', 'url':'https://ge.xhamster.com/pornstars/melissa-benz'},
'https://ge.xhamster.com/pornstars/sindy-dollar':
{'id': 'fa97769c-9e53-4613-b3c3-4cc1a2672d4b', 'name': 'Cindy Dollar', 'url':'https://ge.xhamster.com/pornstars/cindy-dollar'},
} # type: ignore
if actor_url in map_url_actor:
actor_id: str = map_url_actor[actor_url]['id'] # type: ignore
log.debug(f"found actor with id: {actor_id}")
return map_ids_actor[actor_id] # type: ignore
path = actor_url.split('/')[-1]
if path in map_path_actor:
actor_id: str = map_path_actor[path]['id'] # type: ignore
log.debug(f"found actor with id: {actor_id} by path {path}")
return map_ids_actor[actor_id] # type: ignore
if actor_url in alternate_url_actor:
actor_id: str = alternate_url_actor[actor_url]['id']
log.info(f"found actor with id: {actor_id} by alternative {path}")
return alternate_url_actor[actor_url]
log.info(f"found actor {actor_url} missing")
return None
def get_actors(log: logging.Logger, api_data: Dict[str, Any]):
actors_url = {}
actors_id = {}
actors_path = {}
host = api_data["host"]
port = api_data["port"]
token = api_data['token']
url: str = f"http://{host}:{port}/api/media/actors"
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
response = requests.get(url, headers=headers)
data = response.json()
for media_actor in data:
actor_id = media_actor['id']
actor_name = media_actor['name']
actor_url = media_actor['url']
actor = {}
actor['id'] = actor_id
actor['name'] = actor_name
actor['url'] = actor_url
actors_url[actor_url] = actor
actors_id[actor_id] = actor
actors_path[actor_url.split('/')[-1]] = actor
log.debug(f'all actors: {actors_url}')
log.debug(f'all actors: {actors_path}')
return (actors_url, actors_id, actors_path)
def get_actor_name(actor_url: str, log: logging.Logger) -> str | None:
try:
r = requests.get(actor_url)
soup = BeautifulSoup(r.content, "html.parser")
titles = soup.find_all('h1')
for title in titles:
log.info(f"title: {title.get_text()}")
return title.get_text()
except Exception as error:
log.warning(f"something went wrong: {error}")
return None
def create_actor(actor_url: str, actor_name: str, log: logging.Logger, api_data: Dict[str, Any]):
new_actor = { 'name': actor_name, 'url': actor_url}
host = api_data["host"]
port = api_data["port"]
token = api_data['token']
url: str = f"http://{host}:{port}/api/media/actors"
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
actor_response = requests.post(url, headers=headers, json=new_actor)
log.warning(f"add status: {actor_response.status_code}")
if actor_response.status_code == 201:
actor_data = actor_response.json()
log.warning(f"Actor {actor_data} persisted")
else:
log.info(f"Actor with {actor_url} not persisted")
if __name__ == '__main__':
log = get_logger(args.verbose)
log.warning('kontor.find_links started')
log.debug('get all actors')
api_data = get_api_config(log, args.config)
host = api_data["host"]
port = api_data["port"]
token = api_data['token']
(actors_url, actors_id, actors_path) = get_actors(log, api_data=api_data)
data = get_media_files(args.all, api_data)
entries_count = len(data)
mediafile_index = 1
log.debug(f"data: {len(data)}")
missing_actors = {}
if args.limit:
log.warning(f"check the first {args.limit} links")
for media_file in data:
link = media_file['url']
media_file_id = media_file['id']
if not link:
continue
if str(link) == "None":
continue
log.warning(f"{media_file['id']} - {str(link)}")
actor_links: list[str] = get_actor_links(log, link, api_data=api_data)
actor_id_list = get_actor_ids(actor_links, actors_url, actors_id, actors_path, missing_actors, log)
update_media_file_actors(media_file, actor_id_list, actor_links, actors_id, log, api_data=api_data)
result = update_media_file(media_file, log, api_data=api_data)
log.warning(f"processed {mediafile_index}/{entries_count}")
if args.limit and args.limit <= mediafile_index:
break
mediafile_index += 1
for link in missing_actors:
log.info(f"{link}: {missing_actors[link]}")
actor_name = get_actor_name(link, log)
if actor_name and args.add_actor:
create_actor(link, actor_name, log, api_data=api_data)
log.info("Sort missing actors by occurence count:")
sorted_missing = dict(sorted(missing_actors.items(), key=lambda item: item[1]))
for key in sorted_missing:
log.info(f"{key} : {sorted_missing[key]}")
log.warning('kontor.find_links finished')
kontor-scripts/import.py
+160 -26
View File
@@ -1,42 +1,176 @@
"""
import data from json file to MariaDB
import data from json file to PostgreSQL
"""
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
import yaml
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from platformdirs import PlatformDirs
from pathlib import Path
from datetime import datetime, date
from logging import Logger
from typing import Any, Dict, List
import os
import json
from schema import Base, KontorDB
from config import get_logger
from pathlib import Path
import requests
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, Session
from db.models.base import Base
from db.models import registry
from psycopg2.errors import NotNullViolation
from config import get_api_config, get_logger
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--verbose', '-v', action='count', default=0)
parser.add_argument('--config', '-c', default='kontor-docker')
parser.add_argument('--recreate-db', action='store_true')
parser.add_argument("--config", "-c", default="kontor-docker")
parser.add_argument('--dry-run', '-m', action='store_true')
parser.add_argument('--cleanup', '-d', action='store_true')
parser.add_argument('--file', '-f', default='~/data.json')
args = parser.parse_args()
DB_USER: str = os.getenv("DB_USER", "kontor")
DB_PASSWORD: str = os.getenv("DB_PASSWORD", "kontor")
DB_SERVER: str = os.getenv("DB_SERVER", "127.0.0.1")
DB_PORT: int = int(os.getenv("DB_PORT", 5432))
DB_DBNAME: str = os.getenv("DB_DBNAME", "kontor")
DATABASE_URL: str = f"postgresql://{DB_USER}:{DB_PASSWORD}@{DB_SERVER}:{DB_PORT}/{DB_DBNAME}"
def cleanup_database(db: Session, log, dry_run: bool):
log.debug("cleanup_database")
# get tables from registry
for table in registry:
log.info(f"{table}")
model = registry[table]
entries = db.query(model).all()
for entry in entries:
if not dry_run:
db.delete(entry)
db.commit()
def load_data(filename: str, log) -> Dict[str, List[Dict[str, Any]]]:
log.debug("load_data")
import_file = Path(filename)
if not import_file.exists():
log.info(f"File {filename} does not exist. Do nothing.")
raise FileNotFoundError()
log.info("read json file")
with open(filename, 'r') as json_file:
json_load = json.load(json_file)
return json_load
def get_ids(items: List[Any]) -> List[str]:
result: List[str] = []
for item in items:
result.append(item.id)
return result
def update_item(db: Session, import_data: Dict[str, Any], item: Any, dry_run: bool, log):
for (key, value) in import_data.items():
existing_value = getattr(item, str(key))
update: bool = has_changed(existing_value, value, log)
#if key == 'published_on':
# log.info(f"{type(value)}:{value} != {type(existing_value)}:{existing_value} : {update}")
if update:
log.info(f"update {key}({existing_value}) with {value}")
if not dry_run:
if value == 'None':
setattr(item, str(key), None)
else:
setattr(item, str(key), value)
db.add(item)
db.commit()
def has_changed(existing_data: Any, import_data: str, log) -> bool:
if existing_data is None and import_data == 'None':
return False
if isinstance(existing_data, str):
return existing_data != import_data
if isinstance(existing_data, date):
if len(import_data) > 19:
import_date = datetime.strptime(import_data, "%Y-%m-%d %H:%M:%S.%f")
log.debug(f"{type(existing_data)}:{existing_data} == {import_date} : {existing_data != import_date}")
return existing_data != import_date
if len(import_data) > 10:
import_date = datetime.strptime(import_data, "%Y-%m-%d %H:%M:%S")
log.debug(f"{type(existing_data)}:{existing_data} == {import_date} : {existing_data != import_date}")
return existing_data != import_date
return existing_data.strftime("%Y-%m-%d") != import_data
return existing_data != import_data
def item_import(db: Session, import_data: Dict[str, Any], dry_run: bool, log):
log.info(f"import {import_data}")
if not dry_run:
log.debug(f"model: {repr(model)} {import_data}")
try:
new_item = model()
new_item.import_dict(import_data)
log.info(f"new item: {new_item}")
db.add(new_item)
db.commit()
except NotNullViolation as notnull:
log.info(f"import failed: {notnull} {import_data}")
except Exception as error:
log.info(f"import failed: {error}")
def item_delete(table_name: str, item_id: str, api_data: Dict[str, Any], log: Logger):
log.info(f"delete item {item_id} from {table_name}")
host = api_data["host"]
port = api_data["port"]
token = api_data['token']
url = ""
match table_name:
case "media_file":
url = f"http://{host}:{port}/api/media/files/{item_id}"
case "media_actor_file":
url = f"http://{host}:{port}/api/media/actorfiles/{item_id}"
case "media_actor":
url = f"http://{host}:{port}/api/media/actors/{item_id}"
headers: Dict[str, str] = {"Authorization": f"Bearer {token}"}
response = requests.delete(url, headers=headers)
log.debug(f"Status: {response.status_code}")
if __name__ == '__main__':
logger = get_logger(args.verbose, args.config)
logger = get_logger(args.verbose, "kontor")
logger.info('kontor.import started')
dirs = PlatformDirs(args.config)
database_config = Path(dirs.user_config_dir, 'database-config.yaml')
with open(database_config, 'rt') as f:
db_config = yaml.safe_load(f.read())
connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format(
db_config['mariadb']['user'],
db_config['mariadb']['password'],
db_config['mariadb']['host'],
db_config['mariadb']['port'],
db_config['mariadb']['database']
))
engine = create_engine(connect_string)
api_data = get_api_config(logger, args.config)
engine = create_engine(DATABASE_URL)
Base.metadata.create_all(bind=engine, checkfirst=True)
__session__ = sessionmaker(bind=engine)
kontor_db = KontorDB(engine, logger)
kontor_db.import_db(args.file)
SessionLocal = sessionmaker(bind=engine)
with SessionLocal() as db:
if args.cleanup:
cleanup_database(db, logger, args.dry_run)
data: Dict[str, List[Dict[str, Any]]] = load_data(args.file, logger)
table_list: List[str] = list(data.keys())
logger.debug(f"Liste der Tabellen: {table_list}")
sorted_table_list: List[str] = table_list
for tablename in sorted_table_list:
model = registry[tablename]
existing_items = db.query(model).all()
existing_ids: List[str] = get_ids(existing_items)
logger.debug(f"found {len(existing_items)} for table {tablename}")
import_items: List[Dict[str, Any]] = data[tablename]
for import_item in import_items:
item_id: str = import_item['id']
if item_id in existing_ids:
logger.debug(f"update {item_id}")
existing_item = db.get(model, item_id)
update_item(db, import_item, existing_item, args.dry_run, logger)
existing_ids.remove(item_id)
else:
logger.debug(f"import {item_id}")
item_import(db, import_item, args.dry_run, logger)
logger.debug(f"remaining items for {tablename}: {len(existing_ids)}")
if len(existing_ids) > 0:
logger.info(f"remaining items for {tablename}: {existing_ids}")
for item_id in existing_ids:
match tablename:
case "media_file":
item_delete(table_name=tablename, item_id=item_id, api_data=api_data, log=logger)
case "media_actor_file":
item_delete(table_name=tablename, item_id=item_id, api_data=api_data, log=logger)
case "media_actor":
item_delete(table_name=tablename, item_id=item_id, api_data=api_data, log=logger)
case _:
logger.info("Method to remove remaining item not implemented")
logger.info('kontor.import finished')
kontor-scripts/json_to_postgres.py
+152
View File
@@ -0,0 +1,152 @@
"""
copy data from JSON to Postgres
"""
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from pathlib import Path
from typing import Dict, List
from config import get_logger, get_database_cursors
import json
from psycopg2.sql import SQL
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--verbose', '-v', action='count', default=0)
parser.add_argument('--config', '-c', default='kontor-docker')
parser.add_argument('--file', '-f', default='~/.sync/media/data.json')
args = parser.parse_args()
def copy_data(postgres_conn, data_file: Path, log):
postgres_cursor = postgres_conn.cursor()
import_file = Path(data_file)
if not import_file.exists():
log.info(f"File {data_file} does not exist. Do nothing.")
return
log.info("read json file")
with open(data_file, 'r') as json_file:
json_load = json.load(json_file)
postgres_cursor.execute("SET session_replication_role='replica'")
for table in json_load:
log.info(f"{table}: {len(json_load[table])}")
# result[table] = import_table(table, json_load[table])
truncate_statement = 'TRUNCATE {} CASCADE'.format(table)
#log.info(f"truncate: {truncate_statement}")
try:
postgres_cursor.execute(truncate_statement)
except:
log.info(f"statement: {insert_statement} FAILED")
items = json_load[table]
for item in items:
#log.info(f"item: {item}")
values = []
columns = []
for (key, value) in item.items():
columns.append(key)
values.append(value)
row = tuple(values)
#log.info(f"values: {row}")
insert_statement = 'INSERT INTO {}({}) VALUES({})'.format(table, ', '.join(columns), ', '.join(['%s']*len(columns)))
#log.info(f"statement: {insert_statement}")
try:
postgres_cursor.execute(SQL(insert_statement), row)
postgres_conn.commit()
except:
log.info(f'insert failed with {insert_statement}')
postgres_cursor.execute("SET session_replication_role='origin'")
def load_json(data_file, log) -> dict:
import_file = Path(data_file)
if not import_file.exists():
log.info(f"File {data_file} does not exist. Do nothing.")
return
log.info("read json file")
with open(data_file, 'r') as json_file:
json_load = json.load(json_file)
return json_load
def insert_data(postgres_conn, data: dict, log):
postgres_cursor = postgres_conn.cursor()
log.info("insert data")
table_list = []
table_list = ['worktype', 'artist', 'publisher', 'volume', 'comic', 'issue', 'story_arc', 'trade_paperback', 'comic_work']
table_list.extend(['sport', 'team', 'field_position', 'vendor', 'player', 'rooster', 'card_set', 'card'])
#table_list.extend(['card'])
table_list.extend(['media_file', 'media_video', 'media_actor', 'media_article', 'media_actor_file'])
#table_list.extend(['media_actor_file'])
table_list.extend(['profile', 'permission', 'token', 'assignment'])
table_list.extend(['mail', 'mail_account', 'module_data', 'meta_data_table', 'meta_data_column'])
table_list.extend(['meta_data_table', 'meta_data_column'])
table_list.extend(['book', 'author', 'article', 'bookshelf_publisher', 'book_author', 'article_author'])
#if len(table_list) != 37:
# log.info(f"number of tables incorrect: {len(table_list)}")
# return
for table in table_list:
log.info(f"{table}: {len(data[table])}")
truncate_statement = 'DELETE FROM {}'.format(table)
log.info(f"truncate: {truncate_statement}")
try:
postgres_cursor.execute(truncate_statement)
postgres_conn.commit()
except:
log.info(f"statement: {truncate_statement} FAILED")
items = data[table]
for item in items:
# log.info(f"item: {item}")
values = []
columns = []
for (key, value) in item.items():
columns.append(key)
values.append(value)
row = tuple(values)
# log.info(f"values: {row}")
insert_statement = 'INSERT INTO {}({}) VALUES({})'.format(table, ', '.join(columns),
', '.join(['%s'] * len(columns)))
# log.info(f"statement: {insert_statement}")
try:
postgres_cursor.execute(SQL(insert_statement), row)
postgres_conn.commit()
except:
log.info(f'insert failed with {insert_statement}')
def parse_table_order(data: dict, log):
log.info("parse_table_order")
table_refs: Dict[str, List[str]] = {}
for table in data:
log.info(f"{table}: {len(data[table])}")
items = data[table]
table_refs[table] = []
if len(items) == 0:
continue
item = items[0]
for key, _ in item.items():
if key.endswith('_id'):
ref = key[0:-3]
log.info(f"table {table} has reference to {ref}")
if table in table_refs:
table_refs[table].append(ref)
else:
table_refs[table] = [ref]
log.info(f"parsed refs: {table_refs}")
table_order = []
for table in table_refs:
if len(table_refs[table]) == 0:
log.info(f"insert {table} at beginning")
table_order.insert(0, table)
else:
log.info(f"insert {table} at end")
table_order.append(table)
log.info(f"table_list: {len(table_order)}: {table_order}")
if __name__ == '__main__':
logger = get_logger(args.verbose, args.config)
logger.info('kontor.json_to_postgres started')
_, _, p_conn = get_database_cursors(logger, args.config)
data = load_json(args.file, logger)
#parse_table_order(data, logger)
insert_data(p_conn, data, logger)
#copy_data(p_conn, args.file, logger)
p_conn.close()
logger.info('kontor.json_to_postgres finished')
kontor-scripts/kontor.py
+64
View File
@@ -0,0 +1,64 @@
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from pathlib import Path
import logging.config
import yaml
from platformdirs import PlatformDirs
from sqlalchemy import create_engine, select
from sqlalchemy.orm import sessionmaker
from schema import Comic, Publisher, Base
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--recreate-db', action='store_true')
parser.add_argument('--verbose', '-v', action='count', default=0)
parser.add_argument('--file', '-f', default='~/data.json')
parser.add_argument('--config', '-c', default='kontor-docker')
args = parser.parse_args()
def get_logger(level: int, config: str):
dirs = PlatformDirs(config)
logging_config = Path(dirs.user_config_dir, 'logging-config.yaml')
with open(logging_config, 'rt') as f:
configDict = yaml.safe_load(f.read())
logging.config.dictConfig(configDict)
logger = logging.getLogger('development')
if level is not None:
match level:
case 0:
logger.setLevel(logging.INFO)
case 1:
logger.setLevel(logging.DEBUG)
case _:
logger.setLevel(logging.CRITICAL)
return logger
if __name__ == '__main__':
log = get_logger(args.verbose, args.config)
log.info('kontor started')
dirs = PlatformDirs(args.config)
database_config = Path(dirs.user_config_dir, 'database-config.yaml')
with open(database_config, 'rt') as f:
db_config = yaml.safe_load(f.read())
print(db_config)
connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format(
db_config['mariadb']['user'],
db_config['mariadb']['password'],
db_config['mariadb']['host'],
db_config['mariadb']['port'],
db_config['mariadb']['database']
))
engine = create_engine(connect_string)
Base.metadata.create_all(bind=engine, checkfirst=True)
__session__ = sessionmaker(engine)
with __session__() as session:
comics = session.scalars(select(Comic)).all()
for comic in comics:
print(comic)
publishers = session.scalars(select(Publisher)).all()
for publisher in publishers:
print(publisher)
print(len(publisher.comics))
log.info('kontor finished')
kontor-scripts/pyproject.toml
+7 -1
View File
@@ -1,4 +1,5 @@
[project]
requires-python = ">=3.13"
name = "kontor-scripts"
version = "0.1.0"
readme = "README.md"
@@ -10,12 +11,17 @@ maintainers = [
]
dependencies = [
"beautifulsoup4>=4.13.4",
"coverage>=7.8.0",
"fastapi[standard]>=0.115.12",
"mariadb>=1.1.12",
"pathlib>=1.0.1",
"platformdirs>=4.3.7",
"proton>=0.9.1",
"psycopg2-binary>=2.9.10",
"pytest-cov>=6.1.1",
"python-qpid-proton>=0.40.0",
"pyyaml>=6.0.2",
"requests>=2.32.3",
"sqlalchemy>=2.0.40",
"sqlmodel>=0.0.24",
"stomp.py",
]
kontor-scripts/read_list.py
+56 -33
View File
@@ -1,12 +1,12 @@
"""
read file with URLs and store in DB
"""
import uuid
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
import datetime
import mariadb
from setup import get_database_cursors, get_logger, get_scripts, get_meta_data
import logging
import json
from proton import Message, Event
from proton.handlers import MessagingHandler
from proton.reactor import Container
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('-f', '--links', help='file with links')
@@ -20,38 +20,61 @@ def read_links_file(links_file):
return lines
def add_link_to_db(statement, connection, video_url, log):
entry_id = str(uuid.uuid4())
current_date_time = datetime.datetime.now()
try:
cur = connection.cursor()
cur.execute(statement, (entry_id, current_date_time, current_date_time, 0, video_url, True, True, None, None, None, None))
connection.commit()
log.info(f'link {video_url} added to db')
except mariadb.Error as insert_error:
log.debug("insert failed with %s", insert_error)
entry_id = None
return entry_id
class AddLinkMessage(MessagingHandler):
def __init__(self, server, url, log):
super(AddLinkMessage, self).__init__()
log.info("create AddLinkMessage")
self.server = server
self.address = "KontorMediaFile::add_link_file"
self.url = url
self.log = log
def on_start(self, event: Event):
self.log.info("Connection...")
conn = event.container.connect(self.server, user="artemis", password="artemis")
event.container.create_sender(conn, self.address)
def on_connection_opened(self, event: Event) -> None:
self.log.debug("connection open")
def on_connection_error(self, event: Event) -> None:
self.log.info(f"error: {event}")
def on_disconnected(self, event: Event) -> None:
self.log.debug(f"disconnected: {repr(event)}")
def on_sendable(self, event: Event):
self.log.info("send message")
message = Message(body=self.url, address=self.address, content_type="application/json", durable=True)
delivery = event.sender.send(message)
self.log.info(f"Delivery {delivery} sent")
event.connection.close()
def on_accepted(self, event: Event) -> None:
self.log.info(f"accepted Delivery: {event.delivery.remote_state}")
def on_rejected(self, event: Event) -> None:
self.log.info(f"rejected Delivery: {event.delivery}")
if __name__ == '__main__':
logger = get_logger(args.verbose)
logger.info('kontor.read_list started')
s_conn, m_conn = get_database_cursors(logger)
meta_data_tables = get_meta_data(m_conn)
scripts = get_scripts(meta_data_tables, logger)
tables = {}
for table_id in scripts:
tables[scripts[table_id]['name']] = table_id
media_file_id = tables['media_file']
insert_statement = scripts[tables['media_file']]['insert_mariadb']
logging.basicConfig(level=logging.INFO, format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s')
logging.info('kontor.read_list started')
#conn = stomp.Connection([('127.0.0.1', '61616')])
#conn.connect('artemis', 'artemis', wait=True)
if args.links:
logger.info("read links from file")
logging.info("read links from file")
links = read_links_file(args.links)
for link in links:
logger.info("add link to db")
add_link_to_db(insert_statement, m_conn, link.strip(), logger)
else:
logger.info('script used: {}'.format(insert_statement))
logger.info('kontor.read_list finished')
data_dict = {'url': link.strip()}
data = json.dumps(data_dict)
logging.info("send link message")
handler = AddLinkMessage("amqp://127.0.0.1:5672", data, logging)
container = Container(handler)
container.container_id = "process_add_links"
container.run()
# conn.send(body=data, destination='KontorMediaFile::add_link_file', headers={'content-type': 'application/json'})
#conn.disconnect()
logging.info('kontor.read_list finished')
kontor-scripts/read_queue.py
+37
View File
@@ -0,0 +1,37 @@
import stomp
import json
import time
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from config import get_logger
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--verbose', '-v', action='count', default=0)
parser.add_argument('--config', '-c', default='kontor-docker')
args = parser.parse_args()
class MyListener(stomp.ConnectionListener):
def __init__(self, log):
self.log = log
def on_error(self, frame):
self.log.info(f"received an error {frame.body}")
def on_message(self, frame):
self.log.info(f"received a message '{frame.body}'")
data = json.loads(frame.body)
url = data['url']
self.log.info(f"found link: {url}")
if __name__ == '__main__':
log = get_logger(args.verbose, args.config)
log.info("kontor.read_queue started")
host = [('127.0.0.1', 61616)]
conn = stomp.Connection(host_and_ports=host)
conn.set_listener('', MyListener(log))
conn.connect(username='artemis', passcode='artemis', wait=True)
conn.subscribe(destination='KontorMediaFile::add_link_file', id=1, ack='auto', headers={})
time.sleep(5)
conn.disconnect()
log.info("kontor.read_queue finished")
kontor-scripts/requirements.txt
-1
View File
@@ -1,4 +1,3 @@
mariadb
sqlalchemy
pathlib
platformdirs
kontor-scripts/schema/admin.py
-78
View File
@@ -1,78 +0,0 @@
from datetime import datetime
from sqlalchemy import Column, ForeignKey, Integer, String
from sqlalchemy.dialects.mysql import BIT
from sqlalchemy.orm import relationship, mapped_column, Mapped
from .base import Base, BaseMixin
class Profile(Base, BaseMixin):
__tablename__ = 'profile'
first_name = Column(String(255))
last_name = Column(String(255))
user_name = Column(String(255), nullable=False)
email = Column(String(255))
password = Column(String(255))
enabled = Column(BIT(1))
assignments = relationship("Assignment")
tokens = relationship("Token")
def get_full_name(self) -> str:
full_name = ""
if self.first_name is not None:
full_name += self.first_name
if self.last_name is not None:
if len(full_name) > 0:
full_name += " "
full_name += self.last_name
return full_name
class Token(Base, BaseMixin):
__tablename__ = "token"
token = Column(String(255), nullable=False, unique=True)
name = Column(String(255))
last_used_date: Mapped[datetime] = mapped_column()
enabled = Column(BIT(1))
profile_id = Column(String(255), ForeignKey("profile.id"), nullable=False)
profile = relationship("Profile", back_populates="tokens")
class Permission(Base, BaseMixin):
__tablename__ = "permission"
name = Column(String(255), nullable=False)
assignments = relationship("Assignment")
class Assignment(Base, BaseMixin):
__tablename__ = "assignment"
profile_id = Column(String, ForeignKey("profile.id"), nullable=False)
profile = relationship("Profile", back_populates="assignments")
permission_id = Column(String, ForeignKey("permission.id"), nullable=False)
permission = relationship("Permission", back_populates="assignments")
class ModuleData(Base, BaseMixin):
__tablename__ = "module_data"
module_name = Column(String(255), nullable=False)
import_data = Column(BIT(1))
class MailAccount(Base, BaseMixin):
__tablename__ = "mail_account"
host = Column(String(255))
port = Column(Integer)
protocol = Column(String(255))
user_name = Column(String(255))
password = Column(String(255))
start_tls = Column(BIT(1))
class Mail(Base, BaseMixin):
__tablename__ = "mail"
folder: Mapped[str] = mapped_column()
subject: Mapped[str] = mapped_column()
body: Mapped[str] = mapped_column()
sent_date: Mapped[datetime] = mapped_column()
received_date: Mapped[datetime] = mapped_column()
kontor-scripts/schema/bookshelf.py
-50
View File
@@ -1,50 +0,0 @@
from sqlalchemy import Column, ForeignKey, Integer, String
from sqlalchemy.orm import relationship
from .base import Base, BaseMixin
class Article(Base, BaseMixin):
__tablename__ = 'article'
title = Column(String(length=255), unique=True)
article_authors = relationship("ArticleAuthor")
class Author(Base, BaseMixin):
__tablename__ = 'author'
first_name = Column(String(255))
last_name = Column(String(255))
article_authors = relationship("ArticleAuthor")
book_authors = relationship("BookAuthor")
class BookshelfPublisher(Base, BaseMixin):
__tablename__ = 'bookshelf_publisher'
name = Column(String(length=255), unique=True)
books = relationship("Book")
class Book(Base, BaseMixin):
__tablename__ = 'book'
isbn = Column(String(255), unique=True)
title = Column(String(255))
year = Column(Integer, nullable=False)
publisher_id = Column(String, ForeignKey('bookshelf_publisher.id'), nullable=False)
publisher = relationship('BookshelfPublisher', back_populates="books")
book_authors = relationship("BookAuthor")
class ArticleAuthor(Base, BaseMixin):
__tablename__ = 'article_author'
article_id = Column(String, ForeignKey('article.id'), nullable=False)
article = relationship('Article', back_populates="article_authors")
author_id = Column(String, ForeignKey('author.id'), nullable=False)
author = relationship('Author', back_populates="article_authors")
class BookAuthor(Base, BaseMixin):
__tablename__ = 'book_author'
author_id = Column(String, ForeignKey('author.id'), nullable=False)
author = relationship('Author', back_populates="book_authors")
book_id = Column(String, ForeignKey('book.id'), nullable=False)
book = relationship('Book', back_populates="book_authors")
kontor-scripts/schema/comic.py
-100
View File
@@ -1,100 +0,0 @@
from sqlalchemy import Column, ForeignKey, Integer, String
from sqlalchemy.dialects.mysql import BIT
from sqlalchemy.orm import relationship
from .base import Base, BaseMixin
class Publisher(Base, BaseMixin):
__tablename__ = "publisher"
name = Column(String(length=255), unique=True)
comics = relationship("Comic")
def __repr__(self):
return f'Publisher({self.id} {self.name})'
def __str__(self):
return self.__repr__()
class Comic(Base, BaseMixin):
__tablename__ = 'comic'
title = Column(String(length=255), unique=True)
publisher_id = Column(String, ForeignKey('publisher.id'), nullable=False)
publisher = relationship("Publisher", back_populates="comics")
current_order = Column(BIT(1))
completed = Column(BIT(1))
issues = relationship("Issue")
story_arcs = relationship("StoryArc")
trade_paperbacks = relationship("TradePaperback")
volumes = relationship("Volume")
comic_works = relationship("ComicWork")
def __repr__(self):
return f'Comic({self.id} {self.version} {self.title} {self.publisher.name})'
def __str__(self):
return f'{self.title}({self.id})'
class Volume(Base, BaseMixin):
__tablename__ = "volume"
name = Column(String(length=255), nullable=False)
comic_id = Column(String, ForeignKey("comic.id"), nullable=False)
comic = relationship("Comic", back_populates="volumes")
issues = relationship("Issue")
class TradePaperback(Base, BaseMixin):
__tablename__ = "trade_paperback"
name = Column(String(length=255), nullable=False)
issue_start = Column(Integer)
issue_end = Column(Integer)
comic_id = Column(String, ForeignKey("comic.id"), nullable=False)
comic = relationship("Comic", back_populates="trade_paperbacks")
class StoryArc(Base, BaseMixin):
__tablename__ = "story_arc"
name = Column(String(length=255), nullable=False)
comic_id = Column(String, ForeignKey("comic.id"), nullable=False)
comic = relationship("Comic", back_populates="story_arcs")
class Issue(Base, BaseMixin):
__tablename__ = "issue"
issue_number = Column(String(255))
in_stock = Column(BIT(1))
is_read = Column(BIT(1))
comic_id = Column(String, ForeignKey("comic.id"), nullable=False)
comic = relationship("Comic", back_populates="issues")
volume_id = Column(String, ForeignKey("volume.id"), nullable=True)
volume = relationship("Volume", back_populates="issues")
class Artist(Base, BaseMixin):
__tablename__ = "artist"
name = Column(String(length=255), nullable=False)
comic_works = relationship("ComicWork")
class WorkType(Base, BaseMixin):
__tablename__ = "worktype"
name = Column(String(length=255), nullable=False, unique=True)
comic_works = relationship("ComicWork")
def __repr__(self):
return f'Worktype({self.id} {self.version} {self.name} {len(self.comic_works)})'
def __str__(self):
return f'{self.name}({self.id})'
class ComicWork(Base, BaseMixin):
__tablename__ = "comic_work"
comic_id = Column(String, ForeignKey("comic.id"), nullable=False)
comic = relationship("Comic", back_populates="comic_works")
artist_id = Column(String, ForeignKey("artist.id"), nullable=False)
artist = relationship("Artist", back_populates="comic_works")
work_type_id = Column(String, ForeignKey("worktype.id"), nullable=False)
work_type = relationship("WorkType", back_populates="comic_works")
kontor-scripts/schema/database.py
-399
View File
@@ -1,399 +0,0 @@
import json
import uuid
from datetime import datetime
from enum import Enum, auto
from logging import Logger
from pathlib import Path
from typing import Any
from sqlalchemy import UUID, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import sessionmaker
from .tysc import Card, CardSet, Rooster, Team, FieldPosition, Player, Vendor, Sport
from .comic import Issue, TradePaperback, StoryArc, Volume, ComicWork, Artist, Comic, Publisher, WorkType
from .bookshelf import ArticleAuthor, BookAuthor, BookshelfPublisher, Article, Book, Author
from .admin import Mail, MailAccount, ModuleData, Permission, Profile, Token, Assignment
from .metadata import MetaDataTable, MetaDataColumn
from .media import MediaVideo, MediaArticle, MediaFile, MediaActor, MediaActorFile
class ColumnEntry(Enum):
COLUMN_NAME = 'column'
COLUMN_LABEL = 'label'
COLUMN_ORDER = 'order'
COLUMN_REF_COLUMN = 'ref_column'
COLUMN_TYPE = 'type'
COLUMN_WIDGET = 'widget'
class StatusType(Enum):
UNKNOWN = auto()
FILE_NAME = auto()
FILE_ID = auto()
DUPLICATE = auto()
CLOUD_LINK = auto()
CLOUD_LINK_ID = auto()
class ExportType(Enum):
JSON = "JSON"
YAML = "YAML"
SQLITE = "SQLite"
class KontorDB:
def __init__(self, db_engine: Any, log: Logger):
self.engine = db_engine
self.registry = {}
self.init_registry()
self.log = log
def init_registry(self):
self.registry[Card.__tablename__] = Card
self.registry[CardSet.__tablename__] = CardSet
self.registry[Rooster.__tablename__] = Rooster
self.registry[Team.__tablename__] = Team
self.registry[FieldPosition.__tablename__] = FieldPosition
self.registry[Player.__tablename__] = Player
self.registry[Vendor.__tablename__] = Vendor
self.registry[Sport.__tablename__] = Sport
self.registry[Issue.__tablename__] = Issue
self.registry[TradePaperback.__tablename__] = TradePaperback
self.registry[StoryArc.__tablename__] = StoryArc
self.registry[Volume.__tablename__] = Volume
self.registry[ComicWork.__tablename__] = ComicWork
self.registry[Artist.__tablename__] = Artist
self.registry[Comic.__tablename__] = Comic
self.registry[Publisher.__tablename__] = Publisher
self.registry[WorkType.__tablename__] = WorkType
self.registry[ArticleAuthor.__tablename__] = ArticleAuthor
self.registry[BookAuthor.__tablename__] = BookAuthor
self.registry[BookshelfPublisher.__tablename__] = BookshelfPublisher
self.registry[Article.__tablename__] = Article
self.registry[Book.__tablename__] = Book
self.registry[Author.__tablename__] = Author
self.registry[MediaFile.__tablename__] = MediaFile
self.registry[MediaActor.__tablename__] = MediaActor
self.registry[MediaActorFile.__tablename__] = MediaActorFile
self.registry[MediaArticle.__tablename__] = MediaArticle
self.registry[MediaVideo.__tablename__] = MediaVideo
self.registry[MetaDataColumn.__tablename__] = MetaDataColumn
self.registry[MetaDataTable.__tablename__] = MetaDataTable
self.registry[Assignment.__tablename__] = Assignment
self.registry[Token.__tablename__] = Token
self.registry[Profile.__tablename__] = Profile
self.registry[Permission.__tablename__] = Permission
self.registry[ModuleData.__tablename__] = ModuleData
self.registry[MailAccount.__tablename__] = MailAccount
self.registry[Mail.__tablename__] = Mail
def get_table_names(self) -> list:
result = []
__session__ = sessionmaker(self.engine)
with __session__() as session:
tables = session.scalars(select(MetaDataTable)).all()
result = [table.table_name for table in tables]
return result
def get_table_by_name(self, table_name: str) -> dict:
result = {}
__session__ = sessionmaker(self.engine)
_filter = {'table_name': table_name}
with __session__() as session:
table = session.query(MetaDataTable).filter_by(**_filter).one()
result['id'] = table.id
result['table_name'] = table.table_name
return result
def get_column_meta_data(self, table_name: str, view_only=True) -> dict:
meta_data = {}
order = 0
__session__ = sessionmaker(self.engine)
columns = list()
table_info = self.get_table_by_name(table_name)
_filters = {'table_id': table_info['id']}
if view_only:
_filters['is_shown'] = True
with __session__() as session:
columns = session.query(MetaDataColumn).filter_by(**_filters).all()
for column in columns:
# self.log.info("get_column_meta_data: %s %s %d", column.column_name, column.column_label, column.column_order)
meta_data[order] = {
ColumnEntry.COLUMN_NAME: column.column_name,
ColumnEntry.COLUMN_LABEL: column.column_label,
ColumnEntry.COLUMN_ORDER: column.column_order,
ColumnEntry.COLUMN_REF_COLUMN: column.ref_column,
ColumnEntry.COLUMN_TYPE: column.column_type
}
order += 1
return meta_data
def get_columns(self, table_name: str) -> dict:
columns = {}
__session__ = sessionmaker(self.engine)
table_info = self.get_table_by_name(table_name)
_filters = {'table_id': table_info['id']}
with __session__() as session:
for column in session.query(MetaDataColumn).filter_by(**_filters).all():
columns[column.column_name] = {
ColumnEntry.COLUMN_ORDER: column.column_order,
ColumnEntry.COLUMN_TYPE: column.column_type
}
return columns
def get_filters(self, table_name: str) -> dict:
_filter_map = {}
__session__ = sessionmaker(self.engine)
table_info = self.get_table_by_name(table_name)
_filters = {'table_id': table_info['id'], 'show_filter': True}
with __session__() as session:
for column in session.query(MetaDataColumn).filter_by(**_filters).all():
_filter_map[column.column_name] = {
ColumnEntry.COLUMN_LABEL: column.filter_label,
ColumnEntry.COLUMN_WIDGET: None
}
return _filter_map
def data(self, table_name: str, columns: dict, filters: dict) -> list:
data = []
__session__ = sessionmaker(self.engine)
table = self.registry[table_name]
with __session__() as session:
entries = []
if len(filters) == 0:
entries = session.scalars(select(table)).all()
else:
entries = session.scalars(select(table).filter_by(**filters)).all()
for entry in entries:
# self.log.info("data: %s", entry)
row = []
for order in columns.keys():
column_name = columns[order][ColumnEntry.COLUMN_NAME]
ref_column = columns[order][ColumnEntry.COLUMN_REF_COLUMN]
if str(column_name).endswith("_id"):
ref_table = column_name[:-3]
ref = getattr(entry, ref_table)
value = getattr(ref, ref_column)
row.append(value)
else:
row.append(getattr(entry, column_name))
data.append(row)
# self.log.info("data: %s", data)
return data
def export_db(self, export_type: ExportType, export_file_name: str) -> dict:
results = {}
db = {}
export_table_list = self.get_table_names()
for table in export_table_list:
columns = self.get_column_meta_data(table, view_only=False)
if table in self.registry:
model = self.registry[table]
else:
self.log.info(f"table {table} is not registered")
continue
__session__ = sessionmaker(self.engine)
with __session__() as session:
rows = session.query(model).all()
entries = []
for row in rows:
# print(row)
entry = {}
for order in columns:
# print(columns[order])
column_name = columns[order][ColumnEntry.COLUMN_NAME]
# print(f"get value {column_name} from {row} of table {table}")
try:
value = getattr(row, column_name)
if isinstance(value, datetime):
entry[column_name] = str(value)
else:
entry[column_name] = value
except AttributeError:
pass
entries.append(entry)
db[table] = entries
results[table] = len(entries)
match export_type:
case ExportType.JSON:
json_dump = json.dumps(db, indent=4)
with open(export_file_name, "w") as dump_file:
dump_file.write(json_dump)
case ExportType.YAML:
pass
case ExportType.SQLITE:
pass
self.log.info(f"{len(results)} tables exported")
return results
def import_db(self, import_file_name: str) -> dict:
result = {}
import_file = Path(import_file_name)
if not import_file.exists():
self.log.info(f"File {import_file_name} does not exist. Do nothing.")
return result
match import_file.suffix:
case '.json':
print("read json file")
with open(import_file_name, 'r') as json_file:
json_load = json.load(json_file)
for table in json_load:
self.log.info(f"{table}: {len(json_load[table])}")
result[table] = self.import_table(table, json_load[table])
case '.yml':
print("read yaml file")
case '.yaml':
print("read yaml file")
case '.db':
print("read sqlite file")
return result
def import_table(self, table_name: str, items: list) -> dict:
result = {}
updated = []
added = []
remaining = []
existing_ids = self.get_ids(table_name)
self.log.info(f"found {len(existing_ids)} existing ids for table {table_name}")
for item in items:
current_id = item['id']
# print(f"import item: {item}")
found_item = None
__session__ = sessionmaker(self.engine)
with __session__() as session:
found_item = session.get(self.registry[table_name], current_id)
# print(f"found item: {found_item}")
if found_item is not None:
changed = self.update_entry(table_name, current_id, item)
updated.append(item)
if changed:
self.log.info(f"{current_id} has changed")
updated.append(item)
existing_ids.remove(current_id)
else:
try:
self.add_entry(table_name, item)
added.append(item)
except IntegrityError as error:
self.log.info(f"Could not add item, due to: {error.detail}")
if len(existing_ids) > 0:
print(f"remaining items for {table_name}: {existing_ids}")
remaining.extend(existing_ids)
result['updated'] = updated
result['added'] = added
result['remaining'] = remaining
return result
def get_ids(self, table_name: str) -> list:
existing_ids = []
__session__ = sessionmaker(self.engine)
with __session__() as session:
items = session.query(self.registry[table_name]).all()
for item in items:
existing_ids.append(getattr(item, 'id'))
return existing_ids
def add_entry(self, table_name: str, update_item: dict):
self.log.debug(f"add entry to table {table_name} with {update_item}")
__session__ = sessionmaker(self.engine)
with __session__() as session:
add_item = self.registry[table_name]()
for key in update_item.keys():
update_value = update_item[key]
setattr(add_item, key, update_value)
session.add(add_item)
session.commit()
def update_entry(self, table_name, current_id, update_item: dict) -> bool:
# self.log.info("update entry to table %s", table_name)
__session__ = sessionmaker(self.engine)
with __session__() as session:
existing_item = session.query(self.registry[table_name]).get(current_id)
changed = False
for key in update_item.keys():
update_value = update_item[key]
existing_value = getattr(existing_item, key)
if type(existing_value) is not type(update_value):
existing_value = str(existing_value)
if existing_value != update_value:
self.log.info(f"{key} has changed: {existing_value} != {update_value}")
setattr(existing_item, key, update_value)
session.commit()
changed = True
self.log.info(f"update {key} with {update_value}")
return changed
def add_link(self, link: str) -> dict:
result = {}
__session__ = sessionmaker(self.engine)
with __session__() as session:
media_file = MediaFile()
media_file.id = str(uuid.uuid4())
media_file.created_date = datetime.now()
media_file.last_modified_date = datetime.now()
media_file.version = 0
media_file.url = link
media_file.review = 1
media_file.should_download = 1
try:
session.add(media_file)
session.commit()
result['added'] = {'url': media_file.url, 'title': media_file.title, 'review': media_file.review,
'download': media_file.should_download}
except IntegrityError as error:
session.rollback()
result['error'] = error.orig
return result
def update_titles(self) -> dict:
update_list = {}
__session__ = sessionmaker(self.engine)
_filter = {'review': True}
with __session__() as session:
links = session.query(MediaFile).filter_by(**_filter).all()
self.log.info("%d entries found for updating titles", len(links))
for link in links:
url = link.url
if url is None:
continue
link.update_title()
session.commit()
update_list[link.id] = link.title
return update_list
def get_download_list(self) -> list[UUID]:
download_list = []
__session__ = sessionmaker(self.engine)
_filter = {'should_download': True}
with __session__() as session:
links = session.query(MediaFile).filter_by(**_filter).all()
for link in links:
url = link.url
if url is None:
continue
download_list.append(link.id)
return download_list
def download_file(self, entry_id: str, download_dir="/data/media", dl_tool="yt-dlp") -> str:
__session__ = sessionmaker(self.engine)
with __session__() as session:
link = session.query(MediaFile).get(entry_id)
link.download_file(download_dir, dl_tool)
session.commit()
file_name = link.file_name
return file_name
def delete_entries(self):
for (table_name, table) in self.registry.items():
# self.log.info("delete entries from table %s", table_name)
__session__ = sessionmaker(self.engine)
with __session__() as session:
items = session.query(table).all()
for item in items:
session.delete(item)
session.commit()
def check_files(self):
pass
kontor-scripts/schema/media.py
-99
View File
@@ -1,99 +0,0 @@
import re
import subprocess
from datetime import datetime
from pathlib import Path
import requests
from bs4 import BeautifulSoup
from sqlalchemy import Boolean, Column, False_, String, ForeignKey
from sqlalchemy.orm import relationship
from .base import Base, BaseMixin, BaseVideoMixin
class MediaFile(Base, BaseMixin, BaseVideoMixin):
__tablename__ = 'media_file'
media_actor_files = relationship("MediaActorFile")
def __repr__(self):
return f'MediaFile({self.id} {self.title} {self.title})'
def __str__(self):
return f'{self.title}({self.id})'
def update_title(self) -> None:
print(f"update title for {self.url}")
try:
r = requests.get(self.url)
soup = BeautifulSoup(r.content, "html.parser")
title = soup.title.string
self.title = title
self.review = False_
except:
self.title = None
self.review = True
self.last_modified_date = datetime.now()
def download_file(self, download_dir: str, dl_tool: str):
print(f"download file for {self.url} to {download_dir}")
result = subprocess.run([dl_tool, self.url], cwd=download_dir, capture_output=True, text=True)
if result.returncode == 0:
output = result.stdout
output = re.sub(' +', ' ', output)
lines_list = output.splitlines()
file_name = self.__parse_output__(lines_list)
if file_name is None:
self.review = True
self.should_download = True
self.file_name = None
else:
download_file = Path(file_name)
self.should_download = False_
self.file_name = download_file.name
self.cloud_link = str(download_file.absolute())
self.last_modified_date = datetime.now()
def __parse_output__(self, lines_list):
self.file_name = None
for line in lines_list:
if 'has already been downloaded' in line:
end_len = len(' has already been downloaded')
self.file_name = line[11:-end_len]
if 'Destination' in line:
line_len = len(line)
start_len = len('[download] Destination: ')
file_len = line_len - start_len
self.file_name = line[-file_len:]
return self.file_name
class MediaActor(Base, BaseMixin):
__tablename__ = 'media_actor'
name = Column(String(255))
media_actor_files = relationship("MediaActorFile")
class MediaActorFile(Base, BaseMixin):
__tablename__ = 'media_actor_file'
media_actor_id = Column(String(255), ForeignKey("media_actor.id"), nullable=False)
media_actor = relationship("MediaActor", back_populates="media_actor_files")
media_file_id = Column(String(255), ForeignKey("media_file.id"), nullable=True)
media_file = relationship("MediaFile", back_populates="media_actor_files")
class MediaArticle(Base, BaseMixin):
__tablename__ = 'media_article'
review = Column(Boolean)
title = Column(String(255))
url = Column(String(255), unique=True)
class MediaVideo(Base, BaseMixin):
__tablename__ = 'media_video'
cloud_link = Column(String(255))
file_name = Column(String(255))
path = Column(String(255))
review = Column(Boolean)
title = Column(String(255))
url = Column(String(255), unique=True)
should_download = Column(Boolean)
kontor-scripts/schema/metadata.py
-42
View File
@@ -1,42 +0,0 @@
from sqlalchemy import Column, String, ForeignKey, Integer
from sqlalchemy.dialects.mysql import BIT
from sqlalchemy.orm import relationship
from .base import Base, BaseMixin
class MetaDataTable(Base, BaseMixin):
__tablename__ = 'meta_data_table'
table_name = Column(String(255), unique=True)
table_columns = relationship("MetaDataColumn")
def __repr__(self):
return f'MetaDataTable({self.id} {self.table_name})'
def __str__(self):
return f'{self.table_name}({self.id})'
class MetaDataColumn(Base, BaseMixin):
__tablename__ = 'meta_data_column'
column_name = Column(String(255), nullable=False)
column_sync_name = Column(String(255))
column_type = Column(String(255))
column_modifier = Column(String(255), nullable=True)
column_order = Column(Integer)
table_id = Column(String, ForeignKey('meta_data_table.id'))
table = relationship("MetaDataTable", back_populates="table_columns")
column_label = Column(String(255))
filter_label = Column(String(255))
is_shown = Column(BIT(1))
show_filter = Column(BIT(1))
ref_column = Column(String, nullable=True)
def __repr__(self):
if self.column_name is None:
return f'MetaDataColumn({self.id} {self.table.table_name}.__)'
else:
return f'MetaDataColumn({self.id} {self.table.table_name}.{self.column_name})'
def __str__(self):
return f'{self.column_name}({self.id})'
kontor-scripts/schema/tysc.py
-100
View File
@@ -1,100 +0,0 @@
from sqlalchemy import Column, Integer, String, ForeignKey, UniqueConstraint
from sqlalchemy.dialects.mysql import BIT
from sqlalchemy.orm import relationship
from .base import Base, BaseMixin
class Sport(Base, BaseMixin):
__tablename__ = "sport"
__table_args__ = (
UniqueConstraint("name"),
)
name = Column(String(255), nullable=False, index=True, unique=True)
teams = relationship("Team")
positions = relationship("FieldPosition")
class Team(Base, BaseMixin):
__tablename__ = "team"
name = Column(String(255), nullable=False, index=True, unique=True)
short_name = Column(String(255), nullable=False, )
sport_id = Column(String, ForeignKey("sport.id"), nullable=False)
sport = relationship("Sport", back_populates="teams")
roosters = relationship("Rooster")
class FieldPosition(Base, BaseMixin):
__tablename__ = "field_position"
__table_args__ = (
UniqueConstraint("name", "sport_id"),
UniqueConstraint("short_name", "sport_id"),
)
name = Column(String(255), nullable=False, index=True)
short_name = Column(String(255), nullable=False)
sport_id = Column(String, ForeignKey("sport.id"), nullable=False, index=True)
sport = relationship("Sport", back_populates="positions")
roosters = relationship("Rooster")
class Player(Base, BaseMixin):
__tablename__ = "player"
__table_args__ = (
UniqueConstraint("first_name", "last_name"),
)
first_name = Column(String(255), nullable=False, index=True)
last_name = Column(String(255), nullable=False, index=True)
roosters = relationship("Rooster")
def get_full_name(self) -> str:
return f"{self.last_name}, {self.first_name}"
class Rooster(Base, BaseMixin):
__tablename__ = "rooster"
__table_args__ = (
UniqueConstraint("year", "team_id", "player_id", "position_id"),
)
year = Column(Integer)
team_id = Column(String, ForeignKey("team.id"), nullable=False, index=True)
team = relationship("Team", back_populates="roosters")
player_id = Column(String, ForeignKey("player.id"), nullable=False, index=True)
player = relationship("Player", back_populates="roosters")
position_id = Column(String, ForeignKey("field_position.id"), nullable=False, index=True)
position = relationship("FieldPosition", back_populates="roosters")
cards = relationship("Card")
class Vendor(Base, BaseMixin):
__tablename__ = "vendor"
name = Column(String(255), nullable=False, unique=True, index=True)
card_sets = relationship("CardSet")
cards = relationship("Card")
class CardSet(Base, BaseMixin):
__tablename__ = "card_set"
__table_args__ = (
UniqueConstraint("name", "vendor_id"),
)
name = Column(String(255), index=True)
parallel_set = Column(BIT(1))
insert_set = Column(BIT(1))
vendor_id = Column(String, ForeignKey("vendor.id"), nullable=False, index=True)
vendor = relationship("Vendor", back_populates="card_sets")
cards = relationship("Card")
class Card(Base, BaseMixin):
__tablename__ = "card"
__table_args__ = (
UniqueConstraint("card_number", "year", "vendor_id", "card_set_id"),
)
card_number = Column(Integer, index=True)
year = Column(Integer, index=True)
card_set_id = Column(String, ForeignKey("card_set.id"), nullable=False)
card_set = relationship("CardSet", back_populates="cards")
rooster_id = Column(String, ForeignKey("rooster.id"), nullable=False)
rooster = relationship("Rooster", back_populates="cards")
vendor_id = Column(String, ForeignKey("vendor.id"), nullable=False)
vendor = relationship("Vendor", back_populates="cards")
kontor-scripts/update_title.py
+22 -13
View File
@@ -2,7 +2,6 @@
download files with URLs from DB
"""
import logging.config
import requests
import yaml
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
@@ -37,23 +36,33 @@ def get_logger(level: int, config: str):
if __name__ == '__main__':
log = get_logger(args.verbose, args.config)
log.info('kontor.update_titles started')
response = requests.get("http://127.0.0.1:8800/media/files?review=true")
response = requests.get("http://127.0.0.1:8800/api/media/files?review=true")
log.info(f"Status: {response.status_code}")
data = response.json()
log.info(f"data: {len(data)}")
for item in data:
link = item['url']
log.info(f"{item['id']} - {link}")
try:
r = requests.get(link)
soup = BeautifulSoup(r.content, "html.parser")
title = soup.title.string
item['title'] = title
item['review'] = 0
except:
item['title'] = None
item['review'] = 1
update = requests.put(f"http://127.0.0.1:8800/media/files/{item['id']}", json=item)
log.info(f"{item['id']} - {str(link)}")
if not link:
continue
if link == "None":
item['url'] = None
else:
try:
r = requests.get(link)
soup = BeautifulSoup(r.content, "html.parser")
title_tag = soup.find('title')
if title_tag:
title= title_tag.get_text()
title = soup.title.string
item['title'] = title
item['review'] = False
except Exception as error:
log.info(f"something went wrong: {error}")
item['title'] = None
item['review'] = True
update = requests.put(f"http://127.0.0.1:8800/api/media/files/{item['id']}", json=item)
log.info(f"update status: {update.status_code}")
log.info(f"update result: {update.json()}")
log.info('kontor.update_titles finished')
kontor-scripts/uv.lock
Generated
+183 -15
View File
@@ -46,6 +46,28 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/38/fc/bce832fd4fd99766c04d1ee0eead6b0ec6486fb100ae5e74c1d91292b982/certifi-2025.1.31-py3-none-any.whl", hash = "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe", size = 166393, upload_time = "2025-01-31T02:16:45.015Z" },
]
[[package]]
name = "cffi"
version = "1.17.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pycparser" },
]
sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621, upload_time = "2024-09-04T20:45:21.852Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/8d/f8/dd6c246b148639254dad4d6803eb6a54e8c85c6e11ec9df2cffa87571dbe/cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e", size = 182989, upload_time = "2024-09-04T20:44:28.956Z" },
{ url = "https://files.pythonhosted.org/packages/8b/f1/672d303ddf17c24fc83afd712316fda78dc6fce1cd53011b839483e1ecc8/cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2", size = 178802, upload_time = "2024-09-04T20:44:30.289Z" },
{ url = "https://files.pythonhosted.org/packages/0e/2d/eab2e858a91fdff70533cab61dcff4a1f55ec60425832ddfdc9cd36bc8af/cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3", size = 454792, upload_time = "2024-09-04T20:44:32.01Z" },
{ url = "https://files.pythonhosted.org/packages/75/b2/fbaec7c4455c604e29388d55599b99ebcc250a60050610fadde58932b7ee/cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683", size = 478893, upload_time = "2024-09-04T20:44:33.606Z" },
{ url = "https://files.pythonhosted.org/packages/4f/b7/6e4a2162178bf1935c336d4da8a9352cccab4d3a5d7914065490f08c0690/cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5", size = 485810, upload_time = "2024-09-04T20:44:35.191Z" },
{ url = "https://files.pythonhosted.org/packages/c7/8a/1d0e4a9c26e54746dc08c2c6c037889124d4f59dffd853a659fa545f1b40/cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4", size = 471200, upload_time = "2024-09-04T20:44:36.743Z" },
{ url = "https://files.pythonhosted.org/packages/26/9f/1aab65a6c0db35f43c4d1b4f580e8df53914310afc10ae0397d29d697af4/cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd", size = 479447, upload_time = "2024-09-04T20:44:38.492Z" },
{ url = "https://files.pythonhosted.org/packages/5f/e4/fb8b3dd8dc0e98edf1135ff067ae070bb32ef9d509d6cb0f538cd6f7483f/cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed", size = 484358, upload_time = "2024-09-04T20:44:40.046Z" },
{ url = "https://files.pythonhosted.org/packages/f1/47/d7145bf2dc04684935d57d67dff9d6d795b2ba2796806bb109864be3a151/cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9", size = 488469, upload_time = "2024-09-04T20:44:41.616Z" },
{ url = "https://files.pythonhosted.org/packages/bf/ee/f94057fa6426481d663b88637a9a10e859e492c73d0384514a17d78ee205/cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d", size = 172475, upload_time = "2024-09-04T20:44:43.733Z" },
{ url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009, upload_time = "2024-09-04T20:44:45.309Z" },
]
[[package]]
name = "charset-normalizer"
version = "3.4.1"
@@ -89,6 +111,35 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload_time = "2022-10-25T02:36:20.889Z" },
]
[[package]]
name = "coverage"
version = "7.8.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/19/4f/2251e65033ed2ce1e68f00f91a0294e0f80c80ae8c3ebbe2f12828c4cd53/coverage-7.8.0.tar.gz", hash = "sha256:7a3d62b3b03b4b6fd41a085f3574874cf946cb4604d2b4d3e8dca8cd570ca501", size = 811872, upload_time = "2025-03-30T20:36:45.376Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/f3/21/87e9b97b568e223f3438d93072479c2f36cc9b3f6b9f7094b9d50232acc0/coverage-7.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ac46d0c2dd5820ce93943a501ac5f6548ea81594777ca585bf002aa8854cacd", size = 211708, upload_time = "2025-03-30T20:35:47.417Z" },
{ url = "https://files.pythonhosted.org/packages/75/be/882d08b28a0d19c9c4c2e8a1c6ebe1f79c9c839eb46d4fca3bd3b34562b9/coverage-7.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:771eb7587a0563ca5bb6f622b9ed7f9d07bd08900f7589b4febff05f469bea00", size = 211981, upload_time = "2025-03-30T20:35:49.002Z" },
{ url = "https://files.pythonhosted.org/packages/7a/1d/ce99612ebd58082fbe3f8c66f6d8d5694976c76a0d474503fa70633ec77f/coverage-7.8.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42421e04069fb2cbcbca5a696c4050b84a43b05392679d4068acbe65449b5c64", size = 245495, upload_time = "2025-03-30T20:35:51.073Z" },
{ url = "https://files.pythonhosted.org/packages/dc/8d/6115abe97df98db6b2bd76aae395fcc941d039a7acd25f741312ced9a78f/coverage-7.8.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:554fec1199d93ab30adaa751db68acec2b41c5602ac944bb19187cb9a41a8067", size = 242538, upload_time = "2025-03-30T20:35:52.941Z" },
{ url = "https://files.pythonhosted.org/packages/cb/74/2f8cc196643b15bc096d60e073691dadb3dca48418f08bc78dd6e899383e/coverage-7.8.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aaeb00761f985007b38cf463b1d160a14a22c34eb3f6a39d9ad6fc27cb73008", size = 244561, upload_time = "2025-03-30T20:35:54.658Z" },
{ url = "https://files.pythonhosted.org/packages/22/70/c10c77cd77970ac965734fe3419f2c98665f6e982744a9bfb0e749d298f4/coverage-7.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:581a40c7b94921fffd6457ffe532259813fc68eb2bdda60fa8cc343414ce3733", size = 244633, upload_time = "2025-03-30T20:35:56.221Z" },
{ url = "https://files.pythonhosted.org/packages/38/5a/4f7569d946a07c952688debee18c2bb9ab24f88027e3d71fd25dbc2f9dca/coverage-7.8.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f319bae0321bc838e205bf9e5bc28f0a3165f30c203b610f17ab5552cff90323", size = 242712, upload_time = "2025-03-30T20:35:57.801Z" },
{ url = "https://files.pythonhosted.org/packages/bb/a1/03a43b33f50475a632a91ea8c127f7e35e53786dbe6781c25f19fd5a65f8/coverage-7.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:04bfec25a8ef1c5f41f5e7e5c842f6b615599ca8ba8391ec33a9290d9d2db3a3", size = 244000, upload_time = "2025-03-30T20:35:59.378Z" },
{ url = "https://files.pythonhosted.org/packages/6a/89/ab6c43b1788a3128e4d1b7b54214548dcad75a621f9d277b14d16a80d8a1/coverage-7.8.0-cp313-cp313-win32.whl", hash = "sha256:dd19608788b50eed889e13a5d71d832edc34fc9dfce606f66e8f9f917eef910d", size = 214195, upload_time = "2025-03-30T20:36:01.005Z" },
{ url = "https://files.pythonhosted.org/packages/12/12/6bf5f9a8b063d116bac536a7fb594fc35cb04981654cccb4bbfea5dcdfa0/coverage-7.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:a9abbccd778d98e9c7e85038e35e91e67f5b520776781d9a1e2ee9d400869487", size = 214998, upload_time = "2025-03-30T20:36:03.006Z" },
{ url = "https://files.pythonhosted.org/packages/2a/e6/1e9df74ef7a1c983a9c7443dac8aac37a46f1939ae3499424622e72a6f78/coverage-7.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:18c5ae6d061ad5b3e7eef4363fb27a0576012a7447af48be6c75b88494c6cf25", size = 212541, upload_time = "2025-03-30T20:36:04.638Z" },
{ url = "https://files.pythonhosted.org/packages/04/51/c32174edb7ee49744e2e81c4b1414ac9df3dacfcb5b5f273b7f285ad43f6/coverage-7.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:95aa6ae391a22bbbce1b77ddac846c98c5473de0372ba5c463480043a07bff42", size = 212767, upload_time = "2025-03-30T20:36:06.503Z" },
{ url = "https://files.pythonhosted.org/packages/e9/8f/f454cbdb5212f13f29d4a7983db69169f1937e869a5142bce983ded52162/coverage-7.8.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e013b07ba1c748dacc2a80e69a46286ff145935f260eb8c72df7185bf048f502", size = 256997, upload_time = "2025-03-30T20:36:08.137Z" },
{ url = "https://files.pythonhosted.org/packages/e6/74/2bf9e78b321216d6ee90a81e5c22f912fc428442c830c4077b4a071db66f/coverage-7.8.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d766a4f0e5aa1ba056ec3496243150698dc0481902e2b8559314368717be82b1", size = 252708, upload_time = "2025-03-30T20:36:09.781Z" },
{ url = "https://files.pythonhosted.org/packages/92/4d/50d7eb1e9a6062bee6e2f92e78b0998848a972e9afad349b6cdde6fa9e32/coverage-7.8.0-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad80e6b4a0c3cb6f10f29ae4c60e991f424e6b14219d46f1e7d442b938ee68a4", size = 255046, upload_time = "2025-03-30T20:36:11.409Z" },
{ url = "https://files.pythonhosted.org/packages/40/9e/71fb4e7402a07c4198ab44fc564d09d7d0ffca46a9fb7b0a7b929e7641bd/coverage-7.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:b87eb6fc9e1bb8f98892a2458781348fa37e6925f35bb6ceb9d4afd54ba36c73", size = 256139, upload_time = "2025-03-30T20:36:13.86Z" },
{ url = "https://files.pythonhosted.org/packages/49/1a/78d37f7a42b5beff027e807c2843185961fdae7fe23aad5a4837c93f9d25/coverage-7.8.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:d1ba00ae33be84066cfbe7361d4e04dec78445b2b88bdb734d0d1cbab916025a", size = 254307, upload_time = "2025-03-30T20:36:16.074Z" },
{ url = "https://files.pythonhosted.org/packages/58/e9/8fb8e0ff6bef5e170ee19d59ca694f9001b2ec085dc99b4f65c128bb3f9a/coverage-7.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f3c38e4e5ccbdc9198aecc766cedbb134b2d89bf64533973678dfcf07effd883", size = 255116, upload_time = "2025-03-30T20:36:18.033Z" },
{ url = "https://files.pythonhosted.org/packages/56/b0/d968ecdbe6fe0a863de7169bbe9e8a476868959f3af24981f6a10d2b6924/coverage-7.8.0-cp313-cp313t-win32.whl", hash = "sha256:379fe315e206b14e21db5240f89dc0774bdd3e25c3c58c2c733c99eca96f1ada", size = 214909, upload_time = "2025-03-30T20:36:19.644Z" },
{ url = "https://files.pythonhosted.org/packages/87/e9/d6b7ef9fecf42dfb418d93544af47c940aa83056c49e6021a564aafbc91f/coverage-7.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:2e4b6b87bb0c846a9315e3ab4be2d52fac905100565f4b92f02c445c8799e257", size = 216068, upload_time = "2025-03-30T20:36:21.282Z" },
{ url = "https://files.pythonhosted.org/packages/59/f1/4da7717f0063a222db253e7121bd6a56f6fb1ba439dcc36659088793347c/coverage-7.8.0-py3-none-any.whl", hash = "sha256:dbf364b4c5e7bae9250528167dfe40219b62e2d573c854d74be213e1e52069f7", size = 203435, upload_time = "2025-03-30T20:36:43.61Z" },
]
[[package]]
name = "dnspython"
version = "2.7.0"
@@ -98,6 +149,12 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/68/1b/e0a87d256e40e8c888847551b20a017a6b98139178505dc7ffb96f04e954/dnspython-2.7.0-py3-none-any.whl", hash = "sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86", size = 313632, upload_time = "2024-10-05T20:14:57.687Z" },
]
[[package]]
name = "docopt"
version = "0.6.2"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/a2/55/8f8cab2afd404cf578136ef2cc5dfb50baa1761b68c9da1fb1e4eed343c9/docopt-0.6.2.tar.gz", hash = "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491", size = 25901, upload_time = "2014-06-16T11:18:57.406Z" }
[[package]]
name = "email-validator"
version = "2.2.0"
@@ -240,6 +297,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload_time = "2024-09-15T18:07:37.964Z" },
]
[[package]]
name = "iniconfig"
version = "2.1.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload_time = "2025-03-19T20:09:59.721Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload_time = "2025-03-19T20:10:01.071Z" },
]
[[package]]
name = "jinja2"
version = "3.1.6"
@@ -258,40 +324,37 @@ version = "0.1.0"
source = { virtual = "." }
dependencies = [
{ name = "beautifulsoup4" },
{ name = "coverage" },
{ name = "fastapi", extra = ["standard"] },
{ name = "mariadb" },
{ name = "pathlib" },
{ name = "platformdirs" },
{ name = "proton" },
{ name = "psycopg2-binary" },
{ name = "pytest-cov" },
{ name = "python-qpid-proton" },
{ name = "pyyaml" },
{ name = "requests" },
{ name = "sqlalchemy" },
{ name = "sqlmodel" },
{ name = "stomp-py" },
]
[package.metadata]
requires-dist = [
{ name = "beautifulsoup4", specifier = ">=4.13.4" },
{ name = "coverage", specifier = ">=7.8.0" },
{ name = "fastapi", extras = ["standard"], specifier = ">=0.115.12" },
{ name = "mariadb", specifier = ">=1.1.12" },
{ name = "pathlib", specifier = ">=1.0.1" },
{ name = "platformdirs", specifier = ">=4.3.7" },
{ name = "proton", specifier = ">=0.9.1" },
{ name = "psycopg2-binary", specifier = ">=2.9.10" },
{ name = "pytest-cov", specifier = ">=6.1.1" },
{ name = "python-qpid-proton", specifier = ">=0.40.0" },
{ name = "pyyaml", specifier = ">=6.0.2" },
{ name = "requests", specifier = ">=2.32.3" },
{ name = "sqlalchemy", specifier = ">=2.0.40" },
{ name = "sqlmodel", specifier = ">=0.0.24" },
]
[[package]]
name = "mariadb"
version = "1.1.12"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "packaging" },
]
sdist = { url = "https://files.pythonhosted.org/packages/17/bb/4bbc803fbdafedbfba015f7cc1ab1e87a6d1de36725ba058c53e2f8a45ad/mariadb-1.1.12.tar.gz", hash = "sha256:50b02ff2c78b1b4f4628a054e3c8c7dd92972137727a5cc309a64c9ed20c878c", size = 85934, upload_time = "2025-02-13T13:11:48.642Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/2b/1b/b6eca3870ac1b5577a10d3b49ba42ac263c2e5718c9224cc1c8463940422/mariadb-1.1.12-cp313-cp313-win32.whl", hash = "sha256:ba43c42130d41352f32a5786c339cc931d05472ef7640fa3764d428dc294b88e", size = 184338, upload_time = "2025-02-13T13:11:34.935Z" },
{ url = "https://files.pythonhosted.org/packages/fb/ff/c29a543ee1f9009755bc304138f61cd9b0ee1f14533e446513f84ccf143a/mariadb-1.1.12-cp313-cp313-win_amd64.whl", hash = "sha256:b69bc18418e72fcf359d17736cdc3f601a271203aff13ef7c57a415c8fd52ab0", size = 201272, upload_time = "2025-02-13T13:11:38.074Z" },
{ name = "stomp-py" },
]
[[package]]
@@ -370,6 +433,49 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/6d/45/59578566b3275b8fd9157885918fcd0c4d74162928a5310926887b856a51/platformdirs-4.3.7-py3-none-any.whl", hash = "sha256:a03875334331946f13c549dbd8f4bac7a13a50a895a0eb1e8c6a8ace80d40a94", size = 18499, upload_time = "2025-03-19T20:36:09.038Z" },
]
[[package]]
name = "pluggy"
version = "1.5.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/96/2d/02d4312c973c6050a18b314a5ad0b3210edb65a906f868e31c111dede4a6/pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1", size = 67955, upload_time = "2024-04-20T21:34:42.531Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/88/5f/e351af9a41f866ac3f1fac4ca0613908d9a41741cfcf2228f4ad853b697d/pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669", size = 20556, upload_time = "2024-04-20T21:34:40.434Z" },
]
[[package]]
name = "proton"
version = "0.9.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/6a/2f/512c06db681c1cace7773b5615fab88b507b06b287c061d4b56fdeeafb4e/proton-0.9.1.tar.gz", hash = "sha256:aaea5dcbd3f57b4ef59207b92bc34c43e84be256822b4ea66daf55286c9f256f", size = 16983, upload_time = "2021-08-08T22:32:03.958Z" }
[[package]]
name = "psycopg2-binary"
version = "2.9.10"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/cb/0e/bdc8274dc0585090b4e3432267d7be4dfbfd8971c0fa59167c711105a6bf/psycopg2-binary-2.9.10.tar.gz", hash = "sha256:4b3df0e6990aa98acda57d983942eff13d824135fe2250e6522edaa782a06de2", size = 385764, upload_time = "2024-10-16T11:24:58.126Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/3e/30/d41d3ba765609c0763505d565c4d12d8f3c79793f0d0f044ff5a28bf395b/psycopg2_binary-2.9.10-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:26540d4a9a4e2b096f1ff9cce51253d0504dca5a85872c7f7be23be5a53eb18d", size = 3044699, upload_time = "2024-10-16T11:21:42.841Z" },
{ url = "https://files.pythonhosted.org/packages/35/44/257ddadec7ef04536ba71af6bc6a75ec05c5343004a7ec93006bee66c0bc/psycopg2_binary-2.9.10-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:e217ce4d37667df0bc1c397fdcd8de5e81018ef305aed9415c3b093faaeb10fb", size = 3275245, upload_time = "2024-10-16T11:21:51.989Z" },
{ url = "https://files.pythonhosted.org/packages/1b/11/48ea1cd11de67f9efd7262085588790a95d9dfcd9b8a687d46caf7305c1a/psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:245159e7ab20a71d989da00f280ca57da7641fa2cdcf71749c193cea540a74f7", size = 2851631, upload_time = "2024-10-16T11:21:57.584Z" },
{ url = "https://files.pythonhosted.org/packages/62/e0/62ce5ee650e6c86719d621a761fe4bc846ab9eff8c1f12b1ed5741bf1c9b/psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c4ded1a24b20021ebe677b7b08ad10bf09aac197d6943bfe6fec70ac4e4690d", size = 3082140, upload_time = "2024-10-16T11:22:02.005Z" },
{ url = "https://files.pythonhosted.org/packages/27/ce/63f946c098611f7be234c0dd7cb1ad68b0b5744d34f68062bb3c5aa510c8/psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3abb691ff9e57d4a93355f60d4f4c1dd2d68326c968e7db17ea96df3c023ef73", size = 3264762, upload_time = "2024-10-16T11:22:06.412Z" },
{ url = "https://files.pythonhosted.org/packages/43/25/c603cd81402e69edf7daa59b1602bd41eb9859e2824b8c0855d748366ac9/psycopg2_binary-2.9.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8608c078134f0b3cbd9f89b34bd60a943b23fd33cc5f065e8d5f840061bd0673", size = 3020967, upload_time = "2024-10-16T11:22:11.583Z" },
{ url = "https://files.pythonhosted.org/packages/5f/d6/8708d8c6fca531057fa170cdde8df870e8b6a9b136e82b361c65e42b841e/psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:230eeae2d71594103cd5b93fd29d1ace6420d0b86f4778739cb1a5a32f607d1f", size = 2872326, upload_time = "2024-10-16T11:22:16.406Z" },
{ url = "https://files.pythonhosted.org/packages/ce/ac/5b1ea50fc08a9df82de7e1771537557f07c2632231bbab652c7e22597908/psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bb89f0a835bcfc1d42ccd5f41f04870c1b936d8507c6df12b7737febc40f0909", size = 2822712, upload_time = "2024-10-16T11:22:21.366Z" },
{ url = "https://files.pythonhosted.org/packages/c4/fc/504d4503b2abc4570fac3ca56eb8fed5e437bf9c9ef13f36b6621db8ef00/psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f0c2d907a1e102526dd2986df638343388b94c33860ff3bbe1384130828714b1", size = 2920155, upload_time = "2024-10-16T11:22:25.684Z" },
{ url = "https://files.pythonhosted.org/packages/b2/d1/323581e9273ad2c0dbd1902f3fb50c441da86e894b6e25a73c3fda32c57e/psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f8157bed2f51db683f31306aa497311b560f2265998122abe1dce6428bd86567", size = 2959356, upload_time = "2024-10-16T11:22:30.562Z" },
{ url = "https://files.pythonhosted.org/packages/08/50/d13ea0a054189ae1bc21af1d85b6f8bb9bbc5572991055d70ad9006fe2d6/psycopg2_binary-2.9.10-cp313-cp313-win_amd64.whl", hash = "sha256:27422aa5f11fbcd9b18da48373eb67081243662f9b46e6fd07c3eb46e4535142", size = 2569224, upload_time = "2025-01-04T20:09:19.234Z" },
]
[[package]]
name = "pycparser"
version = "2.22"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/1d/b2/31537cf4b1ca988837256c910a668b553fceb8f069bedc4b1c826024b52c/pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", size = 172736, upload_time = "2024-03-30T13:22:22.564Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552, upload_time = "2024-03-30T13:22:20.476Z" },
]
[[package]]
name = "pydantic"
version = "2.11.3"
@@ -422,6 +528,34 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c", size = 1225293, upload_time = "2025-01-06T17:26:25.553Z" },
]
[[package]]
name = "pytest"
version = "8.3.5"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "colorama", marker = "sys_platform == 'win32'" },
{ name = "iniconfig" },
{ name = "packaging" },
{ name = "pluggy" },
]
sdist = { url = "https://files.pythonhosted.org/packages/ae/3c/c9d525a414d506893f0cd8a8d0de7706446213181570cdbd766691164e40/pytest-8.3.5.tar.gz", hash = "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845", size = 1450891, upload_time = "2025-03-02T12:54:54.503Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/30/3d/64ad57c803f1fa1e963a7946b6e0fea4a70df53c1a7fed304586539c2bac/pytest-8.3.5-py3-none-any.whl", hash = "sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820", size = 343634, upload_time = "2025-03-02T12:54:52.069Z" },
]
[[package]]
name = "pytest-cov"
version = "6.1.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "coverage" },
{ name = "pytest" },
]
sdist = { url = "https://files.pythonhosted.org/packages/25/69/5f1e57f6c5a39f81411b550027bf72842c4567ff5fd572bed1edc9e4b5d9/pytest_cov-6.1.1.tar.gz", hash = "sha256:46935f7aaefba760e716c2ebfbe1c216240b9592966e7da99ea8292d4d3e2a0a", size = 66857, upload_time = "2025-04-05T14:07:51.592Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/28/d0/def53b4a790cfb21483016430ed828f64830dd981ebe1089971cd10cab25/pytest_cov-6.1.1-py3-none-any.whl", hash = "sha256:bddf29ed2d0ab6f4df17b4c55b0a657287db8684af9c42ea546b21b1041b3dde", size = 23841, upload_time = "2025-04-05T14:07:49.641Z" },
]
[[package]]
name = "python-dotenv"
version = "1.1.0"
@@ -440,6 +574,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/45/58/38b5afbc1a800eeea951b9285d3912613f2603bdf897a4ab0f4bd7f405fc/python_multipart-0.0.20-py3-none-any.whl", hash = "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104", size = 24546, upload_time = "2024-12-16T19:45:44.423Z" },
]
[[package]]
name = "python-qpid-proton"
version = "0.40.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "cffi" },
]
sdist = { url = "https://files.pythonhosted.org/packages/d5/dd/e9e5066009517bdfee92374264a2b6794fa0987bfeddcbf4d2a08dccaf36/python_qpid_proton-0.40.0.tar.gz", hash = "sha256:7680d607cf6e9684f97bf5b2ba16cda7d8512aab9e4ff78f98d44a4644fc819a", size = 354215, upload_time = "2025-05-19T18:45:37.932Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/7b/dd/a82c1e377f08d62d83898c1aa9b39aef890e910f683fca6dc5242a123f6b/python_qpid_proton-0.40.0-cp313-cp313-win_amd64.whl", hash = "sha256:a19d8c71c908700ceb38f6cbc1eb4a039428570f96bfc2caeeafdfec804fb94f", size = 277376, upload_time = "2025-05-19T19:39:31.201Z" },
]
[[package]]
name = "pyyaml"
version = "6.0.2"
@@ -572,6 +718,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/8b/0c/9d30a4ebeb6db2b25a841afbb80f6ef9a854fc3b41be131d249a977b4959/starlette-0.46.2-py3-none-any.whl", hash = "sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35", size = 72037, upload_time = "2025-04-13T13:56:16.21Z" },
]
[[package]]
name = "stomp-py"
version = "8.2.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "docopt" },
{ name = "websocket-client" },
]
sdist = { url = "https://files.pythonhosted.org/packages/bc/66/c07f01feb5fbc669c4333c76eb02fb8149c653c25ba9769477f8427d5e55/stomp_py-8.2.0.tar.gz", hash = "sha256:9908689361e263bf198e6acfb3c4386759fb7df7d141f4384d7414771c68d7fc", size = 39286, upload_time = "2024-10-31T21:59:38.465Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/7b/b6/ebfd6daef0c19a5ca3ac1fb2fc092331d67af5a30c868f106fcc2504c287/stomp_py-8.2.0-py3-none-any.whl", hash = "sha256:fad24e51b505996015a39ca1632df4e0225c1c552980955e21f2aebfc0d9d85c", size = 42751, upload_time = "2024-10-31T21:59:36.658Z" },
]
[[package]]
name = "typer"
version = "0.15.2"
@@ -678,6 +837,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/a8/b4/c57b99518fadf431f3ef47a610839e46e5f8abf9814f969859d1c65c02c7/watchfiles-1.0.5-cp313-cp313-win_amd64.whl", hash = "sha256:f436601594f15bf406518af922a89dcaab416568edb6f65c4e5bbbad1ea45c11", size = 291087, upload_time = "2025-04-08T10:35:52.458Z" },
]
[[package]]
name = "websocket-client"
version = "1.8.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/e6/30/fba0d96b4b5fbf5948ed3f4681f7da2f9f64512e1d303f94b4cc174c24a5/websocket_client-1.8.0.tar.gz", hash = "sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da", size = 54648, upload_time = "2024-04-23T22:16:16.976Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/5a/84/44687a29792a70e111c5c477230a72c4b957d88d16141199bf9acb7537a3/websocket_client-1.8.0-py3-none-any.whl", hash = "sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526", size = 58826, upload_time = "2024-04-23T22:16:14.422Z" },
]
[[package]]
name = "websockets"
version = "15.0.1"