diff --git a/kontor-scripts/add_actors.py b/kontor-scripts/add_actors.py deleted file mode 100644 index 04194cf..0000000 --- a/kontor-scripts/add_actors.py +++ /dev/null @@ -1,369 +0,0 @@ -""" -add actors -""" -import logging.config -import requests -import re -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter - -from bs4 import BeautifulSoup - -parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) -parser.add_argument('--verbose', '-v', action='count', default=0) -args = parser.parse_args() - -def get_logger(level: int) -> logging.Logger: - logging.config.dictConfig({ - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'simple': { - 'format': '[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s', - 'datefmt': '%Y-%m-%d %H:%M:%S', - }, - }, - 'handlers': { - 'console': { - 'class': logging.StreamHandler, - 'level': logging.DEBUG, - 'formatter': 'simple', - 'stream': 'ext://sys.stdout' - }, - }, - 'loggers': { - 'urllib3.connectionpool': { - 'level': 'WARNING', - 'propagate': False, - }, - 'root': { - 'level': 'DEBUG', - 'handlers': ['console'], - }, - }, - }) - logger = logging.getLogger(__file__) - if level is not None: - match level: - case 0: - logger.setLevel(logging.WARNING) - case 1: - logger.setLevel(logging.INFO) - case 2: - logger.setLevel(logging.DEBUG) - case _: - logger.setLevel(logging.CRITICAL) - return logger - - -if __name__ == '__main__': - log = get_logger(args.verbose) - log.warning('kontor.add_actors started') - log.debug('get all actors') - response = requests.get("http://127.0.0.1:8800/api/media/actors") - data = response.json() - actors = {} - for item in data: - actor = {} - actor['id'] = item['id'] - actor['name'] = item['name'] - actor['url'] = item['url'] - actors[item['url']] = actor - log.debug(f'all actors: {actors}') - - new_actor_list = [ - { 'name': 'Herschel Savage', 'url': 'https://ge.xhamster.com/pornstars/herschel-savage'}, - { 'name': 'Janey Robbins', 'url': 'https://ge.xhamster.com/pornstars/janey-robbins'}, - { 'name': 'Kimberly Carson', 'url': 'https://ge.xhamster.com/pornstars/kimberly-carson'}, - { 'name': 'Paul Thomas', 'url': 'https://ge.xhamster.com/pornstars/paul-thomas'}, - { 'name': 'Shauna Grant', 'url': 'https://ge.xhamster.com/pornstars/shauna-grant'}, - { 'name': 'Tara Aire', 'url': 'https://ge.xhamster.com/pornstars/tara-aire'}, - { 'name': 'Kari Milla', 'url': 'https://ge.xhamster.com/pornstars/kari-milla'}, - { 'name': 'Drago Gvozdik', 'url': 'https://ge.xhamster.com/pornstars/drago-gvozdik'}, - { 'name': 'Cheyne Collins', 'url': 'https://ge.xhamster.com/pornstars/cheyne-collins'}, - { 'name': 'Christian XXX', 'url': 'https://ge.xhamster.com/pornstars/christian-xxx'}, - { 'name': 'Derrick Pierce', 'url': 'https://ge.xhamster.com/pornstars/derrick-pierce'}, - { 'name': 'Holly Halston', 'url': 'https://ge.xhamster.com/pornstars/holly-halston'}, - { 'name': 'Holly West', 'url': 'https://ge.xhamster.com/pornstars/holly-west'}, - { 'name': 'Jarod Diamond', 'url': 'https://ge.xhamster.com/pornstars/jarod-diamond'}, - { 'name': 'Kristina Cross', 'url': 'https://ge.xhamster.com/pornstars/kristina-cross'}, - { 'name': 'Lee Stone', 'url': 'https://ge.xhamster.com/pornstars/lee-stone'}, - { 'name': 'Monica Mayhem', 'url': 'https://ge.xhamster.com/pornstars/monica-mayhem'}, - { 'name': 'Sienna West', 'url': 'https://ge.xhamster.com/pornstars/sienna-west'}, - { 'name': 'Aria Carson', 'url': 'https://ge.xhamster.com/pornstars/aria-carson'}, - { 'name': 'Britney Amber', 'url': 'https://ge.xhamster.com/pornstars/britney-amber'}, - { 'name': 'Kit Mercer', 'url': 'https://ge.xhamster.com/pornstars/kit-mercer'}, - { 'name': 'Riley Reyes', 'url': 'https://ge.xhamster.com/pornstars/riley-reyes'}, - { 'name': 'Ryan Keely', 'url': 'https://ge.xhamster.com/pornstars/ryan-keely'}, - { 'name': 'Alexandra Diamond', 'url': 'https://ge.xhamster.com/pornstars/alexandra-diamond'}, - { 'name': 'Amanda Twice', 'url': 'https://ge.xhamster.com/pornstars/amanda-twice'}, - { 'name': 'Choky Ice', 'url': 'https://ge.xhamster.com/pornstars/choky-ice'}, - { 'name': 'Cindy Cox', 'url': 'https://ge.xhamster.com/pornstars/cindy-cox'}, - { 'name': 'Kelly White', 'url': 'https://ge.xhamster.com/pornstars/kelly-white'}, - { 'name': 'Mike Foster', 'url': 'https://ge.xhamster.com/pornstars/mike-foster'}, - { 'name': 'Susie Sorrento', 'url': 'https://ge.xhamster.com/pornstars/susie-sorrento'}, - { 'name': 'Lara Sanchez', 'url': 'https://ge.xhamster.com/pornstars/lara-sanchez'}, - { 'name': 'Alicia Williams', 'url': 'https://ge.xhamster.com/pornstars/alicia-williams'}, - { 'name': 'Dirty Tina', 'url': 'https://ge.xhamster.com/pornstars/dirty-tina'}, - { 'name': 'Andrew Andretti', 'url': 'https://ge.xhamster.com/pornstars/andrew-andretti'}, - { 'name': 'Dick Nasty', 'url': 'https://ge.xhamster.com/pornstars/dick-nasty'}, - { 'name': 'Vicky Vette', 'url': 'https://ge.xhamster.com/pornstars/vicky-vette'}, - { 'name': 'Alex Gonz', 'url': 'https://ge.xhamster.com/pornstars/alex-gonz'}, - { 'name': 'Audrey Hollander', 'url': 'https://ge.xhamster.com/pornstars/audrey-hollander'}, - { 'name': 'Desiree Diamond', 'url': 'https://ge.xhamster.com/pornstars/desiree-diamond'}, - { 'name': 'Lexi Bardot', 'url': 'https://ge.xhamster.com/pornstars/lexi-bardot'}, - { 'name': 'Adam Ocelot', 'url': 'https://ge.xhamster.com/pornstars/adam-ocelot'}, - { 'name': 'Scarlett Hampton', 'url': 'https://ge.xhamster.com/pornstars/scarlett-hampton'}, - { 'name': 'Laetitia', 'url': 'https://ge.xhamster.com/pornstars/laetitia'}, - { 'name': 'Raffaela Anderson', 'url': 'https://ge.xhamster.com/pornstars/raffaela-anderson'}, - { 'name': 'Michael Swayze', 'url': 'https://ge.xhamster.com/pornstars/michael-swayze'}, - { 'name': 'Winter Jade', 'url': 'https://ge.xhamster.com/pornstars/winter-jade'}, - { 'name': 'Celine Noiret', 'url': 'https://ge.xhamster.com/pornstars/celine-noiret'}, - { 'name': 'Aurora Snow', 'url': 'https://ge.xhamster.com/pornstars/aurora-snow'}, - { 'name': 'Bree Brooks', 'url': 'https://ge.xhamster.com/pornstars/bree-brooks'}, - { 'name': 'Flick Shagwell', 'url': 'https://ge.xhamster.com/pornstars/flick-shagwell'}, - { 'name': 'Noname Jane', 'url': 'https://ge.xhamster.com/pornstars/noname-jane'}, - { 'name': 'Alex Magni', 'url': 'https://ge.xhamster.com/pornstars/alex-magni'}, - { 'name': 'Maeva Dream', 'url': 'https://ge.xhamster.com/pornstars/maeva-dream'}, - { 'name': 'Squirty Alice', 'url': 'https://ge.xhamster.com/pornstars/squirty-alice'}, - { 'name': 'Mandy Rhea', 'url': 'https://ge.xhamster.com/pornstars/mandy-rhea'}, - { 'name': 'Alessia Donati', 'url': 'https://ge.xhamster.com/pornstars/alessia-donati'}, - { 'name': 'Pierre DJ', 'url': 'https://ge.xhamster.com/pornstars/pierre-dj'}, - { 'name': 'Veronica Belli', 'url': 'https://ge.xhamster.com/pornstars/veronica-belli'}, - { 'name': 'Christiana Cinn', 'url': 'https://ge.xhamster.com/pornstars/christiana-cinn'}, - { 'name': 'Jasmine Jae', 'url': 'https://ge.xhamster.com/pornstars/jasmine-jae'}, - { 'name': 'Jay Smooth', 'url': 'https://ge.xhamster.com/pornstars/jay-smooth'}, - { 'name': 'Natalie Knight', 'url': 'https://ge.xhamster.com/pornstars/natalie-knight'}, - { 'name': 'Joshua Lewis', 'url': 'https://ge.xhamster.com/pornstars/joshua-lewis'}, - { 'name': 'Lauren Phillips', 'url': 'https://ge.xhamster.com/pornstars/lauren-phillips'}, - { 'name': 'Matt Cash', 'url': 'https://ge.xhamster.com/pornstars/matt-cash'}, - { 'name': 'Nickey Huntsman', 'url': 'https://ge.xhamster.com/pornstars/nickey-huntsman'}, - { 'name': 'Jade Sin', 'url': 'https://ge.xhamster.com/pornstars/jade-sin'}, - { 'name': 'Wein Lewis', 'url': 'https://ge.xhamster.com/pornstars/wein-lewis'}, - { 'name': 'Megan Murkovski', 'url': 'https://ge.xhamster.com/pornstars/megan-murkovski'}, - { 'name': 'Tara Wild', 'url': 'https://ge.xhamster.com/pornstars/tara-wild'}, - { 'name': 'Lana Roy', 'url': 'https://ge.xhamster.com/pornstars/lana-roy'}, - { 'name': 'Nick Moreno', 'url': 'https://ge.xhamster.com/pornstars/nick-moreno'}, - { 'name': 'Brad Armstrong', 'url': 'https://ge.xhamster.com/pornstars/brad-armstrong'}, - { 'name': 'Kaylani Lei', 'url': 'https://ge.xhamster.com/pornstars/kaylani-lei'}, - { 'name': 'Deborah Wells', 'url': 'https://ge.xhamster.com/pornstars/deborah-wells'}, - { 'name': 'Judith Ramirez', 'url': 'https://ge.xhamster.com/pornstars/judith-ramirez'}, - { 'name': 'Richard Langin', 'url': 'https://ge.xhamster.com/pornstars/richard-langin'}, - { 'name': 'Simona Valli', 'url': 'https://ge.xhamster.com/pornstars/simona-valli'}, - { 'name': 'Mandy Dee', 'url': 'https://ge.xhamster.com/pornstars/mandy-dee'}, - { 'name': 'Allison Wyte', 'url': 'https://ge.xhamster.com/pornstars/allison-wyte'}, - { 'name': 'Ben English', 'url': 'https://ge.xhamster.com/pornstars/ben-english'}, - { 'name': 'Chuck Martino', 'url': 'https://ge.xhamster.com/pornstars/chuck-martino'}, - { 'name': 'Erika Kole', 'url': 'https://ge.xhamster.com/pornstars/erika-kole'}, - { 'name': 'Katie Morgan', 'url': 'https://ge.xhamster.com/pornstars/katie-morgan'}, - { 'name': 'Misty Parks', 'url': 'https://ge.xhamster.com/pornstars/misty-parks'}, - { 'name': 'Mr. Pete', 'url': 'https://ge.xhamster.com/pornstars/mr-pete'}, - { 'name': 'Candie Luciani', 'url': 'https://ge.xhamster.com/pornstars/candie-luciani'}, - { 'name': 'Clara Mia', 'url': 'https://ge.xhamster.com/pornstars/clara-mia'}, - { 'name': 'Tommy Cabrio', 'url': 'https://ge.xhamster.com/pornstars/tommy-cabrio'}, - { 'name': 'Nathan Bronson', 'url': 'https://ge.xhamster.com/pornstars/nathan-bronson'}, - { 'name': 'Octavia Red', 'url': 'https://ge.xhamster.com/pornstars/octavia-red'}, - { 'name': 'Big George', 'url': 'https://ge.xhamster.com/pornstars/big-george'}, - { 'name': 'Cassy Young', 'url': 'https://ge.xhamster.com/pornstars/cassy-young'}, - { 'name': 'Egon Kowalski', 'url': 'https://ge.xhamster.com/pornstars/egon-kowalski'}, - { 'name': 'Mark Aurel', 'url': 'https://ge.xhamster.com/pornstars/mark-aurel'}, - { 'name': 'Samy Fox', 'url': 'https://ge.xhamster.com/pornstars/samy-fox'}, - { 'name': 'Valeria Jones', 'url': 'https://ge.xhamster.com/pornstars/valeria-jones'}, - { 'name': 'Dieter Von Stein', 'url': 'https://ge.xhamster.com/pornstars/dieter-von-stein'}, - { 'name': 'Donna Bell', 'url': 'https://ge.xhamster.com/pornstars/donna-bell'}, - { 'name': 'Molly Jane', 'url': 'https://ge.xhamster.com/pornstars/molly-jane'}, - { 'name': 'Mark Zane', 'url': 'https://ge.xhamster.com/pornstars/mark-zane'}, - { 'name': 'Titus Steel', 'url': 'https://ge.xhamster.com/pornstars/titus-steel'}, - { 'name': 'Mackenzie Mace', 'url': 'https://ge.xhamster.com/pornstars/mackenzie-mace'}, - { 'name': 'Christina Lang', 'url': 'https://ge.xhamster.com/pornstars/christina-lang'}, - { 'name': 'Jens Modena', 'url': 'https://ge.xhamster.com/pornstars/jens-modena'}, - { 'name': 'Karma Rosenberg', 'url': 'https://ge.xhamster.com/pornstars/karma-rosenberg'}, - { 'name': 'Dani Jensen', 'url': 'https://ge.xhamster.com/pornstars/dani-jensen'}, - { 'name': 'Crystal Ray', 'url': 'https://ge.xhamster.com/pornstars/crystal-ray'}, - { 'name': 'Andrea Szabo', 'url': 'https://ge.xhamster.com/pornstars/andrea-szabo'}, - { 'name': 'Desiree Barclay', 'url': 'https://ge.xhamster.com/pornstars/desiree-barclay'}, - { 'name': 'Jada Stevens', 'url': 'https://ge.xhamster.com/pornstars/jada-stevens'}, - { 'name': 'Syren Demer', 'url': 'https://ge.xhamster.com/pornstars/syren-demer'}, - { 'name': 'Ara Mix', 'url': 'https://ge.xhamster.com/pornstars/ara-mix'}, - { 'name': 'Redneck John', 'url': 'https://ge.xhamster.com/pornstars/redneck-john'}, - { 'name': 'Allysin Chaynes', 'url': 'https://ge.xhamster.com/pornstars/allysin-chaynes'}, - { 'name': 'Aspen Brock', 'url': 'https://ge.xhamster.com/pornstars/aspen-brock'}, - { 'name': 'Kaylynn', 'url': 'https://ge.xhamster.com/pornstars/kaylynn'}, - { 'name': 'Monica Cameron', 'url': 'https://ge.xhamster.com/pornstars/monica-cameron'}, - { 'name': 'Sheila Rossi', 'url': 'https://ge.xhamster.com/pornstars/sheila-rossi'}, - { 'name': 'Alex Ginger', 'url': 'https://ge.xhamster.com/pornstars/alex-ginger'}, - { 'name': 'Ally Style', 'url': 'https://ge.xhamster.com/pornstars/ally-style'}, - { 'name': 'Paola Mike', 'url': 'https://ge.xhamster.com/pornstars/paola-mike'}, - { 'name': 'Jeanette Littledove', 'url': 'https://ge.xhamster.com/pornstars/jeanette-littledove'}, - { 'name': 'Melissa Melendez', 'url': 'https://ge.xhamster.com/pornstars/melissa-melendez'}, - { 'name': 'Peter North', 'url': 'https://ge.xhamster.com/pornstars/peter-north'}, - { 'name': 'Siobhan Hunter', 'url': 'https://ge.xhamster.com/pornstars/siobhan-hunter'}, - { 'name': 'Tami White', 'url': 'https://ge.xhamster.com/pornstars/tami-white'}, - { 'name': 'Tracey Adams', 'url': 'https://ge.xhamster.com/pornstars/tracey-adams'}, - { 'name': 'Ashley Haze', 'url': 'https://ge.xhamster.com/pornstars/ashley-haze'}, - { 'name': 'Cailey Taylor', 'url': 'https://ge.xhamster.com/pornstars/cailey-taylor'}, - { 'name': 'Eve Laurence', 'url': 'https://ge.xhamster.com/pornstars/eve-laurence'}, - { 'name': 'Naudia Nyce', 'url': 'https://ge.xhamster.com/pornstars/naudia-nyce'}, - { 'name': 'Candy Apples', 'url': 'https://ge.xhamster.com/pornstars/candy-apples'}, - { 'name': 'Kate Rich', 'url': 'https://ge.xhamster.com/pornstars/kate-rich'}, - { 'name': 'Aniko Kaposi', 'url': 'https://ge.xhamster.com/pornstars/aniko-kaposi'}, - { 'name': 'Beatrice Poggi', 'url': 'https://ge.xhamster.com/pornstars/beatrice-poggi'}, - { 'name': 'Erika Bella', 'url': 'https://ge.xhamster.com/pornstars/erika-bella'}, - { 'name': 'Nikita Gross', 'url': 'https://ge.xhamster.com/pornstars/nikita-gross'}, - { 'name': 'Ursula Moore', 'url': 'https://ge.xhamster.com/pornstars/ursula-moore'}, - { 'name': 'Caty Kiss', 'url': 'https://ge.xhamster.com/pornstars/caty-kiss'}, - { 'name': 'Light Fairy', 'url': 'https://ge.xhamster.com/pornstars/light-fairy'}, - { 'name': 'Flame', 'url': 'https://ge.xhamster.com/pornstars/flame'}, - { 'name': 'Tiffany Tatum', 'url': 'https://ge.xhamster.com/pornstars/tiffany-tatum'}, - { 'name': 'Alyson Sykes', 'url': 'https://ge.xhamster.com/pornstars/alyson-sykes'}, - { 'name': 'Jenifer Stone', 'url': 'https://ge.xhamster.com/pornstars/jenifer-stone'}, - { 'name': 'Lucy Love', 'url': 'https://ge.xhamster.com/pornstars/lucy-love'}, - { 'name': 'Thomas Stone', 'url': 'https://ge.xhamster.com/pornstars/thomas-stone'}, - { 'name': 'Nikki Babe', 'url': 'https://ge.xhamster.com/pornstars/nikki-babe'}, - { 'name': 'Tyra Misoux', 'url': 'https://ge.xhamster.com/pornstars/tyra-misoux'}, - { 'name': 'Kenzie Reeves', 'url': 'https://ge.xhamster.com/pornstars/kenzie-reeves'}, - { 'name': 'Anissa Kate', 'url': 'https://ge.xhamster.com/pornstars/anissa-kate'}, - { 'name': 'Anna Polina', 'url': 'https://ge.xhamster.com/pornstars/anna-polina'}, - { 'name': 'Kimber Delice', 'url': 'https://ge.xhamster.com/pornstars/kimber-delice'}, - { 'name': 'Lucy Heart', 'url': 'https://ge.xhamster.com/pornstars/lucy-heart'}, - { 'name': 'John Strong', 'url': 'https://ge.xhamster.com/pornstars/john-strong'}, - { 'name': 'Markus Dupree', 'url': 'https://ge.xhamster.com/pornstars/markus-dupree'}, - { 'name': 'Mick Blue', 'url': 'https://ge.xhamster.com/pornstars/mick-blue'}, - { 'name': 'Natasha Nice', 'url': 'https://ge.xhamster.com/pornstars/natasha-nice'}, - { 'name': 'Lena Reif', 'url': 'https://ge.xhamster.com/pornstars/lena-reif'}, - { 'name': 'Sonia Paganini', 'url': 'https://ge.xhamster.com/pornstars/sonia-paganini'}, - { 'name': 'Demi Hawks', 'url': 'https://ge.xhamster.com/pornstars/demi-hawks'}, - { 'name': 'Juan El Caballo Loco', 'url': 'https://ge.xhamster.com/pornstars/juan-el-caballo-loco'}, - { 'name': 'Mike Mancini', 'url': 'https://ge.xhamster.com/pornstars/mike-mancini'}, - { 'name': 'Millie Morgan', 'url': 'https://ge.xhamster.com/pornstars/millie-morgan'}, - { 'name': 'Richard Allan', 'url': 'https://ge.xhamster.com/pornstars/richard-allan'}, - { 'name': 'Damon Dice', 'url': 'https://ge.xhamster.com/pornstars/damon-dice'}, - { 'name': 'Sera Ryder', 'url': 'https://ge.xhamster.com/pornstars/sera-ryder'}, - { 'name': 'Zoltan Kaabai', 'url': 'https://ge.xhamster.com/pornstars/zoltan-kabai'}, - { 'name': 'Cathy Heaven', 'url': 'https://ge.xhamster.com/pornstars/cathy-heaven'}, - { 'name': 'Coco Lovelock', 'url': 'https://ge.xhamster.com/pornstars/coco-lovelock'}, - { 'name': 'Percy Sires', 'url': 'https://ge.xhamster.com/pornstars/percy-sires'}, - { 'name': 'Meridian', 'url': 'https://ge.xhamster.com/pornstars/meridian'}, - { 'name': 'Pascal St. James', 'url': 'https://ge.xhamster.com/pornstars/pascal-st-james'}, - { 'name': 'Red Fox', 'url': 'https://ge.xhamster.com/pornstars/red-fox'}, - { 'name': 'Tony Art', 'url': 'https://ge.xhamster.com/pornstars/tony-art'}, - { 'name': 'Addison Lee', 'url': 'https://ge.xhamster.com/pornstars/addison-lee'}, - { 'name': 'Daria Glover', 'url': 'https://ge.xhamster.com/pornstars/daria-glover'}, - { 'name': 'Mandy Bright', 'url': 'https://ge.xhamster.com/pornstars/mandy-bright'}, - { 'name': 'Antonia Sainz', 'url': 'https://ge.xhamster.com/pornstars/antonia-sainz'}, - { 'name': 'Nicole Love', 'url': 'https://ge.xhamster.com/pornstars/nicole-love'}, - { 'name': 'Sarah Kay', 'url': 'https://ge.xhamster.com/pornstars/sarah-kay'}, - { 'name': 'Judith Kostner', 'url': 'https://ge.xhamster.com/pornstars/judith-kostner'}, - { 'name': 'Maria Bellucci', 'url': 'https://ge.xhamster.com/pornstars/maria-bellucci'}, - { 'name': 'Melissa Monet', 'url': 'https://ge.xhamster.com/pornstars/melissa-monet'}, - { 'name': 'Stephanie Cane', 'url': 'https://ge.xhamster.com/pornstars/stephanie-cane'}, - { 'name': 'Will Steiger', 'url': 'https://ge.xhamster.com/pornstars/will-steiger'}, - { 'name': 'Katty West', 'url': 'https://ge.xhamster.com/pornstars/katty-west'}, - { 'name': 'Jean Pallett', 'url': 'https://ge.xhamster.com/pornstars/jean-pallett'}, - { 'name': 'Conny Dachs', 'url': 'https://ge.xhamster.com/pornstars/conny-dachs'}, - { 'name': 'Juicy Leyla', 'url': 'https://ge.xhamster.com/pornstars/juicy-leyla'}, - { 'name': 'Mandy Mystery', 'url': 'https://ge.xhamster.com/pornstars/mandy-mystery'}, - { 'name': 'Jack Vegas', 'url': 'https://ge.xhamster.com/pornstars/jack-vegas'}, - { 'name': 'Jessica Ryan', 'url': 'https://ge.xhamster.com/pornstars/jessica-ryan'}, - { 'name': 'Nathan Bronson', 'url': 'https://ge.xhamster.com/pornstars/nathan-bronson'}, - { 'name': 'Lexi Lore', 'url': 'https://ge.xhamster.com/pornstars/lexi-lore'}, - { 'name': 'Molly Little', 'url': 'https://ge.xhamster.com/pornstars/molly-little'}, - { 'name': 'Dolly Leigh', 'url': 'https://ge.xhamster.com/pornstars/dolly-leigh'}, - { 'name': 'Marcus London', 'url': 'https://ge.xhamster.com/pornstars/marcus-london'}, - { 'name': 'Kendra Sunderland', 'url': 'https://ge.xhamster.com/pornstars/kendra-sunderland'}, - { 'name': 'Manuel Ferrara', 'url': 'https://ge.xhamster.com/pornstars/manuel-ferrara'}, - { 'name': 'Kelly Trump', 'url': 'https://ge.xhamster.com/pornstars/kelly-trump'}, - { 'name': 'Angelica Heaven', 'url': 'https://ge.xhamster.com/pornstars/angelica-heaven'}, - { 'name': 'Luna Lynx', 'url': 'https://ge.xhamster.com/pornstars/luna-lynx'}, - { 'name': 'Princess Lili', 'url': 'https://ge.xhamster.com/pornstars/princess-lili'}, - { 'name': 'Alexa Wild', 'url': 'https://ge.xhamster.com/pornstars/alexa-wild'}, - { 'name': 'Jessyka Swan', 'url': 'https://ge.xhamster.com/pornstars/jessyka-swan'}, - { 'name': 'Nikky Thorne', 'url': 'https://ge.xhamster.com/pornstars/nikky-thorne'}, - { 'name': 'Tristan Summers', 'url': 'https://ge.xhamster.com/pornstars/tristan-summers'}, - { 'name': 'Rhaya Shyne', 'url': 'https://ge.xhamster.com/pornstars/rhaya-shyne'}, - { 'name': 'Desiree West', 'url': 'https://ge.xhamster.com/pornstars/desiree-west'}, - { 'name': 'Joan Devlon', 'url': 'https://ge.xhamster.com/pornstars/joan-devlon'}, - { 'name': 'Jodi Thorpe', 'url': 'https://ge.xhamster.com/pornstars/jodi-thorpe'}, - { 'name': 'Laurien Dominique', 'url': 'https://ge.xhamster.com/pornstars/laurien-dominique'}, - { 'name': 'Paul Scharf', 'url': 'https://ge.xhamster.com/pornstars/paul-scharf'}, - { 'name': 'Ray Wells', 'url': 'https://ge.xhamster.com/pornstars/ray-wells'}, - { 'name': 'Sandy Carey', 'url': 'https://ge.xhamster.com/pornstars/sandy-carey'}, - { 'name': 'Spender Travis', 'url': 'https://ge.xhamster.com/pornstars/spender-travis'}, - { 'name': 'Starlyn Simone', 'url': 'https://ge.xhamster.com/pornstars/starlyn-simone'}, - { 'name': 'Uschi Digard', 'url': 'https://ge.xhamster.com/pornstars/uschi-digard'}, - { 'name': 'Katarina Muti', 'url': 'https://ge.xhamster.com/pornstars/katarina-muti'}, - { 'name': 'Julia Power', 'url': 'https://ge.xhamster.com/pornstars/julia-power'}, - { 'name': 'Salma De Nora', 'url': 'https://ge.xhamster.com/pornstars/salma-de-nora'}, - { 'name': 'Valeria Jones', 'url': 'https://ge.xhamster.com/pornstars/valeria-jones'}, - { 'name': 'Brandy Canyon', 'url': 'https://ge.xhamster.com/pornstars/brandy-canyon'}, - { 'name': 'Marie Berger', 'url': 'https://ge.xhamster.com/pornstars/marie-berger'}, - { 'name': 'Luna Rishi', 'url': 'https://ge.xhamster.com/pornstars/luna-rishi'}, - { 'name': 'Colleen Brennan', 'url': 'https://ge.xhamster.com/pornstars/colleen-brennan'}, - { 'name': 'Roxanne Brewer', 'url': 'https://ge.xhamster.com/pornstars/roxanne-brewer'}, - { 'name': 'Elle Denay', 'url': 'https://ge.xhamster.com/pornstars/elle-denay'}, - { 'name': 'Juan Largo', 'url': 'https://ge.xhamster.com/pornstars/juan-largo'}, - { 'name': 'Codey Steele', 'url': 'https://ge.xhamster.com/pornstars/codey-steele'}, - { 'name': 'Laney Grey', 'url': 'https://ge.xhamster.com/pornstars/laney-grey'}, - { 'name': 'Stirling Cooper', 'url': 'https://ge.xhamster.com/pornstars/stirling-cooper'}, - { 'name': 'Lana Smalls', 'url': 'https://ge.xhamster.com/pornstars/lana-smalls'}, - { 'name': 'Alex Sanders', 'url':'https://ge.xhamster.com/pornstars/alex-sanders'}, - { 'name': 'Felecia Danay', 'url':'https://ge.xhamster.com/pornstars/felecia-danay'}, - { 'name': 'Lita Chase', 'url':'https://ge.xhamster.com/pornstars/lita-chase'}, - { 'name': 'Mark Ashley', 'url':'https://ge.xhamster.com/pornstars/mark-ashley'}, - { 'name': 'Phyllisha Anne', 'url':'https://ge.xhamster.com/pornstars/phyllisha-anne'}, - { 'name': 'Ryan Conner', 'url':'https://ge.xhamster.com/pornstars/ryan-conner'}, - { 'name': 'Tanya Danielle', 'url':'https://ge.xhamster.com/pornstars/tanya-danielle'}, - { 'name': 'Jessica Moore', 'url':'https://ge.xhamster.com/pornstars/jessica-moore'}, - { 'name': 'Mike Angelo', 'url':'https://ge.xhamster.com/pornstars/mike-angelo'}, - { 'name': 'Morgan Moon', 'url':'https://ge.xhamster.com/pornstars/morgan-moon'}, - { 'name': 'Tyler Steel', 'url':'https://ge.xhamster.com/pornstars/tyler-steel'}, - { 'name': 'Abella Danger', 'url':'https://ge.xhamster.com/pornstars/abella-danger'}, - { 'name': 'Alex Jett', 'url':'https://ge.xhamster.com/pornstars/alex-jett'}, - { 'name': 'Alyson Queen', 'url':'https://ge.xhamster.com/pornstars/alyson-queen'}, - { 'name': 'Antynia Rouge', 'url':'https://ge.xhamster.com/pornstars/antynia-rouge'}, - { 'name': 'Bea Dumas', 'url':'https://ge.xhamster.com/pornstars/bea-dumas'}, - { 'name': 'Callie Black', 'url':'https://ge.xhamster.com/pornstars/callie-black'}, - { 'name': 'Caroline Cage', 'url':'https://ge.xhamster.com/pornstars/caroline-cage'}, - { 'name': 'Cindy Dollar', 'url':'https://ge.xhamster.com/pornstars/cindy-dollar'}, - { 'name': 'Crystal Frost', 'url':'https://ge.xhamster.com/pornstars/crystal-frost'}, - { 'name': 'Fanny Steel', 'url':'https://ge.xhamster.com/pornstars/fanny-steel'}, - { 'name': 'Gia Derza', 'url':'https://ge.xhamster.com/pornstars/gia-derza'}, - { 'name': 'Horst Baron', 'url':'https://ge.xhamster.com/pornstars/horst-baron'}, - { 'name': 'Jasmine Rouge', 'url':'https://ge.xhamster.com/pornstars/jasmine-rouge'}, - { 'name': 'Jean-Pierre Armand', 'url':'https://ge.xhamster.com/pornstars/jean-pierre-armand'}, - { 'name': 'Jessa Rhodes', 'url':'https://ge.xhamster.com/pornstars/jessa-rhodes'}, - { 'name': 'Leonie Saint', 'url':'https://ge.xhamster.com/pornstars/leonie-saint'}, - { 'name': 'Linda Ray', 'url':'https://ge.xhamster.com/pornstars/linda-ray'}, - { 'name': 'Luca Ferrero', 'url':'https://ge.xhamster.com/pornstars/luca-ferrero'}, - { 'name': 'Paris Pink', 'url':'https://ge.xhamster.com/pornstars/paris-pink'}, - { 'name': 'Pavlina Stejskalova', 'url':'https://ge.xhamster.com/pornstars/pavlina-stejskalova'}, - { 'name': 'Phoenix Marie', 'url':'https://ge.xhamster.com/pornstars/phoenix-marie'}, - { 'name': 'Ricky Spanish', 'url':'https://ge.xhamster.com/pornstars/ricky-spanish'}, - { 'name': 'Rumika Powers', 'url':'https://ge.xhamster.com/pornstars/rumika-powers'}, - { 'name': 'Sara Blonde', 'url':'https://ge.xhamster.com/pornstars/sara-blonde'}, - { 'name': 'Sean Lawless', 'url':'https://ge.xhamster.com/pornstars/sean-lawless'}, - { 'name': 'Seth Gamble', 'url':'https://ge.xhamster.com/pornstars/seth-gamble'}, - { 'name': 'Siri Dahl', 'url':'https://ge.xhamster.com/pornstars/siri-dahl'}, - { 'name': 'Stephie Staar', 'url':'https://ge.xhamster.com/pornstars/stephie-staar'}, - { 'name': 'Steve Holmes', 'url':'https://ge.xhamster.com/pornstars/steve-holmes'}, - { 'name': 'Suzette Dale', 'url':'https://ge.xhamster.com/pornstars/suzette-dale'}, - { 'name': 'Uncle George', 'url':'https://ge.xhamster.com/pornstars/uncle-george'}, - { 'name': 'Winnie', 'url':'https://ge.xhamster.com/pornstars/winnie'}, - { 'name': 'Zenza Raggi', 'url':'https://ge.xhamster.com/pornstars/zenza-raggi'}, - { 'name': 'Zorah White', 'url':'https://ge.xhamster.com/pornstars/zorah-white'}, - { 'name': 'Marilyn Jess', 'url':'https://ge.xhamster.com/pornstars/marilyn-jess'}, - { 'name': 'Alexis Capri', 'url':'https://ge.xhamster.com/pornstars/alexis-capri'}, - ] - - - for new_actor in new_actor_list: - if new_actor['url'] in actors: - log.warning(f"Actor {new_actor['url']} already persisted") - continue - actor_response = requests.post(f"http://127.0.0.1:8800/api/media/actors", json=new_actor) - log.warning(f"add status: {actor_response.status_code}") - if actor_response.status_code == 201: - log.warning(f"add Actor {new_actor['url']} to existing actor list") - actors[new_actor['url']] = new_actor - actor_data = actor_response.json() - log.warning(f"Actor {actor_data} persisted") - log.warning('kontor.add_actors finished') diff --git a/kontor-scripts/copy_to_mariadb.py b/kontor-scripts/copy_to_mariadb.py deleted file mode 100644 index 059936a..0000000 --- a/kontor-scripts/copy_to_mariadb.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -copy data from SQLite to MariaDB -""" -import sqlite3 -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter - -from config import get_logger, get_database_cursors, get_meta_data, get_scripts - -parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) -parser.add_argument('--recreate-db', action='store_true') -parser.add_argument('--verbose', '-v', action='count', default=0) -args = parser.parse_args() - - -def copy_data(mariadb_conn, sqlite_conn, table_scripts): - mariadb_cursor = mariadb_conn.cursor() - sqlite_cursor = sqlite_conn.cursor() - # logger.info(table_scripts) - for table_id in scripts: - select_statement = scripts[table_id]['select_sqlite'] - # logger.info(select_statement) - insert_statement = scripts[table_id]['insert_mariadb'] - mariadb_cursor.execute("SET FOREIGN_KEY_CHECKS = 0") - mariadb_cursor.execute(scripts[table_id]['truncate_mariadb']) - try: - sqlite_cursor.execute(select_statement) - rows = sqlite_cursor.fetchall() - for row in rows: - try: - mariadb_cursor.execute(insert_statement, row) - except sqlite3.Error as error: - logger.info('insert failed with %s\n%s\n%s', error, insert_statement, row) - mariadb_conn.commit() - mariadb_cursor.execute(scripts[table_id]['count']) - (number_of_rows,) = mariadb_cursor.fetchone() - row = sqlite_cursor.execute(scripts[table_id]['count']).fetchone() - logger.info('%s contains %d : %d entries', scripts[table_id]['name'], number_of_rows, row[0]) - except sqlite3.Error as error: - logger.info('select failed with %s', error) - - -if __name__ == '__main__': - logger = get_logger(args.verbose) - logger.info('kontor.copy_to_sqlite started') - s_conn, m_conn = get_database_cursors(logger) - meta_data_tables = get_meta_data(m_conn) - # logger.info(meta_data_tables) - scripts = get_scripts(meta_data_tables, logger) - copy_data(m_conn, s_conn, scripts) - s_conn.close() - m_conn.close() - logger.info('kontor.copy_to_sqlite finished') diff --git a/kontor-scripts/copy_to_sqlite.py b/kontor-scripts/copy_to_sqlite.py deleted file mode 100644 index 3455cb4..0000000 --- a/kontor-scripts/copy_to_sqlite.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -copy data from MariaDB to SQLite -""" -import sqlite3 -import mariadb -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -from config import get_database_cursors, create_tables, get_logger, get_meta_data, get_scripts - -parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) -parser.add_argument('--recreate-db', action='store_true') -parser.add_argument('--verbose', '-v', action='count', default=0) -args = parser.parse_args() - - -def copy_data(mariadb_conn, sqlite_conn, table_scripts): - mariadb_cursor = mariadb_conn.cursor() - sqlite_cursor = sqlite_conn.cursor() - # logger.info(table_scripts) - for table_id in table_scripts: - select_statement = scripts[table_id]['select_mariadb'] - # logger.info(select_statement) - insert_statement = scripts[table_id]['insert_sqlite'] - try: - mariadb_cursor.execute(select_statement) - rows = mariadb_cursor.fetchall() - for row in rows: - try: - sqlite_cursor.execute(insert_statement, row) - except sqlite3.Error as error: - logger.info('insert failed with %s\n%s\n%s', error, insert_statement, row) - sqlite_conn.commit() - mariadb_cursor.execute(scripts[table_id]['count']) - (number_of_rows,) = mariadb_cursor.fetchone() - row = sqlite_cursor.execute(scripts[table_id]['count']).fetchone() - logger.info('%s contains %d : %d entries', scripts[table_id]['name'], number_of_rows, row[0]) - except mariadb.Error as error: - logger.info('select failed with %s', error) - - -if __name__ == '__main__': - logger = get_logger(args.verbose) - logger.info('kontor.copy_to_sqlite started') - s_conn, m_conn = get_database_cursors(logger) - meta_data_tables = get_meta_data(m_conn) - # logger.info(meta_data_tables) - scripts = get_scripts(meta_data_tables, logger) - create_tables(s_conn, logger, args.recreate_db, scripts) - copy_data(m_conn, s_conn, scripts) - s_conn.close() - m_conn.close() - logger.info('kontor.copy_to_sqlite finished') diff --git a/kontor-scripts/db_structure.py b/kontor-scripts/db_structure.py deleted file mode 100644 index 947ab4a..0000000 --- a/kontor-scripts/db_structure.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -Prints the database kontor structure -""" -import mariadb -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -from config import get_database_cursors, get_logger - -parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) -parser.add_argument('--verbose', '-v', action='count', default=0) -args = parser.parse_args() - - -def show_tables(cur, log): - """ - Retrieves the list of tables from the database - :param cur: - :param log: - :return: - """ - log.info('get list of tables') - table_list = [] - cur.execute("SHOW TABLES") - for (tablename,) in cur.fetchall(): - table_list.append(tablename) - return table_list - - -def get_field_info(cur): - """ - Retrieves the field info associated with a cursor - :param cur: - :return: - """ - field_info = mariadb.fieldinfo() - field_info_text_list = [] - for column in cur.description: - column_name = column[0] - column_type = field_info.type(column) - column_flags = field_info.flag(column) - field_info_text_list.append(f"{column_name}: {column_type} {column_flags}") - return field_info_text_list - - -def get_table_field_info(cur, tablename): - """ - Retrieves the field info associated with a table - :param cur: - :param tablename: - :return: - """ - cur.execute(f"SELECT * FROM {tablename} LIMIT 1") - field_info = get_field_info(cur) - return field_info - - -if __name__ == '__main__': - logger = get_logger(args.verbose) - logger.info("kontor.db_structure started") - _, mariadb_conn = get_database_cursors(logger) - tables = show_tables(mariadb_conn.cursor(), logger) - for table in tables: - field_info_text = get_table_field_info(mariadb_conn.cursor(), table) - print(f"Columns in table {table}:") - print("\n".join(field_info_text)) - print("\n") - mariadb_conn.close() - logger.info("kontor.db_structure finished") diff --git a/kontor-scripts/json_to_mariadb.py b/kontor-scripts/json_to_mariadb.py deleted file mode 100644 index db3916b..0000000 --- a/kontor-scripts/json_to_mariadb.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -copy data from JSON to MariaDB -""" -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -from pathlib import Path -from config import get_logger, get_database_cursors -import mariadb -import json - -parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) -parser.add_argument('--verbose', '-v', action='count', default=0) -parser.add_argument('--config', '-c', default='kontor-docker') -parser.add_argument('--file', '-f', default='~/.sync/media/data.json') -args = parser.parse_args() - -def copy_data(mariadb_conn, data_file: Path, log): - mariadb_cursor = mariadb_conn.cursor() - import_file = Path(data_file) - if not import_file.exists(): - log.info(f"File {data_file} does not exist. Do nothing.") - return - log.info("read json file") - with open(data_file, 'r') as json_file: - json_load = json.load(json_file) - for table in json_load: - log.info(f"{table}: {len(json_load[table])}") - # result[table] = import_table(table, json_load[table]) - truncate_statement = 'TRUNCATE {}'.format(table) - #log.info(f"truncate: {truncate_statement}") - mariadb_cursor.execute("SET FOREIGN_KEY_CHECKS = 0") - mariadb_cursor.execute(truncate_statement) - items = json_load[table] - for item in items: - #log.info(f"item: {item}") - values = [] - columns = [] - for (key, value) in item.items(): - columns.append(key) - values.append(value) - row = tuple(values) - log.info(f"values: {row}") - insert_statement = 'INSERT INTO {}({}) VALUES({})'.format(table, ', '.join(columns), ', '.join(['?']*len(columns))) - #log.info(f"statement: {insert_statement}") - mariadb_cursor.execute(insert_statement, row) - try: - mariadb_conn.commit() - except mariadb.Error as error: - log.info('insert failed with %s', error) - - -if __name__ == '__main__': - logger = get_logger(args.verbose, args.config) - logger.info('kontor.json_to_mariadb started') - _, m_conn = get_database_cursors(logger, args.config) - copy_data(m_conn, args.file, logger) - m_conn.close() - logger.info('kontor.json_to_mariadb finished') - diff --git a/kontor-scripts/json_to_postgres.py b/kontor-scripts/json_to_postgres.py deleted file mode 100644 index 546597c..0000000 --- a/kontor-scripts/json_to_postgres.py +++ /dev/null @@ -1,152 +0,0 @@ -""" -copy data from JSON to Postgres -""" -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -from pathlib import Path -from typing import Dict, List - -from config import get_logger, get_database_cursors -import json -from psycopg2.sql import SQL - -parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) -parser.add_argument('--verbose', '-v', action='count', default=0) -parser.add_argument('--config', '-c', default='kontor-docker') -parser.add_argument('--file', '-f', default='~/.sync/media/data.json') -args = parser.parse_args() - -def copy_data(postgres_conn, data_file: Path, log): - postgres_cursor = postgres_conn.cursor() - import_file = Path(data_file) - if not import_file.exists(): - log.info(f"File {data_file} does not exist. Do nothing.") - return - log.info("read json file") - with open(data_file, 'r') as json_file: - json_load = json.load(json_file) - postgres_cursor.execute("SET session_replication_role='replica'") - for table in json_load: - log.info(f"{table}: {len(json_load[table])}") - # result[table] = import_table(table, json_load[table]) - truncate_statement = 'TRUNCATE {} CASCADE'.format(table) - #log.info(f"truncate: {truncate_statement}") - try: - postgres_cursor.execute(truncate_statement) - except: - log.info(f"statement: {insert_statement} FAILED") - items = json_load[table] - for item in items: - #log.info(f"item: {item}") - values = [] - columns = [] - for (key, value) in item.items(): - columns.append(key) - values.append(value) - row = tuple(values) - #log.info(f"values: {row}") - insert_statement = 'INSERT INTO {}({}) VALUES({})'.format(table, ', '.join(columns), ', '.join(['%s']*len(columns))) - #log.info(f"statement: {insert_statement}") - try: - postgres_cursor.execute(SQL(insert_statement), row) - postgres_conn.commit() - except: - log.info(f'insert failed with {insert_statement}') - postgres_cursor.execute("SET session_replication_role='origin'") - - -def load_json(data_file, log) -> dict: - import_file = Path(data_file) - if not import_file.exists(): - log.info(f"File {data_file} does not exist. Do nothing.") - return - log.info("read json file") - with open(data_file, 'r') as json_file: - json_load = json.load(json_file) - return json_load - - -def insert_data(postgres_conn, data: dict, log): - postgres_cursor = postgres_conn.cursor() - log.info("insert data") - table_list = [] - table_list = ['worktype', 'artist', 'publisher', 'volume', 'comic', 'issue', 'story_arc', 'trade_paperback', 'comic_work'] - table_list.extend(['sport', 'team', 'field_position', 'vendor', 'player', 'rooster', 'card_set', 'card']) - #table_list.extend(['card']) - table_list.extend(['media_file', 'media_video', 'media_actor', 'media_article', 'media_actor_file']) - #table_list.extend(['media_actor_file']) - table_list.extend(['profile', 'permission', 'token', 'assignment']) - table_list.extend(['mail', 'mail_account', 'module_data', 'meta_data_table', 'meta_data_column']) - table_list.extend(['meta_data_table', 'meta_data_column']) - table_list.extend(['book', 'author', 'article', 'bookshelf_publisher', 'book_author', 'article_author']) - #if len(table_list) != 37: - # log.info(f"number of tables incorrect: {len(table_list)}") - # return - for table in table_list: - log.info(f"{table}: {len(data[table])}") - truncate_statement = 'DELETE FROM {}'.format(table) - log.info(f"truncate: {truncate_statement}") - try: - postgres_cursor.execute(truncate_statement) - postgres_conn.commit() - except: - log.info(f"statement: {truncate_statement} FAILED") - items = data[table] - for item in items: - # log.info(f"item: {item}") - values = [] - columns = [] - for (key, value) in item.items(): - columns.append(key) - values.append(value) - row = tuple(values) - # log.info(f"values: {row}") - insert_statement = 'INSERT INTO {}({}) VALUES({})'.format(table, ', '.join(columns), - ', '.join(['%s'] * len(columns))) - # log.info(f"statement: {insert_statement}") - try: - postgres_cursor.execute(SQL(insert_statement), row) - postgres_conn.commit() - except: - log.info(f'insert failed with {insert_statement}') - - -def parse_table_order(data: dict, log): - log.info("parse_table_order") - table_refs: Dict[str, List[str]] = {} - for table in data: - log.info(f"{table}: {len(data[table])}") - items = data[table] - table_refs[table] = [] - if len(items) == 0: - continue - item = items[0] - for key, _ in item.items(): - if key.endswith('_id'): - ref = key[0:-3] - log.info(f"table {table} has reference to {ref}") - if table in table_refs: - table_refs[table].append(ref) - else: - table_refs[table] = [ref] - log.info(f"parsed refs: {table_refs}") - table_order = [] - for table in table_refs: - if len(table_refs[table]) == 0: - log.info(f"insert {table} at beginning") - table_order.insert(0, table) - else: - log.info(f"insert {table} at end") - table_order.append(table) - log.info(f"table_list: {len(table_order)}: {table_order}") - -if __name__ == '__main__': - logger = get_logger(args.verbose, args.config) - logger.info('kontor.json_to_postgres started') - _, _, p_conn = get_database_cursors(logger, args.config) - data = load_json(args.file, logger) - #parse_table_order(data, logger) - insert_data(p_conn, data, logger) - #copy_data(p_conn, args.file, logger) - p_conn.close() - logger.info('kontor.json_to_postgres finished') - diff --git a/kontor-scripts/kontor.py b/kontor-scripts/kontor.py deleted file mode 100644 index 217e15a..0000000 --- a/kontor-scripts/kontor.py +++ /dev/null @@ -1,64 +0,0 @@ -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -from pathlib import Path - -import logging.config -import yaml -from platformdirs import PlatformDirs -from sqlalchemy import create_engine, select -from sqlalchemy.orm import sessionmaker - -from schema import Comic, Publisher, Base - -parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) -parser.add_argument('--recreate-db', action='store_true') -parser.add_argument('--verbose', '-v', action='count', default=0) -parser.add_argument('--file', '-f', default='~/data.json') -parser.add_argument('--config', '-c', default='kontor-docker') -args = parser.parse_args() - - -def get_logger(level: int, config: str): - dirs = PlatformDirs(config) - logging_config = Path(dirs.user_config_dir, 'logging-config.yaml') - with open(logging_config, 'rt') as f: - configDict = yaml.safe_load(f.read()) - logging.config.dictConfig(configDict) - logger = logging.getLogger('development') - if level is not None: - match level: - case 0: - logger.setLevel(logging.INFO) - case 1: - logger.setLevel(logging.DEBUG) - case _: - logger.setLevel(logging.CRITICAL) - return logger - - -if __name__ == '__main__': - log = get_logger(args.verbose, args.config) - log.info('kontor started') - dirs = PlatformDirs(args.config) - database_config = Path(dirs.user_config_dir, 'database-config.yaml') - with open(database_config, 'rt') as f: - db_config = yaml.safe_load(f.read()) - print(db_config) - connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format( - db_config['mariadb']['user'], - db_config['mariadb']['password'], - db_config['mariadb']['host'], - db_config['mariadb']['port'], - db_config['mariadb']['database'] - )) - engine = create_engine(connect_string) - Base.metadata.create_all(bind=engine, checkfirst=True) - __session__ = sessionmaker(engine) - with __session__() as session: - comics = session.scalars(select(Comic)).all() - for comic in comics: - print(comic) - publishers = session.scalars(select(Publisher)).all() - for publisher in publishers: - print(publisher) - print(len(publisher.comics)) - log.info('kontor finished') diff --git a/kontor-scripts/read_list.py b/kontor-scripts/read_list.py deleted file mode 100644 index 4d5ba6e..0000000 --- a/kontor-scripts/read_list.py +++ /dev/null @@ -1,80 +0,0 @@ -""" -read file with URLs and store in DB -""" -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter -import logging -import json -from proton import Message, Event -from proton.handlers import MessagingHandler -from proton.reactor import Container - -parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) -parser.add_argument('-f', '--links', help='file with links') -parser.add_argument('--verbose', '-v', action='count', default=0) -args = parser.parse_args() - - -def read_links_file(links_file): - with open(links_file, 'r') as input_file: - lines = input_file.readlines() - return lines - - -class AddLinkMessage(MessagingHandler): - def __init__(self, server, url, log): - super(AddLinkMessage, self).__init__() - log.info("create AddLinkMessage") - self.server = server - self.address = "KontorMediaFile::add_link_file" - self.url = url - self.log = log - - def on_start(self, event: Event): - self.log.info("Connection...") - conn = event.container.connect(self.server, user="artemis", password="artemis") - event.container.create_sender(conn, self.address) - - def on_connection_opened(self, event: Event) -> None: - self.log.debug("connection open") - - def on_connection_error(self, event: Event) -> None: - self.log.info(f"error: {event}") - - def on_disconnected(self, event: Event) -> None: - self.log.debug(f"disconnected: {repr(event)}") - - - def on_sendable(self, event: Event): - self.log.info("send message") - message = Message(body=self.url, address=self.address, content_type="application/json", durable=True) - delivery = event.sender.send(message) - self.log.info(f"Delivery {delivery} sent") - event.connection.close() - - def on_accepted(self, event: Event) -> None: - self.log.info(f"accepted Delivery: {event.delivery.remote_state}") - - - def on_rejected(self, event: Event) -> None: - self.log.info(f"rejected Delivery: {event.delivery}") - -if __name__ == '__main__': - logging.basicConfig(level=logging.INFO, format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s') - logging.info('kontor.read_list started') - #conn = stomp.Connection([('127.0.0.1', '61616')]) - #conn.connect('artemis', 'artemis', wait=True) - if args.links: - logging.info("read links from file") - links = read_links_file(args.links) - for link in links: - data_dict = {'url': link.strip()} - data = json.dumps(data_dict) - logging.info("send link message") - handler = AddLinkMessage("amqp://127.0.0.1:5672", data, logging) - container = Container(handler) - container.container_id = "process_add_links" - container.run() - # conn.send(body=data, destination='KontorMediaFile::add_link_file', headers={'content-type': 'application/json'}) - #conn.disconnect() - logging.info('kontor.read_list finished') - diff --git a/kontor-scripts/requirements.txt b/kontor-scripts/requirements.txt deleted file mode 100644 index 30839c8..0000000 --- a/kontor-scripts/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -sqlalchemy -pathlib -platformdirs -pyyaml -beautifulsoup4 -sqlmodel -requests -fastapi[standard] - diff --git a/kontor-scripts/sync.py b/kontor-scripts/sync.py new file mode 100644 index 0000000..3f42f2e --- /dev/null +++ b/kontor-scripts/sync.py @@ -0,0 +1,117 @@ +from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser +from dataclasses import dataclass +import logging +import logging.config +from logging import Logger +from pathlib import Path +from typing import Dict, List + +from platformdirs import PlatformDirs +import requests +import yaml + + +parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) +parser.add_argument("--verbose", "-v", action="count", default=0) +parser.add_argument("--config", "-c", default="kontor-api") +parser.add_argument("--dry-run", "-m", action="store_true") +parser.add_argument("--cleanup", "-d", action="store_true") +args = parser.parse_args() + + +@dataclass +class Login: + email: str + password: str + + +@dataclass +class Server: + name: str + url: str + token: str + token_type: str + + def login(self, login: Login, log: Logger): + if not self.token: + log.info("Call login first") + login_url = f"{self.url}/login" + login_data = {} + login_data["email"] = login.email + login_data["password"] = login.password + response = requests.post(login_url, json=login_data) + status = response.status_code + log.info(f"Status: {status}") + if status != 200: + log.fatal("authentication failed") + return + data = response.json() + log.debug(f"got data: {data}") + token = data["access_token"] + token_type = data["token_type"] + self.token = str(token) + self.token_type = str(token_type) + + def request(self, log: Logger): + url: str = f"{self.url}/api/media/files" + headers: Dict[str, str] = {"Authorization": f"Bearer {self.token}"} + response = requests.get(url, headers=headers) + log.info(f"Status: {response.status_code}") + data = response.json() + return data + + +@dataclass +class ApiConfig: + login: Login + server: List[Server] + +def get_logger(level, config: str): + dirs = PlatformDirs(config) + logging_config = Path(dirs.user_config_dir, "logging-config.yaml") + log_config = None + with open(logging_config, "rt") as f: + log_config = yaml.safe_load(f.read()) + logging.config.dictConfig(log_config) + logger = logging.getLogger("development") + if level is not None: + match level: + case 0: + logger.setLevel(logging.CRITICAL) + case 1: + logger.setLevel(logging.INFO) + case 2: + logger.setLevel(logging.DEBUG) + case _: + logger.setLevel(logging.INFO) + return logger + + +def get_api_config(log: Logger, config: str) -> ApiConfig: + # api_data: Dict[str, Any] = {} + dirs = PlatformDirs(config) + api_config = Path(dirs.user_config_dir, "api.yaml") + with open(api_config, "rt") as f: + api_data = yaml.safe_load(f.read()) + servers = [Server(**server) for server in api_data['server']] + login = Login(**(api_data["login"])) + apiConfig = ApiConfig(server=servers, login=login) + log.info(apiConfig) + if not api_data: + log.fatal("API configuration is missing") + return apiConfig + for server in apiConfig.server: + server.login(apiConfig.login, log) + with open(api_config, "w") as f: + yaml.dump(api_data, f) + return apiConfig + + +if __name__== "__main__": + logger = get_logger(args.verbose, "kontor") + logger.info("kontor.sync started") + apiConfig = get_api_config(logger, args.config) + for server in apiConfig.server: + data = server.request(logger) + logger.info(len(data)) + logger.info("kontor.sync finished")