remove obsolete scripts
This commit is contained in:
@@ -1,369 +0,0 @@
|
||||
"""
|
||||
add actors
|
||||
"""
|
||||
import logging.config
|
||||
import requests
|
||||
import re
|
||||
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument('--verbose', '-v', action='count', default=0)
|
||||
args = parser.parse_args()
|
||||
|
||||
def get_logger(level: int) -> logging.Logger:
|
||||
logging.config.dictConfig({
|
||||
'version': 1,
|
||||
'disable_existing_loggers': False,
|
||||
'formatters': {
|
||||
'simple': {
|
||||
'format': '[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s',
|
||||
'datefmt': '%Y-%m-%d %H:%M:%S',
|
||||
},
|
||||
},
|
||||
'handlers': {
|
||||
'console': {
|
||||
'class': logging.StreamHandler,
|
||||
'level': logging.DEBUG,
|
||||
'formatter': 'simple',
|
||||
'stream': 'ext://sys.stdout'
|
||||
},
|
||||
},
|
||||
'loggers': {
|
||||
'urllib3.connectionpool': {
|
||||
'level': 'WARNING',
|
||||
'propagate': False,
|
||||
},
|
||||
'root': {
|
||||
'level': 'DEBUG',
|
||||
'handlers': ['console'],
|
||||
},
|
||||
},
|
||||
})
|
||||
logger = logging.getLogger(__file__)
|
||||
if level is not None:
|
||||
match level:
|
||||
case 0:
|
||||
logger.setLevel(logging.WARNING)
|
||||
case 1:
|
||||
logger.setLevel(logging.INFO)
|
||||
case 2:
|
||||
logger.setLevel(logging.DEBUG)
|
||||
case _:
|
||||
logger.setLevel(logging.CRITICAL)
|
||||
return logger
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
log = get_logger(args.verbose)
|
||||
log.warning('kontor.add_actors started')
|
||||
log.debug('get all actors')
|
||||
response = requests.get("http://127.0.0.1:8800/api/media/actors")
|
||||
data = response.json()
|
||||
actors = {}
|
||||
for item in data:
|
||||
actor = {}
|
||||
actor['id'] = item['id']
|
||||
actor['name'] = item['name']
|
||||
actor['url'] = item['url']
|
||||
actors[item['url']] = actor
|
||||
log.debug(f'all actors: {actors}')
|
||||
|
||||
new_actor_list = [
|
||||
{ 'name': 'Herschel Savage', 'url': 'https://ge.xhamster.com/pornstars/herschel-savage'},
|
||||
{ 'name': 'Janey Robbins', 'url': 'https://ge.xhamster.com/pornstars/janey-robbins'},
|
||||
{ 'name': 'Kimberly Carson', 'url': 'https://ge.xhamster.com/pornstars/kimberly-carson'},
|
||||
{ 'name': 'Paul Thomas', 'url': 'https://ge.xhamster.com/pornstars/paul-thomas'},
|
||||
{ 'name': 'Shauna Grant', 'url': 'https://ge.xhamster.com/pornstars/shauna-grant'},
|
||||
{ 'name': 'Tara Aire', 'url': 'https://ge.xhamster.com/pornstars/tara-aire'},
|
||||
{ 'name': 'Kari Milla', 'url': 'https://ge.xhamster.com/pornstars/kari-milla'},
|
||||
{ 'name': 'Drago Gvozdik', 'url': 'https://ge.xhamster.com/pornstars/drago-gvozdik'},
|
||||
{ 'name': 'Cheyne Collins', 'url': 'https://ge.xhamster.com/pornstars/cheyne-collins'},
|
||||
{ 'name': 'Christian XXX', 'url': 'https://ge.xhamster.com/pornstars/christian-xxx'},
|
||||
{ 'name': 'Derrick Pierce', 'url': 'https://ge.xhamster.com/pornstars/derrick-pierce'},
|
||||
{ 'name': 'Holly Halston', 'url': 'https://ge.xhamster.com/pornstars/holly-halston'},
|
||||
{ 'name': 'Holly West', 'url': 'https://ge.xhamster.com/pornstars/holly-west'},
|
||||
{ 'name': 'Jarod Diamond', 'url': 'https://ge.xhamster.com/pornstars/jarod-diamond'},
|
||||
{ 'name': 'Kristina Cross', 'url': 'https://ge.xhamster.com/pornstars/kristina-cross'},
|
||||
{ 'name': 'Lee Stone', 'url': 'https://ge.xhamster.com/pornstars/lee-stone'},
|
||||
{ 'name': 'Monica Mayhem', 'url': 'https://ge.xhamster.com/pornstars/monica-mayhem'},
|
||||
{ 'name': 'Sienna West', 'url': 'https://ge.xhamster.com/pornstars/sienna-west'},
|
||||
{ 'name': 'Aria Carson', 'url': 'https://ge.xhamster.com/pornstars/aria-carson'},
|
||||
{ 'name': 'Britney Amber', 'url': 'https://ge.xhamster.com/pornstars/britney-amber'},
|
||||
{ 'name': 'Kit Mercer', 'url': 'https://ge.xhamster.com/pornstars/kit-mercer'},
|
||||
{ 'name': 'Riley Reyes', 'url': 'https://ge.xhamster.com/pornstars/riley-reyes'},
|
||||
{ 'name': 'Ryan Keely', 'url': 'https://ge.xhamster.com/pornstars/ryan-keely'},
|
||||
{ 'name': 'Alexandra Diamond', 'url': 'https://ge.xhamster.com/pornstars/alexandra-diamond'},
|
||||
{ 'name': 'Amanda Twice', 'url': 'https://ge.xhamster.com/pornstars/amanda-twice'},
|
||||
{ 'name': 'Choky Ice', 'url': 'https://ge.xhamster.com/pornstars/choky-ice'},
|
||||
{ 'name': 'Cindy Cox', 'url': 'https://ge.xhamster.com/pornstars/cindy-cox'},
|
||||
{ 'name': 'Kelly White', 'url': 'https://ge.xhamster.com/pornstars/kelly-white'},
|
||||
{ 'name': 'Mike Foster', 'url': 'https://ge.xhamster.com/pornstars/mike-foster'},
|
||||
{ 'name': 'Susie Sorrento', 'url': 'https://ge.xhamster.com/pornstars/susie-sorrento'},
|
||||
{ 'name': 'Lara Sanchez', 'url': 'https://ge.xhamster.com/pornstars/lara-sanchez'},
|
||||
{ 'name': 'Alicia Williams', 'url': 'https://ge.xhamster.com/pornstars/alicia-williams'},
|
||||
{ 'name': 'Dirty Tina', 'url': 'https://ge.xhamster.com/pornstars/dirty-tina'},
|
||||
{ 'name': 'Andrew Andretti', 'url': 'https://ge.xhamster.com/pornstars/andrew-andretti'},
|
||||
{ 'name': 'Dick Nasty', 'url': 'https://ge.xhamster.com/pornstars/dick-nasty'},
|
||||
{ 'name': 'Vicky Vette', 'url': 'https://ge.xhamster.com/pornstars/vicky-vette'},
|
||||
{ 'name': 'Alex Gonz', 'url': 'https://ge.xhamster.com/pornstars/alex-gonz'},
|
||||
{ 'name': 'Audrey Hollander', 'url': 'https://ge.xhamster.com/pornstars/audrey-hollander'},
|
||||
{ 'name': 'Desiree Diamond', 'url': 'https://ge.xhamster.com/pornstars/desiree-diamond'},
|
||||
{ 'name': 'Lexi Bardot', 'url': 'https://ge.xhamster.com/pornstars/lexi-bardot'},
|
||||
{ 'name': 'Adam Ocelot', 'url': 'https://ge.xhamster.com/pornstars/adam-ocelot'},
|
||||
{ 'name': 'Scarlett Hampton', 'url': 'https://ge.xhamster.com/pornstars/scarlett-hampton'},
|
||||
{ 'name': 'Laetitia', 'url': 'https://ge.xhamster.com/pornstars/laetitia'},
|
||||
{ 'name': 'Raffaela Anderson', 'url': 'https://ge.xhamster.com/pornstars/raffaela-anderson'},
|
||||
{ 'name': 'Michael Swayze', 'url': 'https://ge.xhamster.com/pornstars/michael-swayze'},
|
||||
{ 'name': 'Winter Jade', 'url': 'https://ge.xhamster.com/pornstars/winter-jade'},
|
||||
{ 'name': 'Celine Noiret', 'url': 'https://ge.xhamster.com/pornstars/celine-noiret'},
|
||||
{ 'name': 'Aurora Snow', 'url': 'https://ge.xhamster.com/pornstars/aurora-snow'},
|
||||
{ 'name': 'Bree Brooks', 'url': 'https://ge.xhamster.com/pornstars/bree-brooks'},
|
||||
{ 'name': 'Flick Shagwell', 'url': 'https://ge.xhamster.com/pornstars/flick-shagwell'},
|
||||
{ 'name': 'Noname Jane', 'url': 'https://ge.xhamster.com/pornstars/noname-jane'},
|
||||
{ 'name': 'Alex Magni', 'url': 'https://ge.xhamster.com/pornstars/alex-magni'},
|
||||
{ 'name': 'Maeva Dream', 'url': 'https://ge.xhamster.com/pornstars/maeva-dream'},
|
||||
{ 'name': 'Squirty Alice', 'url': 'https://ge.xhamster.com/pornstars/squirty-alice'},
|
||||
{ 'name': 'Mandy Rhea', 'url': 'https://ge.xhamster.com/pornstars/mandy-rhea'},
|
||||
{ 'name': 'Alessia Donati', 'url': 'https://ge.xhamster.com/pornstars/alessia-donati'},
|
||||
{ 'name': 'Pierre DJ', 'url': 'https://ge.xhamster.com/pornstars/pierre-dj'},
|
||||
{ 'name': 'Veronica Belli', 'url': 'https://ge.xhamster.com/pornstars/veronica-belli'},
|
||||
{ 'name': 'Christiana Cinn', 'url': 'https://ge.xhamster.com/pornstars/christiana-cinn'},
|
||||
{ 'name': 'Jasmine Jae', 'url': 'https://ge.xhamster.com/pornstars/jasmine-jae'},
|
||||
{ 'name': 'Jay Smooth', 'url': 'https://ge.xhamster.com/pornstars/jay-smooth'},
|
||||
{ 'name': 'Natalie Knight', 'url': 'https://ge.xhamster.com/pornstars/natalie-knight'},
|
||||
{ 'name': 'Joshua Lewis', 'url': 'https://ge.xhamster.com/pornstars/joshua-lewis'},
|
||||
{ 'name': 'Lauren Phillips', 'url': 'https://ge.xhamster.com/pornstars/lauren-phillips'},
|
||||
{ 'name': 'Matt Cash', 'url': 'https://ge.xhamster.com/pornstars/matt-cash'},
|
||||
{ 'name': 'Nickey Huntsman', 'url': 'https://ge.xhamster.com/pornstars/nickey-huntsman'},
|
||||
{ 'name': 'Jade Sin', 'url': 'https://ge.xhamster.com/pornstars/jade-sin'},
|
||||
{ 'name': 'Wein Lewis', 'url': 'https://ge.xhamster.com/pornstars/wein-lewis'},
|
||||
{ 'name': 'Megan Murkovski', 'url': 'https://ge.xhamster.com/pornstars/megan-murkovski'},
|
||||
{ 'name': 'Tara Wild', 'url': 'https://ge.xhamster.com/pornstars/tara-wild'},
|
||||
{ 'name': 'Lana Roy', 'url': 'https://ge.xhamster.com/pornstars/lana-roy'},
|
||||
{ 'name': 'Nick Moreno', 'url': 'https://ge.xhamster.com/pornstars/nick-moreno'},
|
||||
{ 'name': 'Brad Armstrong', 'url': 'https://ge.xhamster.com/pornstars/brad-armstrong'},
|
||||
{ 'name': 'Kaylani Lei', 'url': 'https://ge.xhamster.com/pornstars/kaylani-lei'},
|
||||
{ 'name': 'Deborah Wells', 'url': 'https://ge.xhamster.com/pornstars/deborah-wells'},
|
||||
{ 'name': 'Judith Ramirez', 'url': 'https://ge.xhamster.com/pornstars/judith-ramirez'},
|
||||
{ 'name': 'Richard Langin', 'url': 'https://ge.xhamster.com/pornstars/richard-langin'},
|
||||
{ 'name': 'Simona Valli', 'url': 'https://ge.xhamster.com/pornstars/simona-valli'},
|
||||
{ 'name': 'Mandy Dee', 'url': 'https://ge.xhamster.com/pornstars/mandy-dee'},
|
||||
{ 'name': 'Allison Wyte', 'url': 'https://ge.xhamster.com/pornstars/allison-wyte'},
|
||||
{ 'name': 'Ben English', 'url': 'https://ge.xhamster.com/pornstars/ben-english'},
|
||||
{ 'name': 'Chuck Martino', 'url': 'https://ge.xhamster.com/pornstars/chuck-martino'},
|
||||
{ 'name': 'Erika Kole', 'url': 'https://ge.xhamster.com/pornstars/erika-kole'},
|
||||
{ 'name': 'Katie Morgan', 'url': 'https://ge.xhamster.com/pornstars/katie-morgan'},
|
||||
{ 'name': 'Misty Parks', 'url': 'https://ge.xhamster.com/pornstars/misty-parks'},
|
||||
{ 'name': 'Mr. Pete', 'url': 'https://ge.xhamster.com/pornstars/mr-pete'},
|
||||
{ 'name': 'Candie Luciani', 'url': 'https://ge.xhamster.com/pornstars/candie-luciani'},
|
||||
{ 'name': 'Clara Mia', 'url': 'https://ge.xhamster.com/pornstars/clara-mia'},
|
||||
{ 'name': 'Tommy Cabrio', 'url': 'https://ge.xhamster.com/pornstars/tommy-cabrio'},
|
||||
{ 'name': 'Nathan Bronson', 'url': 'https://ge.xhamster.com/pornstars/nathan-bronson'},
|
||||
{ 'name': 'Octavia Red', 'url': 'https://ge.xhamster.com/pornstars/octavia-red'},
|
||||
{ 'name': 'Big George', 'url': 'https://ge.xhamster.com/pornstars/big-george'},
|
||||
{ 'name': 'Cassy Young', 'url': 'https://ge.xhamster.com/pornstars/cassy-young'},
|
||||
{ 'name': 'Egon Kowalski', 'url': 'https://ge.xhamster.com/pornstars/egon-kowalski'},
|
||||
{ 'name': 'Mark Aurel', 'url': 'https://ge.xhamster.com/pornstars/mark-aurel'},
|
||||
{ 'name': 'Samy Fox', 'url': 'https://ge.xhamster.com/pornstars/samy-fox'},
|
||||
{ 'name': 'Valeria Jones', 'url': 'https://ge.xhamster.com/pornstars/valeria-jones'},
|
||||
{ 'name': 'Dieter Von Stein', 'url': 'https://ge.xhamster.com/pornstars/dieter-von-stein'},
|
||||
{ 'name': 'Donna Bell', 'url': 'https://ge.xhamster.com/pornstars/donna-bell'},
|
||||
{ 'name': 'Molly Jane', 'url': 'https://ge.xhamster.com/pornstars/molly-jane'},
|
||||
{ 'name': 'Mark Zane', 'url': 'https://ge.xhamster.com/pornstars/mark-zane'},
|
||||
{ 'name': 'Titus Steel', 'url': 'https://ge.xhamster.com/pornstars/titus-steel'},
|
||||
{ 'name': 'Mackenzie Mace', 'url': 'https://ge.xhamster.com/pornstars/mackenzie-mace'},
|
||||
{ 'name': 'Christina Lang', 'url': 'https://ge.xhamster.com/pornstars/christina-lang'},
|
||||
{ 'name': 'Jens Modena', 'url': 'https://ge.xhamster.com/pornstars/jens-modena'},
|
||||
{ 'name': 'Karma Rosenberg', 'url': 'https://ge.xhamster.com/pornstars/karma-rosenberg'},
|
||||
{ 'name': 'Dani Jensen', 'url': 'https://ge.xhamster.com/pornstars/dani-jensen'},
|
||||
{ 'name': 'Crystal Ray', 'url': 'https://ge.xhamster.com/pornstars/crystal-ray'},
|
||||
{ 'name': 'Andrea Szabo', 'url': 'https://ge.xhamster.com/pornstars/andrea-szabo'},
|
||||
{ 'name': 'Desiree Barclay', 'url': 'https://ge.xhamster.com/pornstars/desiree-barclay'},
|
||||
{ 'name': 'Jada Stevens', 'url': 'https://ge.xhamster.com/pornstars/jada-stevens'},
|
||||
{ 'name': 'Syren Demer', 'url': 'https://ge.xhamster.com/pornstars/syren-demer'},
|
||||
{ 'name': 'Ara Mix', 'url': 'https://ge.xhamster.com/pornstars/ara-mix'},
|
||||
{ 'name': 'Redneck John', 'url': 'https://ge.xhamster.com/pornstars/redneck-john'},
|
||||
{ 'name': 'Allysin Chaynes', 'url': 'https://ge.xhamster.com/pornstars/allysin-chaynes'},
|
||||
{ 'name': 'Aspen Brock', 'url': 'https://ge.xhamster.com/pornstars/aspen-brock'},
|
||||
{ 'name': 'Kaylynn', 'url': 'https://ge.xhamster.com/pornstars/kaylynn'},
|
||||
{ 'name': 'Monica Cameron', 'url': 'https://ge.xhamster.com/pornstars/monica-cameron'},
|
||||
{ 'name': 'Sheila Rossi', 'url': 'https://ge.xhamster.com/pornstars/sheila-rossi'},
|
||||
{ 'name': 'Alex Ginger', 'url': 'https://ge.xhamster.com/pornstars/alex-ginger'},
|
||||
{ 'name': 'Ally Style', 'url': 'https://ge.xhamster.com/pornstars/ally-style'},
|
||||
{ 'name': 'Paola Mike', 'url': 'https://ge.xhamster.com/pornstars/paola-mike'},
|
||||
{ 'name': 'Jeanette Littledove', 'url': 'https://ge.xhamster.com/pornstars/jeanette-littledove'},
|
||||
{ 'name': 'Melissa Melendez', 'url': 'https://ge.xhamster.com/pornstars/melissa-melendez'},
|
||||
{ 'name': 'Peter North', 'url': 'https://ge.xhamster.com/pornstars/peter-north'},
|
||||
{ 'name': 'Siobhan Hunter', 'url': 'https://ge.xhamster.com/pornstars/siobhan-hunter'},
|
||||
{ 'name': 'Tami White', 'url': 'https://ge.xhamster.com/pornstars/tami-white'},
|
||||
{ 'name': 'Tracey Adams', 'url': 'https://ge.xhamster.com/pornstars/tracey-adams'},
|
||||
{ 'name': 'Ashley Haze', 'url': 'https://ge.xhamster.com/pornstars/ashley-haze'},
|
||||
{ 'name': 'Cailey Taylor', 'url': 'https://ge.xhamster.com/pornstars/cailey-taylor'},
|
||||
{ 'name': 'Eve Laurence', 'url': 'https://ge.xhamster.com/pornstars/eve-laurence'},
|
||||
{ 'name': 'Naudia Nyce', 'url': 'https://ge.xhamster.com/pornstars/naudia-nyce'},
|
||||
{ 'name': 'Candy Apples', 'url': 'https://ge.xhamster.com/pornstars/candy-apples'},
|
||||
{ 'name': 'Kate Rich', 'url': 'https://ge.xhamster.com/pornstars/kate-rich'},
|
||||
{ 'name': 'Aniko Kaposi', 'url': 'https://ge.xhamster.com/pornstars/aniko-kaposi'},
|
||||
{ 'name': 'Beatrice Poggi', 'url': 'https://ge.xhamster.com/pornstars/beatrice-poggi'},
|
||||
{ 'name': 'Erika Bella', 'url': 'https://ge.xhamster.com/pornstars/erika-bella'},
|
||||
{ 'name': 'Nikita Gross', 'url': 'https://ge.xhamster.com/pornstars/nikita-gross'},
|
||||
{ 'name': 'Ursula Moore', 'url': 'https://ge.xhamster.com/pornstars/ursula-moore'},
|
||||
{ 'name': 'Caty Kiss', 'url': 'https://ge.xhamster.com/pornstars/caty-kiss'},
|
||||
{ 'name': 'Light Fairy', 'url': 'https://ge.xhamster.com/pornstars/light-fairy'},
|
||||
{ 'name': 'Flame', 'url': 'https://ge.xhamster.com/pornstars/flame'},
|
||||
{ 'name': 'Tiffany Tatum', 'url': 'https://ge.xhamster.com/pornstars/tiffany-tatum'},
|
||||
{ 'name': 'Alyson Sykes', 'url': 'https://ge.xhamster.com/pornstars/alyson-sykes'},
|
||||
{ 'name': 'Jenifer Stone', 'url': 'https://ge.xhamster.com/pornstars/jenifer-stone'},
|
||||
{ 'name': 'Lucy Love', 'url': 'https://ge.xhamster.com/pornstars/lucy-love'},
|
||||
{ 'name': 'Thomas Stone', 'url': 'https://ge.xhamster.com/pornstars/thomas-stone'},
|
||||
{ 'name': 'Nikki Babe', 'url': 'https://ge.xhamster.com/pornstars/nikki-babe'},
|
||||
{ 'name': 'Tyra Misoux', 'url': 'https://ge.xhamster.com/pornstars/tyra-misoux'},
|
||||
{ 'name': 'Kenzie Reeves', 'url': 'https://ge.xhamster.com/pornstars/kenzie-reeves'},
|
||||
{ 'name': 'Anissa Kate', 'url': 'https://ge.xhamster.com/pornstars/anissa-kate'},
|
||||
{ 'name': 'Anna Polina', 'url': 'https://ge.xhamster.com/pornstars/anna-polina'},
|
||||
{ 'name': 'Kimber Delice', 'url': 'https://ge.xhamster.com/pornstars/kimber-delice'},
|
||||
{ 'name': 'Lucy Heart', 'url': 'https://ge.xhamster.com/pornstars/lucy-heart'},
|
||||
{ 'name': 'John Strong', 'url': 'https://ge.xhamster.com/pornstars/john-strong'},
|
||||
{ 'name': 'Markus Dupree', 'url': 'https://ge.xhamster.com/pornstars/markus-dupree'},
|
||||
{ 'name': 'Mick Blue', 'url': 'https://ge.xhamster.com/pornstars/mick-blue'},
|
||||
{ 'name': 'Natasha Nice', 'url': 'https://ge.xhamster.com/pornstars/natasha-nice'},
|
||||
{ 'name': 'Lena Reif', 'url': 'https://ge.xhamster.com/pornstars/lena-reif'},
|
||||
{ 'name': 'Sonia Paganini', 'url': 'https://ge.xhamster.com/pornstars/sonia-paganini'},
|
||||
{ 'name': 'Demi Hawks', 'url': 'https://ge.xhamster.com/pornstars/demi-hawks'},
|
||||
{ 'name': 'Juan El Caballo Loco', 'url': 'https://ge.xhamster.com/pornstars/juan-el-caballo-loco'},
|
||||
{ 'name': 'Mike Mancini', 'url': 'https://ge.xhamster.com/pornstars/mike-mancini'},
|
||||
{ 'name': 'Millie Morgan', 'url': 'https://ge.xhamster.com/pornstars/millie-morgan'},
|
||||
{ 'name': 'Richard Allan', 'url': 'https://ge.xhamster.com/pornstars/richard-allan'},
|
||||
{ 'name': 'Damon Dice', 'url': 'https://ge.xhamster.com/pornstars/damon-dice'},
|
||||
{ 'name': 'Sera Ryder', 'url': 'https://ge.xhamster.com/pornstars/sera-ryder'},
|
||||
{ 'name': 'Zoltan Kaabai', 'url': 'https://ge.xhamster.com/pornstars/zoltan-kabai'},
|
||||
{ 'name': 'Cathy Heaven', 'url': 'https://ge.xhamster.com/pornstars/cathy-heaven'},
|
||||
{ 'name': 'Coco Lovelock', 'url': 'https://ge.xhamster.com/pornstars/coco-lovelock'},
|
||||
{ 'name': 'Percy Sires', 'url': 'https://ge.xhamster.com/pornstars/percy-sires'},
|
||||
{ 'name': 'Meridian', 'url': 'https://ge.xhamster.com/pornstars/meridian'},
|
||||
{ 'name': 'Pascal St. James', 'url': 'https://ge.xhamster.com/pornstars/pascal-st-james'},
|
||||
{ 'name': 'Red Fox', 'url': 'https://ge.xhamster.com/pornstars/red-fox'},
|
||||
{ 'name': 'Tony Art', 'url': 'https://ge.xhamster.com/pornstars/tony-art'},
|
||||
{ 'name': 'Addison Lee', 'url': 'https://ge.xhamster.com/pornstars/addison-lee'},
|
||||
{ 'name': 'Daria Glover', 'url': 'https://ge.xhamster.com/pornstars/daria-glover'},
|
||||
{ 'name': 'Mandy Bright', 'url': 'https://ge.xhamster.com/pornstars/mandy-bright'},
|
||||
{ 'name': 'Antonia Sainz', 'url': 'https://ge.xhamster.com/pornstars/antonia-sainz'},
|
||||
{ 'name': 'Nicole Love', 'url': 'https://ge.xhamster.com/pornstars/nicole-love'},
|
||||
{ 'name': 'Sarah Kay', 'url': 'https://ge.xhamster.com/pornstars/sarah-kay'},
|
||||
{ 'name': 'Judith Kostner', 'url': 'https://ge.xhamster.com/pornstars/judith-kostner'},
|
||||
{ 'name': 'Maria Bellucci', 'url': 'https://ge.xhamster.com/pornstars/maria-bellucci'},
|
||||
{ 'name': 'Melissa Monet', 'url': 'https://ge.xhamster.com/pornstars/melissa-monet'},
|
||||
{ 'name': 'Stephanie Cane', 'url': 'https://ge.xhamster.com/pornstars/stephanie-cane'},
|
||||
{ 'name': 'Will Steiger', 'url': 'https://ge.xhamster.com/pornstars/will-steiger'},
|
||||
{ 'name': 'Katty West', 'url': 'https://ge.xhamster.com/pornstars/katty-west'},
|
||||
{ 'name': 'Jean Pallett', 'url': 'https://ge.xhamster.com/pornstars/jean-pallett'},
|
||||
{ 'name': 'Conny Dachs', 'url': 'https://ge.xhamster.com/pornstars/conny-dachs'},
|
||||
{ 'name': 'Juicy Leyla', 'url': 'https://ge.xhamster.com/pornstars/juicy-leyla'},
|
||||
{ 'name': 'Mandy Mystery', 'url': 'https://ge.xhamster.com/pornstars/mandy-mystery'},
|
||||
{ 'name': 'Jack Vegas', 'url': 'https://ge.xhamster.com/pornstars/jack-vegas'},
|
||||
{ 'name': 'Jessica Ryan', 'url': 'https://ge.xhamster.com/pornstars/jessica-ryan'},
|
||||
{ 'name': 'Nathan Bronson', 'url': 'https://ge.xhamster.com/pornstars/nathan-bronson'},
|
||||
{ 'name': 'Lexi Lore', 'url': 'https://ge.xhamster.com/pornstars/lexi-lore'},
|
||||
{ 'name': 'Molly Little', 'url': 'https://ge.xhamster.com/pornstars/molly-little'},
|
||||
{ 'name': 'Dolly Leigh', 'url': 'https://ge.xhamster.com/pornstars/dolly-leigh'},
|
||||
{ 'name': 'Marcus London', 'url': 'https://ge.xhamster.com/pornstars/marcus-london'},
|
||||
{ 'name': 'Kendra Sunderland', 'url': 'https://ge.xhamster.com/pornstars/kendra-sunderland'},
|
||||
{ 'name': 'Manuel Ferrara', 'url': 'https://ge.xhamster.com/pornstars/manuel-ferrara'},
|
||||
{ 'name': 'Kelly Trump', 'url': 'https://ge.xhamster.com/pornstars/kelly-trump'},
|
||||
{ 'name': 'Angelica Heaven', 'url': 'https://ge.xhamster.com/pornstars/angelica-heaven'},
|
||||
{ 'name': 'Luna Lynx', 'url': 'https://ge.xhamster.com/pornstars/luna-lynx'},
|
||||
{ 'name': 'Princess Lili', 'url': 'https://ge.xhamster.com/pornstars/princess-lili'},
|
||||
{ 'name': 'Alexa Wild', 'url': 'https://ge.xhamster.com/pornstars/alexa-wild'},
|
||||
{ 'name': 'Jessyka Swan', 'url': 'https://ge.xhamster.com/pornstars/jessyka-swan'},
|
||||
{ 'name': 'Nikky Thorne', 'url': 'https://ge.xhamster.com/pornstars/nikky-thorne'},
|
||||
{ 'name': 'Tristan Summers', 'url': 'https://ge.xhamster.com/pornstars/tristan-summers'},
|
||||
{ 'name': 'Rhaya Shyne', 'url': 'https://ge.xhamster.com/pornstars/rhaya-shyne'},
|
||||
{ 'name': 'Desiree West', 'url': 'https://ge.xhamster.com/pornstars/desiree-west'},
|
||||
{ 'name': 'Joan Devlon', 'url': 'https://ge.xhamster.com/pornstars/joan-devlon'},
|
||||
{ 'name': 'Jodi Thorpe', 'url': 'https://ge.xhamster.com/pornstars/jodi-thorpe'},
|
||||
{ 'name': 'Laurien Dominique', 'url': 'https://ge.xhamster.com/pornstars/laurien-dominique'},
|
||||
{ 'name': 'Paul Scharf', 'url': 'https://ge.xhamster.com/pornstars/paul-scharf'},
|
||||
{ 'name': 'Ray Wells', 'url': 'https://ge.xhamster.com/pornstars/ray-wells'},
|
||||
{ 'name': 'Sandy Carey', 'url': 'https://ge.xhamster.com/pornstars/sandy-carey'},
|
||||
{ 'name': 'Spender Travis', 'url': 'https://ge.xhamster.com/pornstars/spender-travis'},
|
||||
{ 'name': 'Starlyn Simone', 'url': 'https://ge.xhamster.com/pornstars/starlyn-simone'},
|
||||
{ 'name': 'Uschi Digard', 'url': 'https://ge.xhamster.com/pornstars/uschi-digard'},
|
||||
{ 'name': 'Katarina Muti', 'url': 'https://ge.xhamster.com/pornstars/katarina-muti'},
|
||||
{ 'name': 'Julia Power', 'url': 'https://ge.xhamster.com/pornstars/julia-power'},
|
||||
{ 'name': 'Salma De Nora', 'url': 'https://ge.xhamster.com/pornstars/salma-de-nora'},
|
||||
{ 'name': 'Valeria Jones', 'url': 'https://ge.xhamster.com/pornstars/valeria-jones'},
|
||||
{ 'name': 'Brandy Canyon', 'url': 'https://ge.xhamster.com/pornstars/brandy-canyon'},
|
||||
{ 'name': 'Marie Berger', 'url': 'https://ge.xhamster.com/pornstars/marie-berger'},
|
||||
{ 'name': 'Luna Rishi', 'url': 'https://ge.xhamster.com/pornstars/luna-rishi'},
|
||||
{ 'name': 'Colleen Brennan', 'url': 'https://ge.xhamster.com/pornstars/colleen-brennan'},
|
||||
{ 'name': 'Roxanne Brewer', 'url': 'https://ge.xhamster.com/pornstars/roxanne-brewer'},
|
||||
{ 'name': 'Elle Denay', 'url': 'https://ge.xhamster.com/pornstars/elle-denay'},
|
||||
{ 'name': 'Juan Largo', 'url': 'https://ge.xhamster.com/pornstars/juan-largo'},
|
||||
{ 'name': 'Codey Steele', 'url': 'https://ge.xhamster.com/pornstars/codey-steele'},
|
||||
{ 'name': 'Laney Grey', 'url': 'https://ge.xhamster.com/pornstars/laney-grey'},
|
||||
{ 'name': 'Stirling Cooper', 'url': 'https://ge.xhamster.com/pornstars/stirling-cooper'},
|
||||
{ 'name': 'Lana Smalls', 'url': 'https://ge.xhamster.com/pornstars/lana-smalls'},
|
||||
{ 'name': 'Alex Sanders', 'url':'https://ge.xhamster.com/pornstars/alex-sanders'},
|
||||
{ 'name': 'Felecia Danay', 'url':'https://ge.xhamster.com/pornstars/felecia-danay'},
|
||||
{ 'name': 'Lita Chase', 'url':'https://ge.xhamster.com/pornstars/lita-chase'},
|
||||
{ 'name': 'Mark Ashley', 'url':'https://ge.xhamster.com/pornstars/mark-ashley'},
|
||||
{ 'name': 'Phyllisha Anne', 'url':'https://ge.xhamster.com/pornstars/phyllisha-anne'},
|
||||
{ 'name': 'Ryan Conner', 'url':'https://ge.xhamster.com/pornstars/ryan-conner'},
|
||||
{ 'name': 'Tanya Danielle', 'url':'https://ge.xhamster.com/pornstars/tanya-danielle'},
|
||||
{ 'name': 'Jessica Moore', 'url':'https://ge.xhamster.com/pornstars/jessica-moore'},
|
||||
{ 'name': 'Mike Angelo', 'url':'https://ge.xhamster.com/pornstars/mike-angelo'},
|
||||
{ 'name': 'Morgan Moon', 'url':'https://ge.xhamster.com/pornstars/morgan-moon'},
|
||||
{ 'name': 'Tyler Steel', 'url':'https://ge.xhamster.com/pornstars/tyler-steel'},
|
||||
{ 'name': 'Abella Danger', 'url':'https://ge.xhamster.com/pornstars/abella-danger'},
|
||||
{ 'name': 'Alex Jett', 'url':'https://ge.xhamster.com/pornstars/alex-jett'},
|
||||
{ 'name': 'Alyson Queen', 'url':'https://ge.xhamster.com/pornstars/alyson-queen'},
|
||||
{ 'name': 'Antynia Rouge', 'url':'https://ge.xhamster.com/pornstars/antynia-rouge'},
|
||||
{ 'name': 'Bea Dumas', 'url':'https://ge.xhamster.com/pornstars/bea-dumas'},
|
||||
{ 'name': 'Callie Black', 'url':'https://ge.xhamster.com/pornstars/callie-black'},
|
||||
{ 'name': 'Caroline Cage', 'url':'https://ge.xhamster.com/pornstars/caroline-cage'},
|
||||
{ 'name': 'Cindy Dollar', 'url':'https://ge.xhamster.com/pornstars/cindy-dollar'},
|
||||
{ 'name': 'Crystal Frost', 'url':'https://ge.xhamster.com/pornstars/crystal-frost'},
|
||||
{ 'name': 'Fanny Steel', 'url':'https://ge.xhamster.com/pornstars/fanny-steel'},
|
||||
{ 'name': 'Gia Derza', 'url':'https://ge.xhamster.com/pornstars/gia-derza'},
|
||||
{ 'name': 'Horst Baron', 'url':'https://ge.xhamster.com/pornstars/horst-baron'},
|
||||
{ 'name': 'Jasmine Rouge', 'url':'https://ge.xhamster.com/pornstars/jasmine-rouge'},
|
||||
{ 'name': 'Jean-Pierre Armand', 'url':'https://ge.xhamster.com/pornstars/jean-pierre-armand'},
|
||||
{ 'name': 'Jessa Rhodes', 'url':'https://ge.xhamster.com/pornstars/jessa-rhodes'},
|
||||
{ 'name': 'Leonie Saint', 'url':'https://ge.xhamster.com/pornstars/leonie-saint'},
|
||||
{ 'name': 'Linda Ray', 'url':'https://ge.xhamster.com/pornstars/linda-ray'},
|
||||
{ 'name': 'Luca Ferrero', 'url':'https://ge.xhamster.com/pornstars/luca-ferrero'},
|
||||
{ 'name': 'Paris Pink', 'url':'https://ge.xhamster.com/pornstars/paris-pink'},
|
||||
{ 'name': 'Pavlina Stejskalova', 'url':'https://ge.xhamster.com/pornstars/pavlina-stejskalova'},
|
||||
{ 'name': 'Phoenix Marie', 'url':'https://ge.xhamster.com/pornstars/phoenix-marie'},
|
||||
{ 'name': 'Ricky Spanish', 'url':'https://ge.xhamster.com/pornstars/ricky-spanish'},
|
||||
{ 'name': 'Rumika Powers', 'url':'https://ge.xhamster.com/pornstars/rumika-powers'},
|
||||
{ 'name': 'Sara Blonde', 'url':'https://ge.xhamster.com/pornstars/sara-blonde'},
|
||||
{ 'name': 'Sean Lawless', 'url':'https://ge.xhamster.com/pornstars/sean-lawless'},
|
||||
{ 'name': 'Seth Gamble', 'url':'https://ge.xhamster.com/pornstars/seth-gamble'},
|
||||
{ 'name': 'Siri Dahl', 'url':'https://ge.xhamster.com/pornstars/siri-dahl'},
|
||||
{ 'name': 'Stephie Staar', 'url':'https://ge.xhamster.com/pornstars/stephie-staar'},
|
||||
{ 'name': 'Steve Holmes', 'url':'https://ge.xhamster.com/pornstars/steve-holmes'},
|
||||
{ 'name': 'Suzette Dale', 'url':'https://ge.xhamster.com/pornstars/suzette-dale'},
|
||||
{ 'name': 'Uncle George', 'url':'https://ge.xhamster.com/pornstars/uncle-george'},
|
||||
{ 'name': 'Winnie', 'url':'https://ge.xhamster.com/pornstars/winnie'},
|
||||
{ 'name': 'Zenza Raggi', 'url':'https://ge.xhamster.com/pornstars/zenza-raggi'},
|
||||
{ 'name': 'Zorah White', 'url':'https://ge.xhamster.com/pornstars/zorah-white'},
|
||||
{ 'name': 'Marilyn Jess', 'url':'https://ge.xhamster.com/pornstars/marilyn-jess'},
|
||||
{ 'name': 'Alexis Capri', 'url':'https://ge.xhamster.com/pornstars/alexis-capri'},
|
||||
]
|
||||
|
||||
|
||||
for new_actor in new_actor_list:
|
||||
if new_actor['url'] in actors:
|
||||
log.warning(f"Actor {new_actor['url']} already persisted")
|
||||
continue
|
||||
actor_response = requests.post(f"http://127.0.0.1:8800/api/media/actors", json=new_actor)
|
||||
log.warning(f"add status: {actor_response.status_code}")
|
||||
if actor_response.status_code == 201:
|
||||
log.warning(f"add Actor {new_actor['url']} to existing actor list")
|
||||
actors[new_actor['url']] = new_actor
|
||||
actor_data = actor_response.json()
|
||||
log.warning(f"Actor {actor_data} persisted")
|
||||
log.warning('kontor.add_actors finished')
|
||||
@@ -1,52 +0,0 @@
|
||||
"""
|
||||
copy data from SQLite to MariaDB
|
||||
"""
|
||||
import sqlite3
|
||||
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
|
||||
|
||||
from config import get_logger, get_database_cursors, get_meta_data, get_scripts
|
||||
|
||||
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument('--recreate-db', action='store_true')
|
||||
parser.add_argument('--verbose', '-v', action='count', default=0)
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
def copy_data(mariadb_conn, sqlite_conn, table_scripts):
|
||||
mariadb_cursor = mariadb_conn.cursor()
|
||||
sqlite_cursor = sqlite_conn.cursor()
|
||||
# logger.info(table_scripts)
|
||||
for table_id in scripts:
|
||||
select_statement = scripts[table_id]['select_sqlite']
|
||||
# logger.info(select_statement)
|
||||
insert_statement = scripts[table_id]['insert_mariadb']
|
||||
mariadb_cursor.execute("SET FOREIGN_KEY_CHECKS = 0")
|
||||
mariadb_cursor.execute(scripts[table_id]['truncate_mariadb'])
|
||||
try:
|
||||
sqlite_cursor.execute(select_statement)
|
||||
rows = sqlite_cursor.fetchall()
|
||||
for row in rows:
|
||||
try:
|
||||
mariadb_cursor.execute(insert_statement, row)
|
||||
except sqlite3.Error as error:
|
||||
logger.info('insert failed with %s\n%s\n%s', error, insert_statement, row)
|
||||
mariadb_conn.commit()
|
||||
mariadb_cursor.execute(scripts[table_id]['count'])
|
||||
(number_of_rows,) = mariadb_cursor.fetchone()
|
||||
row = sqlite_cursor.execute(scripts[table_id]['count']).fetchone()
|
||||
logger.info('%s contains %d : %d entries', scripts[table_id]['name'], number_of_rows, row[0])
|
||||
except sqlite3.Error as error:
|
||||
logger.info('select failed with %s', error)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
logger = get_logger(args.verbose)
|
||||
logger.info('kontor.copy_to_sqlite started')
|
||||
s_conn, m_conn = get_database_cursors(logger)
|
||||
meta_data_tables = get_meta_data(m_conn)
|
||||
# logger.info(meta_data_tables)
|
||||
scripts = get_scripts(meta_data_tables, logger)
|
||||
copy_data(m_conn, s_conn, scripts)
|
||||
s_conn.close()
|
||||
m_conn.close()
|
||||
logger.info('kontor.copy_to_sqlite finished')
|
||||
@@ -1,51 +0,0 @@
|
||||
"""
|
||||
copy data from MariaDB to SQLite
|
||||
"""
|
||||
import sqlite3
|
||||
import mariadb
|
||||
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
|
||||
from config import get_database_cursors, create_tables, get_logger, get_meta_data, get_scripts
|
||||
|
||||
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument('--recreate-db', action='store_true')
|
||||
parser.add_argument('--verbose', '-v', action='count', default=0)
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
def copy_data(mariadb_conn, sqlite_conn, table_scripts):
|
||||
mariadb_cursor = mariadb_conn.cursor()
|
||||
sqlite_cursor = sqlite_conn.cursor()
|
||||
# logger.info(table_scripts)
|
||||
for table_id in table_scripts:
|
||||
select_statement = scripts[table_id]['select_mariadb']
|
||||
# logger.info(select_statement)
|
||||
insert_statement = scripts[table_id]['insert_sqlite']
|
||||
try:
|
||||
mariadb_cursor.execute(select_statement)
|
||||
rows = mariadb_cursor.fetchall()
|
||||
for row in rows:
|
||||
try:
|
||||
sqlite_cursor.execute(insert_statement, row)
|
||||
except sqlite3.Error as error:
|
||||
logger.info('insert failed with %s\n%s\n%s', error, insert_statement, row)
|
||||
sqlite_conn.commit()
|
||||
mariadb_cursor.execute(scripts[table_id]['count'])
|
||||
(number_of_rows,) = mariadb_cursor.fetchone()
|
||||
row = sqlite_cursor.execute(scripts[table_id]['count']).fetchone()
|
||||
logger.info('%s contains %d : %d entries', scripts[table_id]['name'], number_of_rows, row[0])
|
||||
except mariadb.Error as error:
|
||||
logger.info('select failed with %s', error)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
logger = get_logger(args.verbose)
|
||||
logger.info('kontor.copy_to_sqlite started')
|
||||
s_conn, m_conn = get_database_cursors(logger)
|
||||
meta_data_tables = get_meta_data(m_conn)
|
||||
# logger.info(meta_data_tables)
|
||||
scripts = get_scripts(meta_data_tables, logger)
|
||||
create_tables(s_conn, logger, args.recreate_db, scripts)
|
||||
copy_data(m_conn, s_conn, scripts)
|
||||
s_conn.close()
|
||||
m_conn.close()
|
||||
logger.info('kontor.copy_to_sqlite finished')
|
||||
@@ -1,67 +0,0 @@
|
||||
"""
|
||||
Prints the database kontor structure
|
||||
"""
|
||||
import mariadb
|
||||
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
|
||||
from config import get_database_cursors, get_logger
|
||||
|
||||
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument('--verbose', '-v', action='count', default=0)
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
def show_tables(cur, log):
|
||||
"""
|
||||
Retrieves the list of tables from the database
|
||||
:param cur:
|
||||
:param log:
|
||||
:return:
|
||||
"""
|
||||
log.info('get list of tables')
|
||||
table_list = []
|
||||
cur.execute("SHOW TABLES")
|
||||
for (tablename,) in cur.fetchall():
|
||||
table_list.append(tablename)
|
||||
return table_list
|
||||
|
||||
|
||||
def get_field_info(cur):
|
||||
"""
|
||||
Retrieves the field info associated with a cursor
|
||||
:param cur:
|
||||
:return:
|
||||
"""
|
||||
field_info = mariadb.fieldinfo()
|
||||
field_info_text_list = []
|
||||
for column in cur.description:
|
||||
column_name = column[0]
|
||||
column_type = field_info.type(column)
|
||||
column_flags = field_info.flag(column)
|
||||
field_info_text_list.append(f"{column_name}: {column_type} {column_flags}")
|
||||
return field_info_text_list
|
||||
|
||||
|
||||
def get_table_field_info(cur, tablename):
|
||||
"""
|
||||
Retrieves the field info associated with a table
|
||||
:param cur:
|
||||
:param tablename:
|
||||
:return:
|
||||
"""
|
||||
cur.execute(f"SELECT * FROM {tablename} LIMIT 1")
|
||||
field_info = get_field_info(cur)
|
||||
return field_info
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
logger = get_logger(args.verbose)
|
||||
logger.info("kontor.db_structure started")
|
||||
_, mariadb_conn = get_database_cursors(logger)
|
||||
tables = show_tables(mariadb_conn.cursor(), logger)
|
||||
for table in tables:
|
||||
field_info_text = get_table_field_info(mariadb_conn.cursor(), table)
|
||||
print(f"Columns in table {table}:")
|
||||
print("\n".join(field_info_text))
|
||||
print("\n")
|
||||
mariadb_conn.close()
|
||||
logger.info("kontor.db_structure finished")
|
||||
@@ -1,58 +0,0 @@
|
||||
"""
|
||||
copy data from JSON to MariaDB
|
||||
"""
|
||||
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
|
||||
from pathlib import Path
|
||||
from config import get_logger, get_database_cursors
|
||||
import mariadb
|
||||
import json
|
||||
|
||||
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument('--verbose', '-v', action='count', default=0)
|
||||
parser.add_argument('--config', '-c', default='kontor-docker')
|
||||
parser.add_argument('--file', '-f', default='~/.sync/media/data.json')
|
||||
args = parser.parse_args()
|
||||
|
||||
def copy_data(mariadb_conn, data_file: Path, log):
|
||||
mariadb_cursor = mariadb_conn.cursor()
|
||||
import_file = Path(data_file)
|
||||
if not import_file.exists():
|
||||
log.info(f"File {data_file} does not exist. Do nothing.")
|
||||
return
|
||||
log.info("read json file")
|
||||
with open(data_file, 'r') as json_file:
|
||||
json_load = json.load(json_file)
|
||||
for table in json_load:
|
||||
log.info(f"{table}: {len(json_load[table])}")
|
||||
# result[table] = import_table(table, json_load[table])
|
||||
truncate_statement = 'TRUNCATE {}'.format(table)
|
||||
#log.info(f"truncate: {truncate_statement}")
|
||||
mariadb_cursor.execute("SET FOREIGN_KEY_CHECKS = 0")
|
||||
mariadb_cursor.execute(truncate_statement)
|
||||
items = json_load[table]
|
||||
for item in items:
|
||||
#log.info(f"item: {item}")
|
||||
values = []
|
||||
columns = []
|
||||
for (key, value) in item.items():
|
||||
columns.append(key)
|
||||
values.append(value)
|
||||
row = tuple(values)
|
||||
log.info(f"values: {row}")
|
||||
insert_statement = 'INSERT INTO {}({}) VALUES({})'.format(table, ', '.join(columns), ', '.join(['?']*len(columns)))
|
||||
#log.info(f"statement: {insert_statement}")
|
||||
mariadb_cursor.execute(insert_statement, row)
|
||||
try:
|
||||
mariadb_conn.commit()
|
||||
except mariadb.Error as error:
|
||||
log.info('insert failed with %s', error)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
logger = get_logger(args.verbose, args.config)
|
||||
logger.info('kontor.json_to_mariadb started')
|
||||
_, m_conn = get_database_cursors(logger, args.config)
|
||||
copy_data(m_conn, args.file, logger)
|
||||
m_conn.close()
|
||||
logger.info('kontor.json_to_mariadb finished')
|
||||
|
||||
@@ -1,152 +0,0 @@
|
||||
"""
|
||||
copy data from JSON to Postgres
|
||||
"""
|
||||
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
|
||||
from config import get_logger, get_database_cursors
|
||||
import json
|
||||
from psycopg2.sql import SQL
|
||||
|
||||
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument('--verbose', '-v', action='count', default=0)
|
||||
parser.add_argument('--config', '-c', default='kontor-docker')
|
||||
parser.add_argument('--file', '-f', default='~/.sync/media/data.json')
|
||||
args = parser.parse_args()
|
||||
|
||||
def copy_data(postgres_conn, data_file: Path, log):
|
||||
postgres_cursor = postgres_conn.cursor()
|
||||
import_file = Path(data_file)
|
||||
if not import_file.exists():
|
||||
log.info(f"File {data_file} does not exist. Do nothing.")
|
||||
return
|
||||
log.info("read json file")
|
||||
with open(data_file, 'r') as json_file:
|
||||
json_load = json.load(json_file)
|
||||
postgres_cursor.execute("SET session_replication_role='replica'")
|
||||
for table in json_load:
|
||||
log.info(f"{table}: {len(json_load[table])}")
|
||||
# result[table] = import_table(table, json_load[table])
|
||||
truncate_statement = 'TRUNCATE {} CASCADE'.format(table)
|
||||
#log.info(f"truncate: {truncate_statement}")
|
||||
try:
|
||||
postgres_cursor.execute(truncate_statement)
|
||||
except:
|
||||
log.info(f"statement: {insert_statement} FAILED")
|
||||
items = json_load[table]
|
||||
for item in items:
|
||||
#log.info(f"item: {item}")
|
||||
values = []
|
||||
columns = []
|
||||
for (key, value) in item.items():
|
||||
columns.append(key)
|
||||
values.append(value)
|
||||
row = tuple(values)
|
||||
#log.info(f"values: {row}")
|
||||
insert_statement = 'INSERT INTO {}({}) VALUES({})'.format(table, ', '.join(columns), ', '.join(['%s']*len(columns)))
|
||||
#log.info(f"statement: {insert_statement}")
|
||||
try:
|
||||
postgres_cursor.execute(SQL(insert_statement), row)
|
||||
postgres_conn.commit()
|
||||
except:
|
||||
log.info(f'insert failed with {insert_statement}')
|
||||
postgres_cursor.execute("SET session_replication_role='origin'")
|
||||
|
||||
|
||||
def load_json(data_file, log) -> dict:
|
||||
import_file = Path(data_file)
|
||||
if not import_file.exists():
|
||||
log.info(f"File {data_file} does not exist. Do nothing.")
|
||||
return
|
||||
log.info("read json file")
|
||||
with open(data_file, 'r') as json_file:
|
||||
json_load = json.load(json_file)
|
||||
return json_load
|
||||
|
||||
|
||||
def insert_data(postgres_conn, data: dict, log):
|
||||
postgres_cursor = postgres_conn.cursor()
|
||||
log.info("insert data")
|
||||
table_list = []
|
||||
table_list = ['worktype', 'artist', 'publisher', 'volume', 'comic', 'issue', 'story_arc', 'trade_paperback', 'comic_work']
|
||||
table_list.extend(['sport', 'team', 'field_position', 'vendor', 'player', 'rooster', 'card_set', 'card'])
|
||||
#table_list.extend(['card'])
|
||||
table_list.extend(['media_file', 'media_video', 'media_actor', 'media_article', 'media_actor_file'])
|
||||
#table_list.extend(['media_actor_file'])
|
||||
table_list.extend(['profile', 'permission', 'token', 'assignment'])
|
||||
table_list.extend(['mail', 'mail_account', 'module_data', 'meta_data_table', 'meta_data_column'])
|
||||
table_list.extend(['meta_data_table', 'meta_data_column'])
|
||||
table_list.extend(['book', 'author', 'article', 'bookshelf_publisher', 'book_author', 'article_author'])
|
||||
#if len(table_list) != 37:
|
||||
# log.info(f"number of tables incorrect: {len(table_list)}")
|
||||
# return
|
||||
for table in table_list:
|
||||
log.info(f"{table}: {len(data[table])}")
|
||||
truncate_statement = 'DELETE FROM {}'.format(table)
|
||||
log.info(f"truncate: {truncate_statement}")
|
||||
try:
|
||||
postgres_cursor.execute(truncate_statement)
|
||||
postgres_conn.commit()
|
||||
except:
|
||||
log.info(f"statement: {truncate_statement} FAILED")
|
||||
items = data[table]
|
||||
for item in items:
|
||||
# log.info(f"item: {item}")
|
||||
values = []
|
||||
columns = []
|
||||
for (key, value) in item.items():
|
||||
columns.append(key)
|
||||
values.append(value)
|
||||
row = tuple(values)
|
||||
# log.info(f"values: {row}")
|
||||
insert_statement = 'INSERT INTO {}({}) VALUES({})'.format(table, ', '.join(columns),
|
||||
', '.join(['%s'] * len(columns)))
|
||||
# log.info(f"statement: {insert_statement}")
|
||||
try:
|
||||
postgres_cursor.execute(SQL(insert_statement), row)
|
||||
postgres_conn.commit()
|
||||
except:
|
||||
log.info(f'insert failed with {insert_statement}')
|
||||
|
||||
|
||||
def parse_table_order(data: dict, log):
|
||||
log.info("parse_table_order")
|
||||
table_refs: Dict[str, List[str]] = {}
|
||||
for table in data:
|
||||
log.info(f"{table}: {len(data[table])}")
|
||||
items = data[table]
|
||||
table_refs[table] = []
|
||||
if len(items) == 0:
|
||||
continue
|
||||
item = items[0]
|
||||
for key, _ in item.items():
|
||||
if key.endswith('_id'):
|
||||
ref = key[0:-3]
|
||||
log.info(f"table {table} has reference to {ref}")
|
||||
if table in table_refs:
|
||||
table_refs[table].append(ref)
|
||||
else:
|
||||
table_refs[table] = [ref]
|
||||
log.info(f"parsed refs: {table_refs}")
|
||||
table_order = []
|
||||
for table in table_refs:
|
||||
if len(table_refs[table]) == 0:
|
||||
log.info(f"insert {table} at beginning")
|
||||
table_order.insert(0, table)
|
||||
else:
|
||||
log.info(f"insert {table} at end")
|
||||
table_order.append(table)
|
||||
log.info(f"table_list: {len(table_order)}: {table_order}")
|
||||
|
||||
if __name__ == '__main__':
|
||||
logger = get_logger(args.verbose, args.config)
|
||||
logger.info('kontor.json_to_postgres started')
|
||||
_, _, p_conn = get_database_cursors(logger, args.config)
|
||||
data = load_json(args.file, logger)
|
||||
#parse_table_order(data, logger)
|
||||
insert_data(p_conn, data, logger)
|
||||
#copy_data(p_conn, args.file, logger)
|
||||
p_conn.close()
|
||||
logger.info('kontor.json_to_postgres finished')
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
|
||||
from pathlib import Path
|
||||
|
||||
import logging.config
|
||||
import yaml
|
||||
from platformdirs import PlatformDirs
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from schema import Comic, Publisher, Base
|
||||
|
||||
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument('--recreate-db', action='store_true')
|
||||
parser.add_argument('--verbose', '-v', action='count', default=0)
|
||||
parser.add_argument('--file', '-f', default='~/data.json')
|
||||
parser.add_argument('--config', '-c', default='kontor-docker')
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
def get_logger(level: int, config: str):
|
||||
dirs = PlatformDirs(config)
|
||||
logging_config = Path(dirs.user_config_dir, 'logging-config.yaml')
|
||||
with open(logging_config, 'rt') as f:
|
||||
configDict = yaml.safe_load(f.read())
|
||||
logging.config.dictConfig(configDict)
|
||||
logger = logging.getLogger('development')
|
||||
if level is not None:
|
||||
match level:
|
||||
case 0:
|
||||
logger.setLevel(logging.INFO)
|
||||
case 1:
|
||||
logger.setLevel(logging.DEBUG)
|
||||
case _:
|
||||
logger.setLevel(logging.CRITICAL)
|
||||
return logger
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
log = get_logger(args.verbose, args.config)
|
||||
log.info('kontor started')
|
||||
dirs = PlatformDirs(args.config)
|
||||
database_config = Path(dirs.user_config_dir, 'database-config.yaml')
|
||||
with open(database_config, 'rt') as f:
|
||||
db_config = yaml.safe_load(f.read())
|
||||
print(db_config)
|
||||
connect_string = ('mariadb+mariadbconnector://{}:{}@{}:{}/{}'.format(
|
||||
db_config['mariadb']['user'],
|
||||
db_config['mariadb']['password'],
|
||||
db_config['mariadb']['host'],
|
||||
db_config['mariadb']['port'],
|
||||
db_config['mariadb']['database']
|
||||
))
|
||||
engine = create_engine(connect_string)
|
||||
Base.metadata.create_all(bind=engine, checkfirst=True)
|
||||
__session__ = sessionmaker(engine)
|
||||
with __session__() as session:
|
||||
comics = session.scalars(select(Comic)).all()
|
||||
for comic in comics:
|
||||
print(comic)
|
||||
publishers = session.scalars(select(Publisher)).all()
|
||||
for publisher in publishers:
|
||||
print(publisher)
|
||||
print(len(publisher.comics))
|
||||
log.info('kontor finished')
|
||||
@@ -1,80 +0,0 @@
|
||||
"""
|
||||
read file with URLs and store in DB
|
||||
"""
|
||||
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
|
||||
import logging
|
||||
import json
|
||||
from proton import Message, Event
|
||||
from proton.handlers import MessagingHandler
|
||||
from proton.reactor import Container
|
||||
|
||||
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument('-f', '--links', help='file with links')
|
||||
parser.add_argument('--verbose', '-v', action='count', default=0)
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
def read_links_file(links_file):
|
||||
with open(links_file, 'r') as input_file:
|
||||
lines = input_file.readlines()
|
||||
return lines
|
||||
|
||||
|
||||
class AddLinkMessage(MessagingHandler):
|
||||
def __init__(self, server, url, log):
|
||||
super(AddLinkMessage, self).__init__()
|
||||
log.info("create AddLinkMessage")
|
||||
self.server = server
|
||||
self.address = "KontorMediaFile::add_link_file"
|
||||
self.url = url
|
||||
self.log = log
|
||||
|
||||
def on_start(self, event: Event):
|
||||
self.log.info("Connection...")
|
||||
conn = event.container.connect(self.server, user="artemis", password="artemis")
|
||||
event.container.create_sender(conn, self.address)
|
||||
|
||||
def on_connection_opened(self, event: Event) -> None:
|
||||
self.log.debug("connection open")
|
||||
|
||||
def on_connection_error(self, event: Event) -> None:
|
||||
self.log.info(f"error: {event}")
|
||||
|
||||
def on_disconnected(self, event: Event) -> None:
|
||||
self.log.debug(f"disconnected: {repr(event)}")
|
||||
|
||||
|
||||
def on_sendable(self, event: Event):
|
||||
self.log.info("send message")
|
||||
message = Message(body=self.url, address=self.address, content_type="application/json", durable=True)
|
||||
delivery = event.sender.send(message)
|
||||
self.log.info(f"Delivery {delivery} sent")
|
||||
event.connection.close()
|
||||
|
||||
def on_accepted(self, event: Event) -> None:
|
||||
self.log.info(f"accepted Delivery: {event.delivery.remote_state}")
|
||||
|
||||
|
||||
def on_rejected(self, event: Event) -> None:
|
||||
self.log.info(f"rejected Delivery: {event.delivery}")
|
||||
|
||||
if __name__ == '__main__':
|
||||
logging.basicConfig(level=logging.INFO, format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s')
|
||||
logging.info('kontor.read_list started')
|
||||
#conn = stomp.Connection([('127.0.0.1', '61616')])
|
||||
#conn.connect('artemis', 'artemis', wait=True)
|
||||
if args.links:
|
||||
logging.info("read links from file")
|
||||
links = read_links_file(args.links)
|
||||
for link in links:
|
||||
data_dict = {'url': link.strip()}
|
||||
data = json.dumps(data_dict)
|
||||
logging.info("send link message")
|
||||
handler = AddLinkMessage("amqp://127.0.0.1:5672", data, logging)
|
||||
container = Container(handler)
|
||||
container.container_id = "process_add_links"
|
||||
container.run()
|
||||
# conn.send(body=data, destination='KontorMediaFile::add_link_file', headers={'content-type': 'application/json'})
|
||||
#conn.disconnect()
|
||||
logging.info('kontor.read_list finished')
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
sqlalchemy
|
||||
pathlib
|
||||
platformdirs
|
||||
pyyaml
|
||||
beautifulsoup4
|
||||
sqlmodel
|
||||
requests
|
||||
fastapi[standard]
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
|
||||
from dataclasses import dataclass
|
||||
import logging
|
||||
import logging.config
|
||||
from logging import Logger
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
|
||||
from platformdirs import PlatformDirs
|
||||
import requests
|
||||
import yaml
|
||||
|
||||
|
||||
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
|
||||
parser.add_argument("--verbose", "-v", action="count", default=0)
|
||||
parser.add_argument("--config", "-c", default="kontor-api")
|
||||
parser.add_argument("--dry-run", "-m", action="store_true")
|
||||
parser.add_argument("--cleanup", "-d", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
@dataclass
|
||||
class Login:
|
||||
email: str
|
||||
password: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class Server:
|
||||
name: str
|
||||
url: str
|
||||
token: str
|
||||
token_type: str
|
||||
|
||||
def login(self, login: Login, log: Logger):
|
||||
if not self.token:
|
||||
log.info("Call login first")
|
||||
login_url = f"{self.url}/login"
|
||||
login_data = {}
|
||||
login_data["email"] = login.email
|
||||
login_data["password"] = login.password
|
||||
response = requests.post(login_url, json=login_data)
|
||||
status = response.status_code
|
||||
log.info(f"Status: {status}")
|
||||
if status != 200:
|
||||
log.fatal("authentication failed")
|
||||
return
|
||||
data = response.json()
|
||||
log.debug(f"got data: {data}")
|
||||
token = data["access_token"]
|
||||
token_type = data["token_type"]
|
||||
self.token = str(token)
|
||||
self.token_type = str(token_type)
|
||||
|
||||
def request(self, log: Logger):
|
||||
url: str = f"{self.url}/api/media/files"
|
||||
headers: Dict[str, str] = {"Authorization": f"Bearer {self.token}"}
|
||||
response = requests.get(url, headers=headers)
|
||||
log.info(f"Status: {response.status_code}")
|
||||
data = response.json()
|
||||
return data
|
||||
|
||||
|
||||
@dataclass
|
||||
class ApiConfig:
|
||||
login: Login
|
||||
server: List[Server]
|
||||
|
||||
def get_logger(level, config: str):
|
||||
dirs = PlatformDirs(config)
|
||||
logging_config = Path(dirs.user_config_dir, "logging-config.yaml")
|
||||
log_config = None
|
||||
with open(logging_config, "rt") as f:
|
||||
log_config = yaml.safe_load(f.read())
|
||||
logging.config.dictConfig(log_config)
|
||||
logger = logging.getLogger("development")
|
||||
if level is not None:
|
||||
match level:
|
||||
case 0:
|
||||
logger.setLevel(logging.CRITICAL)
|
||||
case 1:
|
||||
logger.setLevel(logging.INFO)
|
||||
case 2:
|
||||
logger.setLevel(logging.DEBUG)
|
||||
case _:
|
||||
logger.setLevel(logging.INFO)
|
||||
return logger
|
||||
|
||||
|
||||
def get_api_config(log: Logger, config: str) -> ApiConfig:
|
||||
# api_data: Dict[str, Any] = {}
|
||||
dirs = PlatformDirs(config)
|
||||
api_config = Path(dirs.user_config_dir, "api.yaml")
|
||||
with open(api_config, "rt") as f:
|
||||
api_data = yaml.safe_load(f.read())
|
||||
servers = [Server(**server) for server in api_data['server']]
|
||||
login = Login(**(api_data["login"]))
|
||||
apiConfig = ApiConfig(server=servers, login=login)
|
||||
log.info(apiConfig)
|
||||
if not api_data:
|
||||
log.fatal("API configuration is missing")
|
||||
return apiConfig
|
||||
for server in apiConfig.server:
|
||||
server.login(apiConfig.login, log)
|
||||
with open(api_config, "w") as f:
|
||||
yaml.dump(api_data, f)
|
||||
return apiConfig
|
||||
|
||||
|
||||
if __name__== "__main__":
|
||||
logger = get_logger(args.verbose, "kontor")
|
||||
logger.info("kontor.sync started")
|
||||
apiConfig = get_api_config(logger, args.config)
|
||||
for server in apiConfig.server:
|
||||
data = server.request(logger)
|
||||
logger.info(len(data))
|
||||
logger.info("kontor.sync finished")
|
||||
Reference in New Issue
Block a user