Thomas Peetz
60 lines
2.4 KiB
Python
60 lines
2.4 KiB
Python
"""
|
|
copy data from JSON to Postgres
|
|
"""
|
|
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
|
|
from pathlib import Path
|
|
from config import get_logger, get_database_cursors
|
|
import json
|
|
import psycopg2
|
|
from psycopg2.sql import SQL
|
|
|
|
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
|
|
parser.add_argument('--verbose', '-v', action='count', default=0)
|
|
parser.add_argument('--config', '-c', default='kontor-docker')
|
|
parser.add_argument('--file', '-f', default='~/.sync/media/data.json')
|
|
args = parser.parse_args()
|
|
|
|
def copy_data(postgres_conn, data_file: Path, log):
|
|
postgres_cursor = postgres_conn.cursor()
|
|
import_file = Path(data_file)
|
|
if not import_file.exists():
|
|
log.info(f"File {data_file} does not exist. Do nothing.")
|
|
return
|
|
log.info("read json file")
|
|
with open(data_file, 'r') as json_file:
|
|
json_load = json.load(json_file)
|
|
for table in json_load:
|
|
log.info(f"{table}: {len(json_load[table])}")
|
|
# result[table] = import_table(table, json_load[table])
|
|
truncate_statement = 'TRUNCATE {}'.format(table)
|
|
#log.info(f"truncate: {truncate_statement}")
|
|
postgres_cursor.execute("SET FOREIGN_KEY_CHECKS = 0")
|
|
postgres_cursor.execute(truncate_statement)
|
|
items = json_load[table]
|
|
for item in items:
|
|
#log.info(f"item: {item}")
|
|
values = []
|
|
columns = []
|
|
for (key, value) in item.items():
|
|
columns.append(key)
|
|
values.append(value)
|
|
row = tuple(values)
|
|
log.info(f"values: {row}")
|
|
insert_statement = 'INSERT INTO {}({}) VALUES({})'.format(table, ', '.join(columns), ', '.join(['%s']*len(columns)))
|
|
#log.info(f"statement: {insert_statement}")
|
|
postgres_cursor.execute(SQL(insert_statement), row)
|
|
try:
|
|
postgres_conn.commit()
|
|
except psycopg2.Error as error:
|
|
log.info('insert failed with %s', error)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
logger = get_logger(args.verbose, args.config)
|
|
logger.info('kontor.json_to_postgres started')
|
|
_, _, p_conn = get_database_cursors(logger, args.config)
|
|
copy_data(p_conn, args.file, logger)
|
|
p_conn.close()
|
|
logger.info('kontor.json_to_postgres finished')
|
|
|