""" copy data from JSON to Postgres """ from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter from pathlib import Path from config import get_logger, get_database_cursors import json import psycopg2 from psycopg2.sql import SQL parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument('--verbose', '-v', action='count', default=0) parser.add_argument('--config', '-c', default='kontor-docker') parser.add_argument('--file', '-f', default='~/.sync/media/data.json') args = parser.parse_args() def copy_data(postgres_conn, data_file: Path, log): postgres_cursor = postgres_conn.cursor() import_file = Path(data_file) if not import_file.exists(): log.info(f"File {data_file} does not exist. Do nothing.") return log.info("read json file") with open(data_file, 'r') as json_file: json_load = json.load(json_file) for table in json_load: log.info(f"{table}: {len(json_load[table])}") # result[table] = import_table(table, json_load[table]) truncate_statement = 'TRUNCATE {}'.format(table) #log.info(f"truncate: {truncate_statement}") postgres_cursor.execute("SET FOREIGN_KEY_CHECKS = 0") postgres_cursor.execute(truncate_statement) items = json_load[table] for item in items: #log.info(f"item: {item}") values = [] columns = [] for (key, value) in item.items(): columns.append(key) values.append(value) row = tuple(values) log.info(f"values: {row}") insert_statement = 'INSERT INTO {}({}) VALUES({})'.format(table, ', '.join(columns), ', '.join(['%s']*len(columns))) #log.info(f"statement: {insert_statement}") postgres_cursor.execute(SQL(insert_statement), row) try: postgres_conn.commit() except psycopg2.Error as error: log.info('insert failed with %s', error) if __name__ == '__main__': logger = get_logger(args.verbose, args.config) logger.info('kontor.json_to_postgres started') _, _, p_conn = get_database_cursors(logger, args.config) copy_data(p_conn, args.file, logger) p_conn.close() logger.info('kontor.json_to_postgres finished')