add script to copy from json to mariadb

2025-04-18 22:55:40 +02:00
parent 1997957069
commit 20f29d88fa
3 changed files with 63 additions and 3 deletions
@@ -0,0 +1,60 @@
+"""
+copy data from SQLite to MariaDB
+"""
+from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
+from pathlib import Path
+from typing import Dict
+from config import get_logger, get_database_cursors, get_meta_data, get_scripts
+import mariadb
+import json
+
+parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
+parser.add_argument('--verbose', '-v', action='count', default=0)
+parser.add_argument('--config', '-c', default='kontor-docker')
+parser.add_argument('--file', '-f', default='~/.sync/media/data.json')
+args = parser.parse_args()
+
+def copy_data(mariadb_conn, data_file: Path, log):
+    mariadb_cursor = mariadb_conn.cursor()
+    result = {}
+    import_file = Path(data_file)
+    if not import_file.exists():
+        log.info(f"File {data_file} does not exist. Do nothing.")
+        return
+    log.info("read json file")
+    with open(data_file, 'r') as json_file:
+        json_load = json.load(json_file)
+        for table in json_load:
+            log.info(f"{table}: {len(json_load[table])}")
+            # result[table] = import_table(table, json_load[table])
+            truncate_statement = 'TRUNCATE {}'.format(table)
+            #log.info(f"truncate: {truncate_statement}")
+            mariadb_cursor.execute("SET FOREIGN_KEY_CHECKS = 0")
+            mariadb_cursor.execute(truncate_statement)
+            items = json_load[table]
+            for item in items:
+                #log.info(f"item: {item}")
+                values = []
+                columns = []
+                for (key, value) in item.items():
+                    columns.append(key)
+                    values.append(value)
+                row = tuple(values)
+                #log.info(f"values: {row}")
+                insert_statement = 'INSERT INTO {}({}) VALUES({})'.format(table, ', '.join(columns), ', '.join(['?']*len(columns)))
+                #log.info(f"statement: {insert_statement}")
+                mariadb_cursor.execute(insert_statement, row)
+                try:
+                    mariadb_conn.commit()
+                except mariadb.Error as error:
+                    log.info('insert failed with %s', error)
+
+
+if __name__ == '__main__':
+    logger = get_logger(args.verbose, args.config)
+    logger.info('kontor.json_to_mariadb started')
+    _, m_conn = get_database_cursors(logger, args.config)
+    copy_data(m_conn, args.file, logger)
+    m_conn.close()
+    logger.info('kontor.json_to_mariadb finished')
+