Optimize merge function to reduce memory usage with iterative merging…

src/ucis/__main__.py

@@ -42,9 +42,12 @@ def get_parser():
         help="Specifies the format of the input databases. Defaults to 'xml'")
     merge.add_argument("--output-format", "-of",
         help="Specifies the format of the input databases. Defaults to 'xml'")
+    merge.add_argument("--file-list", "-f",
+        help="File containing list of databases to merge (one per line)")
     merge.add_argument("--libucis", "-l",
         help="Specifies the name/path of the UCIS shared library")
-    merge.add_argument("db", nargs="+")
+    merge.add_argument("db", nargs="*",
+        help="Database files to merge (can be combined with --file-list)")
     merge.set_defaults(func=cmd_merge.merge)
 
     list_db_formats = subparser.add_parser("list-db-formats",

src/ucis/cmd/cmd_merge.py

@@ -22,32 +22,56 @@ def merge(args):
     if not rgy.hasDatabaseFormat(args.output_format):
         raise Exception("Output format %s not recognized" % args.output_format)
 
+    # Build list of databases to merge
+    db_files = list(args.db) if args.db else []
+
+    # Read from file list if provided
+    if args.file_list:
+        import os
+        if not os.path.exists(args.file_list):
+            raise Exception("File list not found: %s" % args.file_list)
+
+        with open(args.file_list, 'r') as f:
+            for line in f:
+                line = line.strip()
+                # Skip comments and blank lines
+                if line and not line.startswith('#'):
+                    db_files.append(line)
+
+    # Validate we have at least one database
+    if len(db_files) == 0:
+        raise Exception("No input databases specified. Use --file-list or provide database arguments.")
+
     input_desc : FormatDescDb = rgy.getDatabaseDesc(args.input_format)
     output_desc : FormatDescDb = rgy.getDatabaseDesc(args.output_format)
 
-    db_l : List[UCIS] = []
-    for input in args.db:
-        db_if : FormatIfDb = input_desc.fmt_if()
+    out_if = output_desc.fmt_if()
+    out_db : UCIS = out_if.create()
+    db_if : FormatIfDb = input_desc.fmt_if()
+    merger = DbMerger()
+
+    for input in db_files:
+        print("read and merge: ", input)
+        out_db_ref : UCIS = out_if.create()
+        db_l : List[UCIS] = []
         try:
             db = db_if.read(input)
             db_l.append(db)
+            db_l.append(out_db)
         except Exception as e:
             raise Exception("Failed to read input file %s: %s" % (
                 input,
                 str(e)
             ))
 
-    out_if = output_desc.fmt_if()
-    out_db : UCIS = out_if.create() 
+        try:
+            merger.merge(out_db_ref, db_l)
+        except Exception as e:
+            raise Exception("Merge operation failed: %s" % str(e))
 
-    merger = DbMerger()
-    try:
-        merger.merge(out_db, db_l)
-    except Exception as e:
-        raise Exception("Merge operation failed: %s" % str(e))
+        out_db = out_db_ref
+        db.close()
 
     out_db.write(args.out)
-    for db in db_l:
-        db.close()