Parse args from config files independently (#395)

* Parse config files independently'

* fix foramt

ducktape/command_line/parse_args.py

@@ -128,6 +128,20 @@ def config_file_to_args_list(config_file):
     return list(itertools.chain(*[line.split() for line in config_lines]))
 
 
+def parse_non_default_args(parser: argparse.ArgumentParser, defaults: dict, args: list) -> dict:
+    """
+    Parse and remove default args from a list of args, and return the dict of the parsed args.
+    """
+    parsed_args = vars(parser.parse_args(args))
+
+    # remove defaults
+    for key, value in defaults.items():
+        if parsed_args[key] == value:
+            del parsed_args[key]
+
+    return parsed_args
+
+
 def parse_args(args):
     """Parse in command-line and config file options.
 
@@ -144,21 +158,50 @@ def parse_args(args):
 
     # Collect arguments from project config file, user config file, and command line
     # later arguments supersede earlier arguments
-    args_list = []
+    parsed_args_list = []
+
+    # First collect all the default values
+    defaults = vars(parser.parse_args([]))
 
     project_config_file = ConsoleDefaults.PROJECT_CONFIG_FILE
+    # Load all non-default args from project config file.
     if os.path.exists(project_config_file):
-        args_list.extend(config_file_to_args_list(project_config_file))
 
+        parsed_args_list.append(
+            parse_non_default_args(
+                parser,
+                defaults,
+                config_file_to_args_list(project_config_file)
+            )
+        )
+
+    # Load all non-default args from user config file.
     user_config_file = get_user_config_file(args)
     if os.path.exists(user_config_file):
-        args_list.extend(config_file_to_args_list(user_config_file))
-
-    args_list.extend(args)
-    parsed_args_dict = vars(parser.parse_args(args_list))
+        parsed_args_list.append(
+            parse_non_default_args(
+                parser,
+                defaults,
+                config_file_to_args_list(user_config_file)
+            )
+        )
+
+    # Load all non-default args from the command line.
+    parsed_args_list.append(
+        parse_non_default_args(
+            parser,
+            defaults,
+            args
+        )
+    )
+
+    # Don't need to copy, done with the defaults dict.
+    # Start with the default args, and layer on changes.
+    parsed_args_dict = defaults
+    for parsed_args in parsed_args_list:
+        parsed_args_dict.update(parsed_args)
 
     if parsed_args_dict["version"]:
         print(ducktape_version())
         sys.exit(0)
-
     return parsed_args_dict

systests/cluster/test_runner_operations.py

@@ -49,3 +49,25 @@ def timeout_test(self):
         while self.service.count < 100000000:
             self.service.echo()
             time.sleep(.2)
+
+    @cluster(num_nodes=1)
+    def quick1_test(self):
+        """
+        a simple quick test to test basic execution.
+        """
+        self.service.start()
+
+        while self.service.count < 20:
+            self.service.echo()
+            time.sleep(.2)
+
+    @cluster(num_nodes=1)
+    def quick2_test(self):
+        """
+        a simple quick test to test basic execution.
+        """
+        self.service.start()
+
+        while self.service.count < 20:
+            self.service.echo()
+            time.sleep(.2)

tests/command_line/check_parse_args.py

@@ -146,3 +146,41 @@ def check_config_file_option(self):
             assert args_dict["parameters"] == "PARAMETERS-user"
         finally:
             shutil.rmtree(tmpdir)
+
+    def check_config_overrides_for_n_args(self, monkeypatch):
+        """Check that parsed arguments pick up values from config files, and that overrides match precedence."""
+
+        tmpdir = tempfile.mkdtemp(dir="/tmp")
+        # Create tmp file for global config
+        project_cfg_filename = os.path.join(tmpdir, "ducktape-project.cfg")
+        user_cfg_filename = os.path.join(tmpdir, "ducktape-user.cfg")
+
+        project_cfg = [
+            "--exclude", "test1", "test2", "test3",
+        ]
+
+        # user_cfg options should override project_cfg
+        user_cfg = [
+            "test1", "test2", "test3",
+        ]
+
+        try:
+            monkeypatch.setattr("ducktape.command_line.defaults.ConsoleDefaults.PROJECT_CONFIG_FILE",
+                                project_cfg_filename)
+            monkeypatch.setattr("ducktape.command_line.defaults.ConsoleDefaults.USER_CONFIG_FILE", user_cfg_filename)
+
+            with open(project_cfg_filename, "w") as project_f:
+                project_f.write("\n".join(project_cfg))
+
+            with open(user_cfg_filename, "w") as user_f:
+                user_f.write("\n".join(user_cfg))
+
+            # command-line options should override user_cfg and project_cfg
+            args_dict = parse_args(["test1", "test2", "test3", "test4", "--max-parallel", "9000"])
+
+            assert args_dict["test_path"] == ["test1", "test2", "test3", "test4"]
+            assert args_dict["exclude"] == ["test1", "test2", "test3"]
+            assert args_dict["max_parallel"] == 9000
+
+        finally:
+            shutil.rmtree(tmpdir)