Merge pull request #1762 from buildtesters/write-config-file

Write configuration file to alternate location when saving profiles

.github/workflows/nightly_regression.yml

@@ -49,4 +49,4 @@ jobs:
       with:
         verbose: true
       env:
-        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

bash_completion.sh

@@ -190,7 +190,7 @@ _buildtest ()
   case "$next" in
     build|bd)
       local shortoption="-b -e -et -f -m -n -s -t -u -x -xt"
-      local longoption="--buildspec --dry-run --executor --executor-type --exclude --exclude-tags --filter --helpfilter --limit --maxpendtime --max-jobs --modules --module-purge --name --nodes --pollinterval --procs --profile --rerun --remove-stagedir --retry --save-profile --tags --timeout --unload-modules --validate"
+      local longoption="--buildspec --dry-run --executor --executor-type --exclude --exclude-tags --filter --helpfilter --limit --maxpendtime --max-jobs --modules --module-purge --name --nodes --pollinterval --procs --profile --rerun --remove-stagedir --retry --save-profile --tags --timeout --unload-modules --validate --write-config-file"
       local allopts="${longoption} ${shortoption}"
 
       COMPREPLY=( $( compgen -W "$allopts" -- "${cur}" ) )

buildtest/cli/__init__.py

@@ -920,6 +920,10 @@ def build_menu(self):
                         "help": "Maximum number of jobs that can be run concurrently.",
                     },
                 ),
+                (
+                    ["--profile"],
+                    {"help": "Specify a profile to load from configuration file"},
+                ),
                 (
                     ["--remove-stagedir"],
                     {
@@ -943,10 +947,9 @@ def build_menu(self):
                     },
                 ),
                 (
-                    ["--validate"],
+                    ["--save-profile"],
                     {
-                        "action": "store_true",
-                        "help": "Validate given buildspecs and control behavior of buildtest build to stop execution after parsing the YAML files.",
+                        "help": "Save buildtest command options into a profile and update configuration file"
                     },
                 ),
                 (
@@ -963,14 +966,18 @@ def build_menu(self):
                     },
                 ),
                 (
-                    ["--save-profile"],
+                    ["--validate"],
                     {
-                        "help": "Save buildtest command options into a profile and update configuration file"
+                        "action": "store_true",
+                        "help": "Validate given buildspecs and control behavior of buildtest build to stop execution after parsing the YAML files.",
                     },
                 ),
                 (
-                    ["--profile"],
-                    {"help": "Specify a profile to load from configuration file"},
+                    ["--write-config-file"],
+                    {
+                        "type": str,
+                        "help": "Specify path to configuration file to write changes when saving profile",
+                    },
                 ),
             ],
         }

buildtest/cli/build.py

@@ -627,6 +627,7 @@ def __init__(
         profile=None,
         max_jobs=None,
         verbose=None,
+        write_config_file=None,
     ):
         """The initializer method is responsible for checking input arguments for type
         check, if any argument fails type check we raise an error. If all arguments pass
@@ -665,6 +666,7 @@ def __init__(
             profile (str, optional): Profile to load from buildtest configuration specified by ``buildtest build --profile``
             max_jobs (int, optional): Maximum number of jobs to run concurrently. This option is specified by ``buildtest build --max-jobs``
             verbose (bool, optional): Enable verbose output for buildtest that is specified by ``buildtest --verbose``
+            write_config_file (str, optional): Write configuration file to specified location. This is specified by ``buildtest build --write-config-file``
         """
         self.verbose = verbose
 
@@ -690,7 +692,7 @@ def __init__(
                 raise BuildTestError(f"{arg_name} is not of type list")
 
         # check for input arguments that are expected to be a string
-        for arg_name in [testdir, save_profile, profile]:
+        for arg_name in [testdir, save_profile, profile, write_config_file]:
             if arg_name and not isinstance(arg_name, str):
                 raise BuildTestError(f"{arg_name} is not of type str")
 
@@ -740,6 +742,7 @@ def __init__(
         self.save_profile = save_profile
         self.profile = profile
         self.max_jobs = max_jobs
+        self.write_config_file = write_config_file
 
         # this variable contains the detected buildspecs that will be processed by buildtest.
         self.detected_buildspecs = None
@@ -930,7 +933,24 @@ def save_profile_to_configuration(self):
         """This method will save profile to configuration file. This method is called when ``buildtest build --save-profile`` is invoked. We will open the configuration
         file and update the profile section, if profile already exist we will override it, otherwise we will insert into the configuration file.
         """
+        config_file_path = None
+        if self.write_config_file:
+            config_file_path = resolve_path(self.write_config_file, exist=False)
+            if not config_file_path:
+                raise BuildTestError(
+                    f"Unable to resolve path for {self.write_config_file}"
+                )
+            if is_dir(config_file_path):
+                raise BuildTestError(
+                    f"{config_file_path} is a directory, please specify a file path"
+                )
+
+            if os.path.exists(config_file_path):
+                raise BuildTestError(
+                    f"[red]Configuration file {config_file_path} already exists. Please specify a new file path"
+                )
 
+        config_file_path = config_file_path or self.configuration.file
         resolved_buildspecs = []
         if self.buildspecs:
             for file in self.buildspecs:
@@ -1002,10 +1022,10 @@ def save_profile_to_configuration(self):
         )
 
         console.print(
-            f"Saved profile {self.save_profile} to configuration file {self.configuration.file}"
+            f"Saved profile {self.save_profile} to configuration file {config_file_path}"
         )
 
-        with open(self.configuration.file, "w") as fd:
+        with open(config_file_path, "w") as fd:
             yaml.safe_dump(
                 self.configuration.config, fd, default_flow_style=False, sort_keys=False
             )

buildtest/main.py

@@ -193,6 +193,7 @@ def main():
                 profile=args.profile,
                 max_jobs=args.max_jobs,
                 verbose=args.verbose,
+                write_config_file=args.write_config_file,
             )
             cmd.build()
 

docs/gettingstarted/buildingtest.rst

@@ -586,6 +586,22 @@ Next, let's build the tests via newly created profile and take note that it will
 
     .. command-output:: buildtest build --profile=python-tests
 
+
+You can also specify an alternate location to write configuration file via ``--write-config-file`` when saving profile configuration.
+This can be useful if one wants to use a new configuration file without overwriting the current file for testing purposes.
+To demonstrate this, we will save the profile to configuration file ``/tmp/my_config.yml``
+
+.. dropdown:: ``buildtest build -t python --save-profile=python --write-config-file=/tmp/my_config.yml``
+
+    .. command-output:: buildtest build -t python --save-profile=python --write-config-file=/tmp/my_config.yml
+
+    We can view the profile configuration file by specifying the path to the configuration file.
+
+    .. command-output:: buildtest --config /tmp/my_config.yml config view
+
+Please note that when using ``-write-config-file``, the path must be a file path and file must not exist. If you specify
+a directory path or file already exists you will get an error message.
+
 .. _limit_max_jobs:
 
 Limit Maximum Jobs that can run concurrently (``buildtest build --max-jobs``)

tests/cli/test_build.py

@@ -8,6 +8,7 @@
 from buildtest.cli.build import BuildTest, discover_buildspecs
 from buildtest.cli.buildspec import BuildspecCache
 from buildtest.cli.clean import clean
+from buildtest.cli.config import list_profiles
 from buildtest.config import SiteConfiguration
 from buildtest.defaults import BUILDTEST_RERUN_FILE, BUILDTEST_ROOT
 from buildtest.exceptions import BuildTestError
@@ -452,6 +453,47 @@ def test_save_profile(self):
         )
         cmd.build()
 
+    def test_save_profile_and_write_to_alternate_configuration_file(self):
+        # generate a unique file name and close file handle so we can write to this file
+        tf = tempfile.NamedTemporaryFile(suffix=".yml")
+        tf.close()
+
+        BuildTest(
+            configuration=configuration,
+            buildspecs=[os.path.join(BUILDTEST_ROOT, "tutorials", "hello_world.yml")],
+            save_profile="demo",
+            write_config_file=tf.name,
+        )
+        new_config_file = SiteConfiguration(settings_file=tf.name)
+        new_config_file.detect_system()
+        new_config_file.validate()
+        list_profiles(configuration=new_config_file, print_yaml=True)
+
+        # writing to same configuration file should raise an error since file must not exist when using --write-config-file
+        with pytest.raises(BuildTestError):
+            print(f"Writing to same configuration file: {tf.name} is not allowed")
+            BuildTest(
+                configuration=configuration,
+                buildspecs=[
+                    os.path.join(BUILDTEST_ROOT, "tutorials", "hello_world.yml")
+                ],
+                save_profile="demo",
+                write_config_file=tf.name,
+            )
+
+        tf = tempfile.TemporaryDirectory()
+        # writing to directory is not allowed
+        with pytest.raises(BuildTestError):
+            print(f"Writing to directory: {tf.name} is not allowed")
+            BuildTest(
+                configuration=configuration,
+                buildspecs=[
+                    os.path.join(BUILDTEST_ROOT, "tutorials", "hello_world.yml")
+                ],
+                save_profile="demo",
+                write_config_file=tf.name,
+            )
+
     @pytest.mark.cli
     def test_retry(self):
         buildspecs = [