Merge pull request #1820 from buildtesters/merge_directory_and_file_i…

…nto_single_option

Merge --directory and --file for buildspec buildspec find to option --search

.github/workflows/regressiontest_container.yml

@@ -38,4 +38,4 @@ jobs:
         with:
           verbose: true
         env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

bash_completion.sh

@@ -413,8 +413,8 @@ _buildtest ()
            COMPREPLY=( $( compgen -W "${opts}" -- "${cur}" ) );;
          # completion for rest of arguments
          *)
-           local longopts="--buildspec --count --directory --executors --file --filter --filterfields --format --formatfields --group-by-executor --group-by-tags --help --helpfilter --helpformat --no-header --pager --paths --quiet --rebuild --row-count --tags --terse"
-           local shortopts="-b -d -e -f -h -n -p -q -r -t"
+           local longopts="--buildspec --count --executors --filter --filterfields --format --formatfields --group-by-executor --group-by-tags --help --helpfilter --helpformat --no-header --pager --paths --quiet --rebuild --row-count --search --tags --terse"
+           local shortopts="-b -d -e  -h -n -p -q -r -s -t"
            local cmds="invalid"
 
            COMPREPLY=( $( compgen -W "${cmds} ${longopts} ${shortopts}" -- "${cur}" ) )

buildtest/builders/base.py

@@ -606,7 +606,7 @@ def _write_build_script(self, modules=None, modulepurge=None, unload_modules=Non
 function cleanup() {
     echo "Signal trapped. Performing cleanup before exiting."
     exitcode=$?
-    echo "buildtest: command \`$BASH_COMMAND' failed (exit code: $exitcode)"
+    echo "buildtest: command '$BASH_COMMAND' failed (exit code: $exitcode)"
     exit $exitcode
 }
 

buildtest/cli/__init__.py

@@ -1287,19 +1287,11 @@ def buildspec_find_menu(self, buildspec_find_parser):
                     },
                 ),
                 (
-                    ["-d", "--directory"],
+                    ["-s", "--search"],
                     {
                         "type": str,
                         "action": "append",
-                        "help": "Specify root buildspecs (directory) path to load buildspecs into buildspec cache.",
-                    },
-                ),
-                (
-                    ["-f", "--file"],
-                    {
-                        "type": str,
-                        "action": "append",
-                        "help": "Specify buildspec file to load into buildspec cache.",
+                        "help": "Search for buildspecs to load into cache. This can be a file or a directory. A file must end in '.yml' extension, for directory buildtest will recursively search for all '.yml' extensions. This option can be specified multiple times to search multiple files",
                     },
                 ),
             ],

buildtest/cli/buildspec.py

@@ -49,14 +49,13 @@ def __init__(
         rebuild=False,
         filterfields=None,
         formatfields=None,
-        directory=None,
-        buildspec_files=None,
         header=None,
         terse=None,
         pager=None,
         color=None,
         count=None,
         row_count=None,
+        search_buildspecs=None,
     ):
         """The initializer method for BuildspecCache class is responsible for loading and finding buildspecs into buildspec cache. First we
         resolve paths to directory where buildspecs will be searched. This can be specified via ``--directory`` option on command line or one can
@@ -71,13 +70,12 @@ def __init__(
             rebuild (bool, optional): rebuild the buildspec cache by validating all buildspecs when using ``buildtest buildspec find --rebuild``. Defaults to ``False`` if ``--rebuild`` is not specified
             filterfields (str, optional): The filter options specified via ``buildtest buildspec find --filter`` that contains list of key value pairs for filtering buildspecs
             formatfields (str, optional): The format options used for formating table. The format option is a comma separated list of format fields specified via ``buildtest buildspec find --format``
-            directory (list, optional): List of directories to search for buildspecs. This argument contains value of ``buildtest buildspec find --directory``
-            buildspec_files (list, optional): List of buildspec files to add to cache. This argument contains value of ``buildtest buildspec find --file``
             headers (bool, optional):  Option to control whether header are printed in terse output. This argument contains value of ``buildtest buildspec find --no-header``
             terse (bool, optional): Enable terse mode when printing output. In this mode we don't print output in table format instead output is printed in parseable format. This option can be specified via ``buildtest buildspec find --terse``
             color (str, optional): An instance of a string class that selects the color to use when printing table output
             count (int, optional): Number of entries to display in output. This argument contains value of ``buildtest buildspec find --count``
             row_count (bool, optional): Print total number of records from the table
+            search_buildspecs (list, optional): List of buildspecs to search and add into cache. This can be file or directory and this argument contains value of ``buildtest buildspec find --search``
         """
 
         if not is_dir(BUILDTEST_BUILDSPEC_DIR):
@@ -99,17 +97,17 @@ def __init__(
         )
         self.row_count = row_count
 
-        # if --root is not specified we set to empty list instead of None
-        self.directory = (
-            directory
-            or self.configuration.target_config["buildspecs"].get("directory")
+        # if --search is not specified we set to empty list instead of None
+        self.search = (
+            search_buildspecs
+            or self.configuration.target_config["buildspecs"].get("search")
             or []
         )
 
         # list of buildspec directories to search for .yml files
         self.paths = []
 
-        self.buildspec_files = buildspec_files
+        self.buildspec_files_to_add = []
 
         # stores invalid buildspecs and the error messages
         self.invalid_buildspecs = {}
@@ -122,13 +120,12 @@ def __init__(
         self.rebuild = rebuild or self.configuration.target_config["buildspecs"].get(
             "rebuild"
         )
-        # if --root is specified we set rebuild to True
-        if self.directory:
+        # if --search is specified we set rebuild to True
+        if self.search:
             self.rebuild = True
 
         self.cache = {}
 
-        self.load_paths()
         self.build()
 
         self._check_filter_fields()
@@ -150,20 +147,26 @@ def load_paths(self):
         """
 
         # if no directory is specified we load the default buildspec.
-        if not self.directory:
+        if not self.search:
             self.paths += BUILDSPEC_DEFAULT_PATH
 
-        # for every root buildspec defined in configuration or via --directory option,
+        # for every root buildspec defined in configuration or via --search option,
         # we resolve path and if path exist add to self.paths. The path must be a
-        # directory. If its file, we ignore it
-        if self.directory:
-            for dirname in self.directory:
+        # file or directory. If it's file, we accept if it ends with .yml extension
+        if self.search:
+            for dirname in self.search:
                 path = resolve_path(dirname, exist=False)
                 if not os.path.exists(path):
                     console.print(f"[red]Path: {path} does not exist!")
-
+                # if its a file, then we check if file ends with .yml extension
                 if is_file(path):
-                    console.print(f"[red]Path: {path} must be a directory not a file")
+                    if not path.endswith(".yml"):
+                        console.print(
+                            f"[red]{path} does not end in .yml extension, please specify a valid buildspec file"
+                        )
+                        continue
+                    else:
+                        self.buildspec_files_to_add.append(path)
 
                 if is_dir(path):
                     self.paths.append(path)
@@ -173,31 +176,8 @@ def build(self):
         rebuild cache we remove the file and recreate cache. If cache file
         exists, we simply load from cache
         """
-        buildspecs = []
-        # this method will check if buildspec_files are valid files and end with .yml.
-        # If it's not a file or does not end with .yml we skip the file and report a message
 
-        if self.buildspec_files:
-            for buildspec in self.buildspec_files:
-                path = resolve_path(buildspec, exist=False)
-                if not os.path.exists(path):
-                    console.print(f"[red]Path: {path} does not exist!")
-                    continue
-                if not is_file(path):
-                    console.print(
-                        f"[red]{path} is not a file, please specify a file when adding buildspec to cache"
-                    )
-                    continue
-                if not path.endswith(".yml"):
-                    console.print(
-                        f"[red]{path} does not end in .yml extension, please specify a valid buildspec file"
-                    )
-                    continue
-
-                buildspecs.append(path)
-
-        # set self.buildspec_files to list of valid buildspec files which will be used to build cache
-        self.buildspec_files = buildspecs
+        self.load_paths()
 
         # implements buildtest buildspec find --rebuild which removes cache file
         # before finding all buildspecs. We only remove file if file exists
@@ -226,21 +206,20 @@ def _discover_buildspecs(self):
         """
 
         buildspecs = []
-        # add all buildspecs from each repo. walk_tree will find all .yml files
-        # recursively and add them to list
-
-        if not self.paths:
-            raise BuildTestError(
-                "Unable to search any buildspecs, please specify a directory"
-            )
-
+        # recursively search all .yml files in directory and add to list
         if self.paths:
             for path in self.paths:
                 buildspec = walk_tree(path, ".yml")
                 buildspecs += buildspec
 
-        if self.buildspec_files:
-            buildspecs += self.buildspec_files
+        if self.buildspec_files_to_add:
+            buildspecs += self.buildspec_files_to_add
+
+        # if no buildspecs found we raise an exception and exit
+        if not buildspecs:
+            raise BuildTestError(
+                "Unable to find any buildspecs, please specify a valid file or directory when searching for buildspec."
+            )
 
         if not self.terse:
             print(f"Buildspec Paths: {self.paths}")
@@ -1436,8 +1415,7 @@ def buildspec_find(args, configuration):
         rebuild=args.rebuild,
         filterfields=args.filter,
         formatfields=args.format,
-        directory=args.directory,
-        buildspec_files=args.file,
+        search_buildspecs=args.search,
         configuration=configuration,
         header=args.no_header,
         terse=args.terse,

buildtest/cli/show.py

@@ -134,7 +134,7 @@ def print_buildspec_show():
         "buildtest buildspec find --pager", "Paginate output of buildspec cache"
     )
     table.add_row(
-        "buildtest buildspec find --directory /tmp --rebuild",
+        "buildtest buildspec find --search /tmp --search $BUILDTEST_ROOT/tutorials/sleep.yml --rebuild",
         "Discover buildspecs in /tmp and rebuild buildspec cache",
     )
     table.add_row(

buildtest/main.py

@@ -251,18 +251,11 @@ def setup(args):
 
     # build buildspec cache file automatically if it doesn't exist
     if not is_file(BUILDSPEC_CACHE_FILE):
-        root_buildspecs = []
-        buildspec_files = []
-        if hasattr(args, "directory"):
-            root_buildspecs = args.directory
-        if hasattr(args, "file"):
-            buildspec_files = args.file
-
-        BuildspecCache(
-            directory=root_buildspecs,
-            buildspec_files=buildspec_files,
-            configuration=configuration,
-        )
+        search_buildspecs = []
+        if hasattr(args, "search"):
+            search_buildspecs = args.search
+
+        BuildspecCache(search_buildspecs=search_buildspecs, configuration=configuration)
 
     return system, configuration, buildtest_editor, report_file
 

buildtest/schemas/examples/settings.schema.json/valid/local-executor.yml

@@ -20,6 +20,8 @@ system:
       format: "name,description"
       # enable terse mode
       terse: False
+      # search for buildspecs in these directories
+      search: ["$BUILDTEST_ROOT/tutorials/containers"]
 
     report:
       count: 25

buildtest/schemas/settings.schema.json

@@ -108,12 +108,12 @@
               "$ref": "#/definitions/format"
             },
             "terse": { "$ref": "#/definitions/terse" },
-            "directory": {
+            "search": {
               "type": "array",
               "items": {
                 "type": "string"
               },
-              "description": "Specify a list of directory paths to search buildspecs. This field can be used with ``buildtest buildspec find`` to rebuild buildspec cache or build tests using ``buildtest build`` command"
+              "description": "Specify a list of file paths to search buildspecs. This field can be used with ``buildtest buildspec find`` to rebuild buildspec cache or build tests using ``buildtest build`` command"
             }
           }
         },

buildtest/settings/aws.yml

@@ -10,7 +10,7 @@ system:
       count: 15
       format: name,description
       terse: false
-      directory:
+      search:
       - $BUILDTEST_ROOT/aws_tutorial
     report:
       count: 25

buildtest/settings/aws_oddc_pbs.yml

@@ -10,7 +10,7 @@ system:
       count: 15
       format: name,description
       terse: false
-      directory:
+      search:
       - $BUILDTEST_ROOT/aws_oddc
     report:
       count: 25

buildtest/settings/config.yml

@@ -33,8 +33,8 @@ system:
       # display output in terse mode
       terse: false
 
-      # list of paths to search for buildspecs
-      #directory: ["$BUILDTEST_ROOT/tutorials", "$BUILDTEST_ROOT/examples"]
+      # list of paths to search for buildspecs where it can be file or directory
+      # search: ["$BUILDTEST_ROOT/tutorials/containers", "$BUILDTEST_ROOT/examples", "demo.yml"]
 
     # options for buildtest report command
     report:

buildtest/settings/spack_container.yml

@@ -11,7 +11,7 @@ system:
       count: 15
       format: name,description
       terse: false
-      directory:
+      search:
       - $BUILDTEST_ROOT/examples
     report:
       count: 25

docs/configuring_buildtest/overview.rst

@@ -227,24 +227,34 @@ Searching for buildspecs when building Buildspec Cache
 
 When building the buildspec cache, buildtest will search for buildspecs in a list of directories specified in the configuration file.
 
-Buildtest will search for buildspecs by recursively searching for files with **.yml** extension. The ``directory`` property in configuration file
-is a list of directories to search for buildspecs. The ``directory`` property is not **required** in configuration file, but it can be a good
+Buildtest will search for buildspecs by recursively searching for files with **.yml** extension. The ``search`` property in configuration file
+is a list of files or directories to search for buildspecs. The ``search`` property is not **required** in configuration file, but it can be a good
 idea to set this value if you have a pre-determined location where buildspecs are stored.
 
-You can specify the directory path via command line ``buildtest buildspec find --directory <dir1> --directory <dir2>`` which will override the configuration value. In a
-practical situation, you will want to write your buildspecs in a separate repository which you can clone in your filesystem. Let's say they are cloned in
-your **$HOME** directory named **$HOME/buildtest-examples**. You have one of two options, one you can specify the  path in configuration file as shown below:
+You can specify the file path via command line ``buildtest buildspec find --search <filepath1> --search <filepath2>`` which will override the configuration value. In a
+practical situation, you will want to write your buildspecs in a separate repository which you can clone in your filesystem. Let's say your tests are cloned in
+your **$HOME** directory named **$HOME/buildtest-examples**. To load all buildspecs from this directory you can set the following in your configuration file.:
 
 .. code-block:: yaml
 
     buildspecs:
-      directory: [ $HOME/buildtest-examples ]
+      search: [ $HOME/buildtest-examples ]
 
 This configuration will instruct buildtest to search for buildspecs in ``$HOME/buildtest-examples`` directory, and you won't
-have to specify the ``--directory`` option when running ``buildtest buildspec find``. The second option would be to specify the ``--directory`` everytime
+have to specify the ``--search`` option when running ``buildtest buildspec find``. The second option would be to specify the ``--search`` option everytime
 you need to build the cache. If neither is specified, buildtest will load the default buildspecs which are **$BUILDTEST_ROOT/tutorials** and
 **$BUILDTEST_ROOT/general_tests**.
 
+If you want to specify multiple directories and individual files to be loaded into buildspec cache, you can do that in configuration file. Let's assume you want
+to specify the following directories `$HOME/buildtest-examples/containers` and `$HOME/buildtest-examples/apps`  and file `$HOME/buildtest-examples/job.yml` you
+can set the following in your configuration file.
+
+.. code-block:: yaml
+
+    buildspecs:
+      search: [ $HOME/buildtest-examples/containers, $HOME/buildtest-examples/apps, $HOME/buildtest-examples/job.yml ]
+
+
 .. _configuring_buildtest_report:
 
 Configuring buildtest report