From c7b51f4a23d0617e2047c11f18ccf5d1e61a3f7c Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Thu, 26 Sep 2024 11:12:29 -0400 Subject: [PATCH 01/11] a shell syntax warning error when running echo statement in cleanup function --- buildtest/builders/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildtest/builders/base.py b/buildtest/builders/base.py index c39cb4423..d9442ddc4 100644 --- a/buildtest/builders/base.py +++ b/buildtest/builders/base.py @@ -606,7 +606,7 @@ def _write_build_script(self, modules=None, modulepurge=None, unload_modules=Non function cleanup() { echo "Signal trapped. Performing cleanup before exiting." exitcode=$? - echo "buildtest: command \`$BASH_COMMAND' failed (exit code: $exitcode)" + echo "buildtest: command '$BASH_COMMAND' failed (exit code: $exitcode)" exit $exitcode } From aab33e9528ac9036186c6dd050c7e169d06fb162 Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Thu, 26 Sep 2024 11:25:12 -0400 Subject: [PATCH 02/11] rename option --directory to --search and update bash completion script --- bash_completion.sh | 4 ++-- buildtest/cli/__init__.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bash_completion.sh b/bash_completion.sh index 7887621f5..7b4be6af4 100644 --- a/bash_completion.sh +++ b/bash_completion.sh @@ -413,8 +413,8 @@ _buildtest () COMPREPLY=( $( compgen -W "${opts}" -- "${cur}" ) );; # completion for rest of arguments *) - local longopts="--buildspec --count --directory --executors --file --filter --filterfields --format --formatfields --group-by-executor --group-by-tags --help --helpfilter --helpformat --no-header --pager --paths --quiet --rebuild --row-count --tags --terse" - local shortopts="-b -d -e -f -h -n -p -q -r -t" + local longopts="--buildspec --count --executors --file --filter --filterfields --format --formatfields --group-by-executor --group-by-tags --help --helpfilter --helpformat --no-header --pager --paths --quiet --rebuild --row-count --search --tags --terse" + local shortopts="-b -d -e -f -h -n -p -q -r -s -t" local cmds="invalid" COMPREPLY=( $( compgen -W "${cmds} ${longopts} ${shortopts}" -- "${cur}" ) ) diff --git a/buildtest/cli/__init__.py b/buildtest/cli/__init__.py index ae991a659..5319c3b28 100644 --- a/buildtest/cli/__init__.py +++ b/buildtest/cli/__init__.py @@ -1287,11 +1287,11 @@ def buildspec_find_menu(self, buildspec_find_parser): }, ), ( - ["-d", "--directory"], + ["-s", "--search"], { "type": str, "action": "append", - "help": "Specify root buildspecs (directory) path to load buildspecs into buildspec cache.", + "help": "Search for buildspecs to load into cache. This can be a file or a directory. A file must end in '.yml' extension, for directory buildtest will recursively search for all '.yml' extensions. This option can be specified multiple times to search multiple files", }, ), ( From fbed2b33634b4c4a0d664fbab003529aff9ad068 Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Thu, 26 Sep 2024 11:40:11 -0400 Subject: [PATCH 03/11] remove --file option and refactor method invocation for BuildspecCache to remove input arguments since we dont need them. We have renamed some variables and initial attempt at implementation of merging the directory and file option --- bash_completion.sh | 4 ++-- buildtest/cli/__init__.py | 8 -------- buildtest/cli/buildspec.py | 36 ++++++++++++++++++++---------------- buildtest/main.py | 14 ++++---------- 4 files changed, 26 insertions(+), 36 deletions(-) diff --git a/bash_completion.sh b/bash_completion.sh index 7b4be6af4..0aec5e599 100644 --- a/bash_completion.sh +++ b/bash_completion.sh @@ -413,8 +413,8 @@ _buildtest () COMPREPLY=( $( compgen -W "${opts}" -- "${cur}" ) );; # completion for rest of arguments *) - local longopts="--buildspec --count --executors --file --filter --filterfields --format --formatfields --group-by-executor --group-by-tags --help --helpfilter --helpformat --no-header --pager --paths --quiet --rebuild --row-count --search --tags --terse" - local shortopts="-b -d -e -f -h -n -p -q -r -s -t" + local longopts="--buildspec --count --executors --filter --filterfields --format --formatfields --group-by-executor --group-by-tags --help --helpfilter --helpformat --no-header --pager --paths --quiet --rebuild --row-count --search --tags --terse" + local shortopts="-b -d -e -h -n -p -q -r -s -t" local cmds="invalid" COMPREPLY=( $( compgen -W "${cmds} ${longopts} ${shortopts}" -- "${cur}" ) ) diff --git a/buildtest/cli/__init__.py b/buildtest/cli/__init__.py index 5319c3b28..9a0e49095 100644 --- a/buildtest/cli/__init__.py +++ b/buildtest/cli/__init__.py @@ -1294,14 +1294,6 @@ def buildspec_find_menu(self, buildspec_find_parser): "help": "Search for buildspecs to load into cache. This can be a file or a directory. A file must end in '.yml' extension, for directory buildtest will recursively search for all '.yml' extensions. This option can be specified multiple times to search multiple files", }, ), - ( - ["-f", "--file"], - { - "type": str, - "action": "append", - "help": "Specify buildspec file to load into buildspec cache.", - }, - ), ], } diff --git a/buildtest/cli/buildspec.py b/buildtest/cli/buildspec.py index aa33a8044..a2124bb65 100644 --- a/buildtest/cli/buildspec.py +++ b/buildtest/cli/buildspec.py @@ -49,14 +49,13 @@ def __init__( rebuild=False, filterfields=None, formatfields=None, - directory=None, - buildspec_files=None, header=None, terse=None, pager=None, color=None, count=None, row_count=None, + search_buildspecs=None, ): """The initializer method for BuildspecCache class is responsible for loading and finding buildspecs into buildspec cache. First we resolve paths to directory where buildspecs will be searched. This can be specified via ``--directory`` option on command line or one can @@ -71,13 +70,12 @@ def __init__( rebuild (bool, optional): rebuild the buildspec cache by validating all buildspecs when using ``buildtest buildspec find --rebuild``. Defaults to ``False`` if ``--rebuild`` is not specified filterfields (str, optional): The filter options specified via ``buildtest buildspec find --filter`` that contains list of key value pairs for filtering buildspecs formatfields (str, optional): The format options used for formating table. The format option is a comma separated list of format fields specified via ``buildtest buildspec find --format`` - directory (list, optional): List of directories to search for buildspecs. This argument contains value of ``buildtest buildspec find --directory`` - buildspec_files (list, optional): List of buildspec files to add to cache. This argument contains value of ``buildtest buildspec find --file`` headers (bool, optional): Option to control whether header are printed in terse output. This argument contains value of ``buildtest buildspec find --no-header`` terse (bool, optional): Enable terse mode when printing output. In this mode we don't print output in table format instead output is printed in parseable format. This option can be specified via ``buildtest buildspec find --terse`` color (str, optional): An instance of a string class that selects the color to use when printing table output count (int, optional): Number of entries to display in output. This argument contains value of ``buildtest buildspec find --count`` row_count (bool, optional): Print total number of records from the table + search_buildspecs (list, optional): List of buildspecs to search and add into cache. This can be file or directory and this argument contains value of ``buildtest buildspec find --search`` """ if not is_dir(BUILDTEST_BUILDSPEC_DIR): @@ -100,8 +98,8 @@ def __init__( self.row_count = row_count # if --root is not specified we set to empty list instead of None - self.directory = ( - directory + self.search = ( + search_buildspecs or self.configuration.target_config["buildspecs"].get("directory") or [] ) @@ -109,7 +107,7 @@ def __init__( # list of buildspec directories to search for .yml files self.paths = [] - self.buildspec_files = buildspec_files + self.buildspec_files_to_add = [] # stores invalid buildspecs and the error messages self.invalid_buildspecs = {} @@ -122,8 +120,8 @@ def __init__( self.rebuild = rebuild or self.configuration.target_config["buildspecs"].get( "rebuild" ) - # if --root is specified we set rebuild to True - if self.directory: + # if --search is specified we set rebuild to True + if self.search: self.rebuild = True self.cache = {} @@ -150,20 +148,26 @@ def load_paths(self): """ # if no directory is specified we load the default buildspec. - if not self.directory: + if not self.search: self.paths += BUILDSPEC_DEFAULT_PATH - # for every root buildspec defined in configuration or via --directory option, + # for every root buildspec defined in configuration or via --search option, # we resolve path and if path exist add to self.paths. The path must be a - # directory. If its file, we ignore it - if self.directory: - for dirname in self.directory: + # file or directory. If it's file, we accept if it ends with .yml extension + if self.search: + for dirname in self.search: path = resolve_path(dirname, exist=False) if not os.path.exists(path): console.print(f"[red]Path: {path} does not exist!") - + # if its a file, then we check if file ends with .yml extension if is_file(path): - console.print(f"[red]Path: {path} must be a directory not a file") + if not path.endswith(".yml"): + console.print( + f"[red]{path} does not end in .yml extension, please specify a valid buildspec file" + ) + continue + else: + self.buildspec_files_to_add.append(path) if is_dir(path): self.paths.append(path) diff --git a/buildtest/main.py b/buildtest/main.py index 209d4afd6..03c18a539 100644 --- a/buildtest/main.py +++ b/buildtest/main.py @@ -253,16 +253,10 @@ def setup(args): if not is_file(BUILDSPEC_CACHE_FILE): root_buildspecs = [] buildspec_files = [] - if hasattr(args, "directory"): - root_buildspecs = args.directory - if hasattr(args, "file"): - buildspec_files = args.file - - BuildspecCache( - directory=root_buildspecs, - buildspec_files=buildspec_files, - configuration=configuration, - ) + if hasattr(args, "search"): + search_buildspecs = args.search + + BuildspecCache(search_buildspecs=search_buildspecs, configuration=configuration) return system, configuration, buildtest_editor, report_file From 5feea45ef60bcba7b0dd2fca8c9c6d94e4461c49 Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Thu, 26 Sep 2024 12:09:55 -0400 Subject: [PATCH 04/11] finish implementation for 'buildtest buildspec find --search' that supports both file and directory. --- buildtest/cli/buildspec.py | 48 +++++++++----------------------------- 1 file changed, 11 insertions(+), 37 deletions(-) diff --git a/buildtest/cli/buildspec.py b/buildtest/cli/buildspec.py index a2124bb65..7cc110cde 100644 --- a/buildtest/cli/buildspec.py +++ b/buildtest/cli/buildspec.py @@ -126,7 +126,6 @@ def __init__( self.cache = {} - self.load_paths() self.build() self._check_filter_fields() @@ -177,31 +176,8 @@ def build(self): rebuild cache we remove the file and recreate cache. If cache file exists, we simply load from cache """ - buildspecs = [] - # this method will check if buildspec_files are valid files and end with .yml. - # If it's not a file or does not end with .yml we skip the file and report a message - - if self.buildspec_files: - for buildspec in self.buildspec_files: - path = resolve_path(buildspec, exist=False) - if not os.path.exists(path): - console.print(f"[red]Path: {path} does not exist!") - continue - if not is_file(path): - console.print( - f"[red]{path} is not a file, please specify a file when adding buildspec to cache" - ) - continue - if not path.endswith(".yml"): - console.print( - f"[red]{path} does not end in .yml extension, please specify a valid buildspec file" - ) - continue - buildspecs.append(path) - - # set self.buildspec_files to list of valid buildspec files which will be used to build cache - self.buildspec_files = buildspecs + self.load_paths() # implements buildtest buildspec find --rebuild which removes cache file # before finding all buildspecs. We only remove file if file exists @@ -230,21 +206,20 @@ def _discover_buildspecs(self): """ buildspecs = [] - # add all buildspecs from each repo. walk_tree will find all .yml files - # recursively and add them to list - - if not self.paths: - raise BuildTestError( - "Unable to search any buildspecs, please specify a directory" - ) - + # recursively search all .yml files in directory and add to list if self.paths: for path in self.paths: buildspec = walk_tree(path, ".yml") buildspecs += buildspec - if self.buildspec_files: - buildspecs += self.buildspec_files + if self.buildspec_files_to_add: + buildspecs += self.buildspec_files_to_add + + # if no buildspecs found we raise an exception and exit + if not buildspec: + raise BuildTestError( + f"Unable to find any buildspecs, please specify a valid file or directory when searching for buildspec." + ) if not self.terse: print(f"Buildspec Paths: {self.paths}") @@ -1440,8 +1415,7 @@ def buildspec_find(args, configuration): rebuild=args.rebuild, filterfields=args.filter, formatfields=args.format, - directory=args.directory, - buildspec_files=args.file, + search_buildspecs=args.search, configuration=configuration, header=args.no_header, terse=args.terse, From 4ef0cf0a92f26bd53f356d36f70c531ac2081429 Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Thu, 26 Sep 2024 12:28:08 -0400 Subject: [PATCH 05/11] refactor regression test for searching buildspecs by file and directory and fix a typo in variable name when no buildspecs were found --- buildtest/cli/buildspec.py | 2 +- tests/cli/test_buildspec.py | 25 ++++++++++--------------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/buildtest/cli/buildspec.py b/buildtest/cli/buildspec.py index 7cc110cde..b52f9b77d 100644 --- a/buildtest/cli/buildspec.py +++ b/buildtest/cli/buildspec.py @@ -216,7 +216,7 @@ def _discover_buildspecs(self): buildspecs += self.buildspec_files_to_add # if no buildspecs found we raise an exception and exit - if not buildspec: + if not buildspecs: raise BuildTestError( f"Unable to find any buildspecs, please specify a valid file or directory when searching for buildspec." ) diff --git a/tests/cli/test_buildspec.py b/tests/cli/test_buildspec.py index 51ef54b38..77866418a 100644 --- a/tests/cli/test_buildspec.py +++ b/tests/cli/test_buildspec.py @@ -349,32 +349,27 @@ def test_edit_file(): @pytest.mark.cli def test_buildspec_find_by_directory_and_files(): - root_buildspecs = [ + + tf = tempfile.NamedTemporaryFile(delete=True) + tf.close() + + search_buildspecs = [ os.path.join(BUILDTEST_ROOT, "tests", "buildsystem"), os.path.join(BUILDTEST_ROOT, "tutorials"), - ] - # list of buildspec files to add to cache, we have one valid file that exists, one with invalid extension and one file that doesn't exist - bp_files = [ os.path.join(BUILDTEST_ROOT, "tutorials", "vars.yml"), os.path.join(BUILDTEST_ROOT, "README.rst"), # invalid extension - os.path.join(BUILDTEST_ROOT, "badfile.yml"), # file doesn't exist - os.path.join(BUILDTEST_ROOT), # directory path + tf.name, # file doesn't exist ] + # buildtest buildspec find --directory $BUILDTEST_ROOT/tests/buildsystem --directory $BUILDTEST_ROOT/tutorials BuildspecCache( - directory=root_buildspecs, - buildspec_files=bp_files, - configuration=configuration, - rebuild=False, + search_buildspecs=search_buildspecs, configuration=configuration, rebuild=False ) with pytest.raises(BuildTestError): - # buildtest buildspec find --directory $BUILDTEST_ROOT/README.rst --directory $BUILDTEST_ROOT/environment.yml + # buildtest buildspec find --search $BUILDTEST_ROOT/README.rst BuildspecCache( - directory=[ - os.path.join(BUILDTEST_ROOT, "README.rst"), - os.path.join(BUILDTEST_ROOT, "tutorials", "environment.yml"), - ], + search_buildspecs=[os.path.join(BUILDTEST_ROOT, "README.rst")], configuration=configuration, ) From af0a5c15aa8678953b30404e1037418f43637c86 Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Thu, 26 Sep 2024 12:40:06 -0400 Subject: [PATCH 06/11] update property name from directory to search. update json schema update lookup key in implementation when searching for buildspecs --- buildtest/cli/buildspec.py | 2 +- buildtest/schemas/settings.schema.json | 4 ++-- buildtest/settings/config.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/buildtest/cli/buildspec.py b/buildtest/cli/buildspec.py index b52f9b77d..51dbfea06 100644 --- a/buildtest/cli/buildspec.py +++ b/buildtest/cli/buildspec.py @@ -100,7 +100,7 @@ def __init__( # if --root is not specified we set to empty list instead of None self.search = ( search_buildspecs - or self.configuration.target_config["buildspecs"].get("directory") + or self.configuration.target_config["buildspecs"].get("search") or [] ) diff --git a/buildtest/schemas/settings.schema.json b/buildtest/schemas/settings.schema.json index 1656e2074..4d8d06d60 100644 --- a/buildtest/schemas/settings.schema.json +++ b/buildtest/schemas/settings.schema.json @@ -108,12 +108,12 @@ "$ref": "#/definitions/format" }, "terse": { "$ref": "#/definitions/terse" }, - "directory": { + "search": { "type": "array", "items": { "type": "string" }, - "description": "Specify a list of directory paths to search buildspecs. This field can be used with ``buildtest buildspec find`` to rebuild buildspec cache or build tests using ``buildtest build`` command" + "description": "Specify a list of file paths to search buildspecs. This field can be used with ``buildtest buildspec find`` to rebuild buildspec cache or build tests using ``buildtest build`` command" } } }, diff --git a/buildtest/settings/config.yml b/buildtest/settings/config.yml index eef02db9c..92747e236 100644 --- a/buildtest/settings/config.yml +++ b/buildtest/settings/config.yml @@ -33,8 +33,8 @@ system: # display output in terse mode terse: false - # list of paths to search for buildspecs - #directory: ["$BUILDTEST_ROOT/tutorials", "$BUILDTEST_ROOT/examples"] + # list of paths to search for buildspecs where it can be file or directory + # search: ["$BUILDTEST_ROOT/tutorials/containers", "$BUILDTEST_ROOT/examples", "demo.yml"] # options for buildtest report command report: From c84013a9db403091fdf182345efae435685cdeb6 Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Thu, 26 Sep 2024 12:54:00 -0400 Subject: [PATCH 07/11] rewrite documentation page --- docs/gettingstarted/buildspecs_interface.rst | 24 ++++++-------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/docs/gettingstarted/buildspecs_interface.rst b/docs/gettingstarted/buildspecs_interface.rst index 33f03e452..46db84521 100644 --- a/docs/gettingstarted/buildspecs_interface.rst +++ b/docs/gettingstarted/buildspecs_interface.rst @@ -90,36 +90,26 @@ or pass them via command line. .. command-output:: buildtest buildspec find --paths buildtest will :ref:`search buildspecs when building cache ` that can be configured via -configuration or command line. If you want to load buildspecs from a directory, you can use the ``--directory`` option. -buildtest will rebuild cache when `--directory` option is specified. Note that to rebuild cache you typically -need to pass **--rebuild** option but that is not required when using **--directory** option because we want -buildtest to load buildspecs into cache. +configuration or command line. If you want to load buildspecs from the command line you can specify ``--search`` option. +buildtest will rebuild cache when `--search` option is specified. -The **--directory** option must be path to a directory, if you specify a file path then buildtest will report an error message similar -to one below +.. Note:: You can rebuild cache via **--rebuild** option, but this is automatically done when **--search** option is specified -.. dropdown:: ``buildtest buildspec find --directory $BUILDTEST_ROOT/README.rst`` - :color: warning - - .. command-output:: buildtest buildspec find --directory $BUILDTEST_ROOT/README.rst - :returncode: 1 -If you want to specify multiple root paths you can specify the **--directory** options multiple times. +If you want to specify multiple file paths, you can specify the **--search** options multiple times. Let's rebuild the cache again by running ``buildtest buildspec find`` which will load the default buildspecs into the cache .. command-output:: buildtest buildspec find --rebuild --quiet -In addition to ``--directory`` option, one can specify a list of files to load into cache using the ``--file`` option. This can be useful -if you want to load specific buildspecs into cache without having to specify ``--directory``. You can use ``--file`` option with ``--directory`` -and buildtest will recursively search directories and load files specified in ``--file`` option. +You can specify a file path with ``--search`` option to load specific buildspecs into cache, which can be useful if want to load specific buildspecs. If you specify an invalid file path, a directory or file without ``.yml`` extension, buildtest will report a message and skip to next file. Shown below, we specify a list of files to load into cache using ``--file`` option. -.. dropdown:: ``buildtest buildspec find --file $BUILDTEST_ROOT/tutorials/vars.yml`` +.. dropdown:: ``buildtest buildspec find --search $BUILDTEST_ROOT/tutorials/vars.yml`` - .. command-output:: buildtest buildspec find --file $BUILDTEST_ROOT/tutorials/vars.yml + .. command-output:: buildtest buildspec find --search $BUILDTEST_ROOT/tutorials/vars.yml We can confirm the file is loaded into cache using the `-b` option which list all buildspecs in cache and pipe via `grep` to search for `vars.yml`. Note that we specify ``--count=-1`` to show all buildspecs in cache. From f94213f67ae9b5b027392fdc50beb7e9cc5815fa Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Thu, 26 Sep 2024 12:54:13 -0400 Subject: [PATCH 08/11] make some more changes that are needed to be done --- buildtest/cli/show.py | 2 +- buildtest/main.py | 3 +-- .../examples/settings.schema.json/valid/local-executor.yml | 2 ++ 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/buildtest/cli/show.py b/buildtest/cli/show.py index 93562c82d..8e52a0c9b 100644 --- a/buildtest/cli/show.py +++ b/buildtest/cli/show.py @@ -134,7 +134,7 @@ def print_buildspec_show(): "buildtest buildspec find --pager", "Paginate output of buildspec cache" ) table.add_row( - "buildtest buildspec find --directory /tmp --rebuild", + "buildtest buildspec find --search /tmp --search $BUILDTEST_ROOT/tutorials/sleep.yml --rebuild", "Discover buildspecs in /tmp and rebuild buildspec cache", ) table.add_row( diff --git a/buildtest/main.py b/buildtest/main.py index 03c18a539..fd2ca1c08 100644 --- a/buildtest/main.py +++ b/buildtest/main.py @@ -251,8 +251,7 @@ def setup(args): # build buildspec cache file automatically if it doesn't exist if not is_file(BUILDSPEC_CACHE_FILE): - root_buildspecs = [] - buildspec_files = [] + search_buildspecs = [] if hasattr(args, "search"): search_buildspecs = args.search diff --git a/buildtest/schemas/examples/settings.schema.json/valid/local-executor.yml b/buildtest/schemas/examples/settings.schema.json/valid/local-executor.yml index 2fd4b1b4e..545bb7918 100644 --- a/buildtest/schemas/examples/settings.schema.json/valid/local-executor.yml +++ b/buildtest/schemas/examples/settings.schema.json/valid/local-executor.yml @@ -20,6 +20,8 @@ system: format: "name,description" # enable terse mode terse: False + # search for buildspecs in these directories + search: ["$BUILDTEST_ROOT/tutorials/containers"] report: count: 25 From 03384c4054dab6c615d852dec0b3459e42d03b4c Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Thu, 26 Sep 2024 13:55:40 -0400 Subject: [PATCH 09/11] fix issue with pyflakes, yamllint update configuration file with keyword 'search' for searching buildspecs since 'directory' is no longer supported --- .github/workflows/regressiontest_container.yml | 2 +- buildtest/cli/buildspec.py | 2 +- buildtest/settings/aws.yml | 2 +- buildtest/settings/aws_oddc_pbs.yml | 2 +- buildtest/settings/spack_container.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/regressiontest_container.yml b/.github/workflows/regressiontest_container.yml index ca8b310cf..0db47451d 100644 --- a/.github/workflows/regressiontest_container.yml +++ b/.github/workflows/regressiontest_container.yml @@ -38,4 +38,4 @@ jobs: with: verbose: true env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/buildtest/cli/buildspec.py b/buildtest/cli/buildspec.py index 51dbfea06..b68043b51 100644 --- a/buildtest/cli/buildspec.py +++ b/buildtest/cli/buildspec.py @@ -218,7 +218,7 @@ def _discover_buildspecs(self): # if no buildspecs found we raise an exception and exit if not buildspecs: raise BuildTestError( - f"Unable to find any buildspecs, please specify a valid file or directory when searching for buildspec." + "Unable to find any buildspecs, please specify a valid file or directory when searching for buildspec." ) if not self.terse: diff --git a/buildtest/settings/aws.yml b/buildtest/settings/aws.yml index c25029ffa..77d11dc2a 100644 --- a/buildtest/settings/aws.yml +++ b/buildtest/settings/aws.yml @@ -10,7 +10,7 @@ system: count: 15 format: name,description terse: false - directory: + search: - $BUILDTEST_ROOT/aws_tutorial report: count: 25 diff --git a/buildtest/settings/aws_oddc_pbs.yml b/buildtest/settings/aws_oddc_pbs.yml index 50dfe30b2..46da94738 100644 --- a/buildtest/settings/aws_oddc_pbs.yml +++ b/buildtest/settings/aws_oddc_pbs.yml @@ -10,7 +10,7 @@ system: count: 15 format: name,description terse: false - directory: + search: - $BUILDTEST_ROOT/aws_oddc report: count: 25 diff --git a/buildtest/settings/spack_container.yml b/buildtest/settings/spack_container.yml index 3f1f23dd6..783fd4ecb 100644 --- a/buildtest/settings/spack_container.yml +++ b/buildtest/settings/spack_container.yml @@ -11,7 +11,7 @@ system: count: 15 format: name,description terse: false - directory: + search: - $BUILDTEST_ROOT/examples report: count: 25 From 5687504f8a03690fb27c8bc985e88e8273d4f716 Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Thu, 26 Sep 2024 14:01:46 -0400 Subject: [PATCH 10/11] update configuration documentation with keyword 'search' and reword some sentences --- docs/configuring_buildtest/overview.rst | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/docs/configuring_buildtest/overview.rst b/docs/configuring_buildtest/overview.rst index 596a6f6ea..672cbd2c3 100644 --- a/docs/configuring_buildtest/overview.rst +++ b/docs/configuring_buildtest/overview.rst @@ -227,24 +227,34 @@ Searching for buildspecs when building Buildspec Cache When building the buildspec cache, buildtest will search for buildspecs in a list of directories specified in the configuration file. -Buildtest will search for buildspecs by recursively searching for files with **.yml** extension. The ``directory`` property in configuration file -is a list of directories to search for buildspecs. The ``directory`` property is not **required** in configuration file, but it can be a good +Buildtest will search for buildspecs by recursively searching for files with **.yml** extension. The ``search`` property in configuration file +is a list of files or directories to search for buildspecs. The ``search`` property is not **required** in configuration file, but it can be a good idea to set this value if you have a pre-determined location where buildspecs are stored. -You can specify the directory path via command line ``buildtest buildspec find --directory --directory `` which will override the configuration value. In a -practical situation, you will want to write your buildspecs in a separate repository which you can clone in your filesystem. Let's say they are cloned in -your **$HOME** directory named **$HOME/buildtest-examples**. You have one of two options, one you can specify the path in configuration file as shown below: +You can specify the file path via command line ``buildtest buildspec find --search --search `` which will override the configuration value. In a +practical situation, you will want to write your buildspecs in a separate repository which you can clone in your filesystem. Let's say your tests are cloned in +your **$HOME** directory named **$HOME/buildtest-examples**. To load all buildspecs from this directory you can set the following in your configuration file.: .. code-block:: yaml buildspecs: - directory: [ $HOME/buildtest-examples ] + search: [ $HOME/buildtest-examples ] This configuration will instruct buildtest to search for buildspecs in ``$HOME/buildtest-examples`` directory, and you won't -have to specify the ``--directory`` option when running ``buildtest buildspec find``. The second option would be to specify the ``--directory`` everytime +have to specify the ``--search`` option when running ``buildtest buildspec find``. The second option would be to specify the ``--search`` option everytime you need to build the cache. If neither is specified, buildtest will load the default buildspecs which are **$BUILDTEST_ROOT/tutorials** and **$BUILDTEST_ROOT/general_tests**. +If you want to specify multiple directories and individual files to be loaded into buildspec cache, you can do that in configuration file. Let's assume you want +to specify the following directories `$HOME/buildtest-examples/containers` and `$HOME/buildtest-examples/apps` and file `$HOME/buildtest-examples/job.yml` you +can set the following in your configuration file. + +.. code-block:: yaml + + buildspecs: + search: [ $HOME/buildtest-examples/containers, $HOME/buildtest-examples/apps, $HOME/buildtest-examples/job.yml ] + + .. _configuring_buildtest_report: Configuring buildtest report From c44867f209fceed400a3a881812b005cecbe23c9 Mon Sep 17 00:00:00 2001 From: Shahzeb Siddiqui Date: Fri, 27 Sep 2024 11:03:05 -0400 Subject: [PATCH 11/11] fix comment --- buildtest/cli/buildspec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildtest/cli/buildspec.py b/buildtest/cli/buildspec.py index b68043b51..0c4330a1e 100644 --- a/buildtest/cli/buildspec.py +++ b/buildtest/cli/buildspec.py @@ -97,7 +97,7 @@ def __init__( ) self.row_count = row_count - # if --root is not specified we set to empty list instead of None + # if --search is not specified we set to empty list instead of None self.search = ( search_buildspecs or self.configuration.target_config["buildspecs"].get("search")