diff --git a/bash_completion.sh b/bash_completion.sh index ab1ef1a65..34af3919f 100644 --- a/bash_completion.sh +++ b/bash_completion.sh @@ -190,7 +190,7 @@ _buildtest () case "$next" in build|bd) local shortoption="-b -e -et -f -m -n -s -t -u -x -xt" - local longoption="--buildspec --dry-run --executor --executor-type --exclude --exclude-tags --filter --helpfilter --limit --maxpendtime --max-jobs --modules --module-purge --name --nodes --pollinterval --procs --profile --rerun --remove-stagedir --retry --save-profile --strict --tags --timeout --unload-modules --validate --write-config-file" + local longoption="--account --buildspec --display --dry-run --executor --executor-type --exclude --exclude-tags --filter --helpfilter --limit --maxpendtime --max-jobs --modules --module-purge --name --nodes --pollinterval --procs --profile --rebuild --rerun --remove-stagedir --retry --save-profile --strict --tags --testdir --timeout --unload-modules --validate --write-config-file" local allopts="${longoption} ${shortoption}" COMPREPLY=( $( compgen -W "$allopts" -- "${cur}" ) ) diff --git a/buildtest/builders/base.py b/buildtest/builders/base.py index 67ad7d4d1..c39cb4423 100644 --- a/buildtest/builders/base.py +++ b/buildtest/builders/base.py @@ -71,6 +71,7 @@ def __init__( numprocs=None, numnodes=None, compiler=None, + display=None, ): """The BuilderBase provides common functions for any builder. The builder is an instance of BuilderBase. The initializer method will setup the builder @@ -82,6 +83,7 @@ def __init__( buildspec (str): Full path to buildspec file buildexecutor (:obj:`buildtest.executors.setup.BuildExecutor`): An instance of BuildExecutor class used for accessing executors testdir (str): Test directory where tests are written. Must be full path on filesystem. + display (list, optional): Display content of output/error or test. """ self.name = name @@ -95,7 +97,7 @@ def __init__( self.fflags = None self.ldflags = None self.cppflags = None - + self.display = display or [] self.metadata = {} self.duration = 0 @@ -378,21 +380,22 @@ def execute_post_run_script(self): f"[blue]{self}[/]: Post run script exit code: {post_run.returncode()}" ) - print_content( - output, - title=f"[blue]{self}[/]: Start of Post Run Output", - theme="monokai", - lexer="text", - show_last_lines=10, - ) + if "output" in self.display: + print_content( + output, + title=f"[blue]{self}[/]: Start of Post Run Output", + theme="monokai", + lexer="text", + show_last_lines=10, + ) - print_content( - error, - title=f"[blue]{self}[/]: Start of Post Run Error", - theme="monokai", - lexer="text", - show_last_lines=10, - ) + print_content( + error, + title=f"[blue]{self}[/]: Start of Post Run Error", + theme="monokai", + lexer="text", + show_last_lines=10, + ) def handle_run_result(self, command_result, timeout): """This method will handle the result of running test. If the test is successful we will record endtime, @@ -405,26 +408,28 @@ def handle_run_result(self, command_result, timeout): output_msg = "".join(command_result.get_output()) err_msg = "".join(command_result.get_error()) - print_content( - output_msg, - title=f"[blue]{self}[/]: Start of Output", - theme="monokai", - lexer="text", - show_last_lines=10, - ) + if "output" in self.display: + print_content( + output_msg, + title=f"[blue]{self}[/]: Start of Output", + theme="monokai", + lexer="text", + show_last_lines=10, + ) if not self._retry or ret == 0: return command_result console.print(f"[red]{self}: failed to submit job with returncode: {ret}") - print_content( - err_msg, - title=f"[blue]{self}[/]: Start of Error", - theme="monokai", - lexer="text", - show_last_lines=30, - ) + if "output" in self.display: + print_content( + err_msg, + title=f"[blue]{self}[/]: Start of Error", + theme="monokai", + lexer="text", + show_last_lines=30, + ) console.print( f"[red]{self}: Detected failure in running test, will attempt to retry test: {self._retry} times" @@ -663,12 +668,14 @@ def _write_build_script(self, modules=None, modulepurge=None, unload_modules=Non self.build_script = dest self.metadata["build_script"] = self.build_script - print_file_content( - file_path=self.build_script, - title=f"[blue]{self}[/]: Start of Build Script", - lexer="bash", - theme="monokai", - ) + + if "test" in self.display: + print_file_content( + file_path=self.build_script, + title=f"[blue]{self}[/]: Start of Build Script", + lexer="bash", + theme="monokai", + ) def _write_post_run_script(self): """This method will write the content of post run script that is run after the test is complete. @@ -690,12 +697,13 @@ def _write_post_run_script(self): console.print( f"[blue]{self}[/]: Writing Post Run Script: {self.post_run_script}" ) - print_file_content( - file_path=self.post_run_script, - title=f"[blue]{self}[/]: Start of Post Run Script", - lexer="bash", - theme="monokai", - ) + if "test" in self.display: + print_file_content( + file_path=self.post_run_script, + title=f"[blue]{self}[/]: Start of Post Run Script", + lexer="bash", + theme="monokai", + ) def _write_test(self): """This method is responsible for invoking ``generate_script`` that @@ -722,12 +730,13 @@ def _write_test(self): shutil.copy2( self.testpath, os.path.join(self.test_root, os.path.basename(self.testpath)) ) - print_file_content( - file_path=self.testpath, - title=f"[blue]{self}[/]: Start of Test Script", - lexer="bash", - theme="monokai", - ) + if "test" in self.display: + print_file_content( + file_path=self.testpath, + title=f"[blue]{self}[/]: Start of Test Script", + lexer="bash", + theme="monokai", + ) def get_container_invocation(self): """This method returns a list of lines containing the container invocation""" diff --git a/buildtest/builders/script.py b/buildtest/builders/script.py index b2639d11c..8177133d6 100644 --- a/buildtest/builders/script.py +++ b/buildtest/builders/script.py @@ -26,6 +26,7 @@ def __init__( numnodes=None, compiler=None, strict=None, + display=None, ): super().__init__( name=name, @@ -37,6 +38,7 @@ def __init__( numprocs=numprocs, numnodes=numnodes, compiler=compiler, + display=display, ) self.compiler_settings = {"vars": None, "env": None, "modules": None} diff --git a/buildtest/builders/spack.py b/buildtest/builders/spack.py index 357d2380a..61b484fcc 100644 --- a/buildtest/builders/spack.py +++ b/buildtest/builders/spack.py @@ -28,6 +28,7 @@ def __init__( numprocs=None, numnodes=None, strict=None, + display=None, ): super().__init__( name=name, @@ -38,6 +39,7 @@ def __init__( testdir=testdir, numprocs=numprocs, numnodes=numnodes, + display=display, ) self.strict = strict self.status = deep_get( diff --git a/buildtest/buildsystem/builders.py b/buildtest/buildsystem/builders.py index 3b26c2440..57a5ca085 100644 --- a/buildtest/buildsystem/builders.py +++ b/buildtest/buildsystem/builders.py @@ -35,6 +35,7 @@ def __init__( executor_type=None, exclude_tags=None, strict=None, + display=None, ): """Based on a loaded Buildspec file, return the correct builder for each based on the type. Each type is associated with a known @@ -52,13 +53,13 @@ def __init__( executor_type (str, optional): Filter test by executor type (local, batch) exclude_tags (list, optional): List of tags to exclude tests from buildspec file strict (bool, optional): This is a boolean used for enable strict mode for test that will run the 'set' command in test. + display (list, optional): Show content of test or output via ``buildtest build --display`` """ self.configuration = configuration self.logger = logging.getLogger(__name__) self.testdir = testdir self.buildexecutor = buildexecutor - self.rebuild = rebuild or 1 self.numprocs = numprocs self.numnodes = numnodes @@ -68,7 +69,7 @@ def __init__( self.bp = bp self.bc = buildtest_compilers self.filters = filters - + self.display = display self.builders = [] # skip property defined at top-level then skip test @@ -172,6 +173,7 @@ def create_script_builders( testdir=self.testdir, compiler=compiler_name, strict=self.strict, + display=self.display, ) builders.append(builder) @@ -188,6 +190,7 @@ def create_script_builders( numnodes=node, compiler=compiler_name, strict=self.strict, + display=self.display, ) builders.append(builder) if procs: @@ -203,6 +206,7 @@ def create_script_builders( numprocs=proc, compiler=compiler_name, strict=self.strict, + display=self.display, ) builders.append(builder) @@ -228,6 +232,7 @@ def create_spack_builders(self, name, recipe, executor, nodes=None, procs=None): buildexecutor=self.buildexecutor, testdir=self.testdir, strict=self.strict, + display=self.display, ) builders.append(builder) @@ -242,6 +247,7 @@ def create_spack_builders(self, name, recipe, executor, nodes=None, procs=None): testdir=self.testdir, numnodes=node, strict=self.strict, + display=self.display, ) builders.append(builder) if procs: @@ -255,6 +261,7 @@ def create_spack_builders(self, name, recipe, executor, nodes=None, procs=None): testdir=self.testdir, numprocs=proc, strict=self.strict, + display=self.display, ) builders.append(builder) diff --git a/buildtest/cli/__init__.py b/buildtest/cli/__init__.py index 46b5a19a0..c6b74edca 100644 --- a/buildtest/cli/__init__.py +++ b/buildtest/cli/__init__.py @@ -899,6 +899,15 @@ def build_menu(self): ), ], "extra": [ + ( + ["--display"], + { + "action": "append", + "type": str, + "help": "Display content of output/error or test", + "choices": ["output", "test"], + }, + ), ( ["--dry-run"], { diff --git a/buildtest/cli/build.py b/buildtest/cli/build.py index 694536969..335cfee3f 100644 --- a/buildtest/cli/build.py +++ b/buildtest/cli/build.py @@ -595,38 +595,39 @@ class BuildTest: def __init__( self, - configuration=None, + account=None, buildspecs=None, + configuration=None, + display=None, + dry_run=None, exclude_buildspecs=None, - tags=None, - name=None, exclude_tags=None, executors=None, - testdir=None, - validate=None, - dry_run=None, + executor_type=None, filter_buildspecs=None, - rebuild=None, - report_file=None, + helpfilter=None, + limit=None, + max_jobs=None, maxpendtime=None, + modulepurge=None, + modules=None, + name=None, + numnodes=None, + numprocs=None, poll_interval=None, + profile=None, + rebuild=None, remove_stagedir=None, - retry=None, - account=None, - helpfilter=None, - numprocs=None, - numnodes=None, - modules=None, - modulepurge=None, - unload_modules=None, + report_file=None, rerun=None, - executor_type=None, - timeout=None, - limit=None, + retry=None, save_profile=None, - profile=None, - max_jobs=None, strict=None, + tags=None, + testdir=None, + timeout=None, + unload_modules=None, + validate=None, verbose=None, write_config_file=None, ): @@ -635,40 +636,43 @@ def __init__( we assign the values and proceed with building the test. Args: - configuration (buildtest.config.SiteConfiguration, optional): Loaded configuration content which is an instance of SiteConfiguration + + account (str, optional): Project account to charge jobs. This takes input argument ``buildtest build --account`` buildspecs (list, optional): list of buildspecs from command line ``buildtest build --buildspec`` + configuration (buildtest.config.SiteConfiguration, optional): Loaded configuration content which is an instance of SiteConfiguration + display (list, optional): Display content of output or test. This is specified via ``buildtest build --display`` + dry_run (bool, optional): Show a list of tests that will potentially be run without actually running them via ``buildtest build --dry-run`` exclude_buildspecs (list, optional): list of excluded buildspecs from command line ``buildtest build --exclude`` - tags (list, optional): list if tags to discover tests specified via command line ``buildtest build --tags`` - name (list, optional): list of test names to run specified via command line ``buildtest build --name`` exclude_tags (list, optional): list if tags to exclude specified via command line ``buildtest build --exclude-tags`` + executor_type (bool, optional): Filter test by executor type. This option will filter test after discovery by local or batch executors. This can be specified via ``buildtest build --exec-type`` executors (list, optional): list of executors passed from command line ``buildtest build --executors`` - testdir (str): Path to test directory where tests are written. This argument can be passed from command line ``buildtest build --testdir`` - validate (bool, optional): Validate given buildspecs and buildtest will stop after parse stage which can be configured via ``buildtest build --validate`` option - dry_run (bool, optional): Show a list of tests that will potentially be run without actually running them via ``buildtest build --dry-run`` filter_buildspecs (dict, optional): filters buildspecs and tests based on ``buildtest build --filter`` argument which is a key/value dictionary that can filter tests based on **tags**, **type**, and **maintainers** - rebuild (int, optional): Rebuild tests X times based on ``buildtest build --rebuild`` option. - report_file (str, optional): Location to report file where test data will be written upon completion. This can be specified via ``buildtest build --report`` command + helpfilter (bool, optional): Display available filter fields for ``buildtest build --filter`` command. This argument is set to ``True`` if one specifies ``buildtest build --helpfilter`` + limit (int, optional): Limit number of tests that can be run. This option is specified by ``buildtest build --limit`` + max_jobs (int, optional): Maximum number of jobs to run concurrently. This option is specified by ``buildtest build --max-jobs`` maxpendtime (int, optional): Specify maximum pending time in seconds for batch job until job is cancelled + modulepurge (bool, optional): Determine whether to run 'module purge' before running test. This is specified via ``buildtest build --modulepurge``. + modules (str, optional): List of modules to load for every test specified via ``buildtest build --modules``. + name (list, optional): list of test names to run specified via command line ``buildtest build --name`` + numnodes (list, optional): List of comma separated nodes values to run batch jobs specified via ``buildtest build --nodes`` + numprocs (list, optional): List of comma separated process values to run batch jobs specified via ``buildtest build --procs`` poll_interval (int, optional): Specify poll interval in seconds for polling batch jobs. + profile (str, optional): Profile to load from buildtest configuration specified by ``buildtest build --profile`` + rebuild (int, optional): Rebuild tests X times based on ``buildtest build --rebuild`` option. remove_stagedir (bool, optional): remove stage directory after job completion - retry (int, optional): Number of retry for failed jobs - account (str, optional): Project account to charge jobs. This takes input argument ``buildtest build --account`` - helpfilter (bool, optional): Display available filter fields for ``buildtest build --filter`` command. This argument is set to ``True`` if one specifies ``buildtest build --helpfilter`` - numprocs (list, optional): List of comma separated process values to run batch jobs specified via ``buildtest build --procs`` - numnodes (list, optional): List of comma separated nodes values to run batch jobs specified via ``buildtest build --nodes`` - modules (str, optional): List of modules to load for every test specified via ``buildtest build --modules``. - modulepurge (bool, optional): Determine whether to run 'module purge' before running test. This is specified via ``buildtest build --modulepurge``. - unload_modules (str, optional): List of modules to unload for every test specified via ``buildtest build --unload-modules``. + report_file (str, optional): Location to report file where test data will be written upon completion. This can be specified via ``buildtest build --report`` command rerun (bool, optional): Rerun last successful **buildtest build** command. This is specified via ``buildtest build --rerun``. All other options will be ignored and buildtest will read buildtest options from file **BUILDTEST_RERUN_FILE**. - executor_type (bool, optional): Filter test by executor type. This option will filter test after discovery by local or batch executors. This can be specified via ``buildtest build --exec-type`` - timeout (int, optional): Test timeout in seconds specified by ``buildtest build --timeout`` - limit (int, optional): Limit number of tests that can be run. This option is specified by ``buildtest build --limit`` + retry (int, optional): Number of retry for failed jobs save_profile (str, optional): Save profile to buildtest configuration specified by ``buildtest build --save-profile`` - profile (str, optional): Profile to load from buildtest configuration specified by ``buildtest build --profile`` - max_jobs (int, optional): Maximum number of jobs to run concurrently. This option is specified by ``buildtest build --max-jobs`` strict (bool, optional): Enable strict mode for buildtest. This option is specified by ``buildtest build --strict`` + tags (list, optional): list if tags to discover tests specified via command line ``buildtest build --tags`` + testdir (str): Path to test directory where tests are written. This argument can be passed from command line ``buildtest build --testdir`` + timeout (int, optional): Test timeout in seconds specified by ``buildtest build --timeout`` + unload_modules (str, optional): List of modules to unload for every test specified via ``buildtest build --unload-modules``. + validate (bool, optional): Validate given buildspecs and buildtest will stop after parse stage which can be configured via ``buildtest build --validate`` option verbose (bool, optional): Enable verbose output for buildtest that is specified by ``buildtest --verbose`` write_config_file (str, optional): Write configuration file to specified location. This is specified by ``buildtest build --write-config-file`` + """ self.verbose = verbose @@ -689,6 +693,7 @@ def __init__( exclude_tags, executors, name, + display, ]: if arg_name and not isinstance(arg_name, list): raise BuildTestError(f"{arg_name} is not of type list") @@ -718,6 +723,7 @@ def __init__( self.remove_stagedir = remove_stagedir self.configuration = configuration self.buildspecs = buildspecs + self.display = display self.exclude_buildspecs = exclude_buildspecs self.tags = tags self.name = name @@ -869,8 +875,8 @@ def load_rerun_file(self): configuration.detect_system() configuration.validate() self.configuration = configuration - self.buildspecs = content["buildspecs"] + self.display = content["display"] self.tags = content["tags"] self.exclude_tags = content["exclude_tags"] self.name = content["name"] @@ -903,6 +909,7 @@ def save_rerun_file(self): buildtest_cmd = { "configuration": self.configuration.file, "buildspecs": self.buildspecs, + "display": self.display, "tags": self.tags, "exclude_tags": self.exclude_tags, "name": self.name, @@ -969,6 +976,7 @@ def save_profile_to_configuration(self): profile_configuration = { "buildspecs": resolved_buildspecs or None, "exclude-buildspecs": self.exclude_buildspecs, + "display": self.display, "tags": self.tags, "exclude-tags": self.exclude_tags, "name": self.name, @@ -1053,6 +1061,7 @@ def load_profile(self): self.buildspecs = profile_configuration.get("buildspecs") self.exclude_buildspecs = profile_configuration.get("exclude-buildspecs") + self.display = profile_configuration.get("display") self.tags = profile_configuration.get("tags") self.exclude_tags = profile_configuration.get("exclude-tags") self.name = profile_configuration.get("name") @@ -1219,6 +1228,7 @@ def parse_buildspecs(self): executor_type=self.executor_type, exclude_tags=self.exclude_tags, strict=self.strict, + display=self.display, ) if not builder.get_builders(): diff --git a/buildtest/main.py b/buildtest/main.py index 70efa69eb..7cd4dfdda 100644 --- a/buildtest/main.py +++ b/buildtest/main.py @@ -294,38 +294,39 @@ def handle_build_command(args, configuration, report_file): stdout_file = tempfile.NamedTemporaryFile(delete=True, suffix=".txt") with Tee(stdout_file.name): cmd = BuildTest( - configuration=configuration, + account=args.account, buildspecs=args.buildspec, + configuration=configuration, + display=args.display, + dry_run=args.dry_run, exclude_buildspecs=args.exclude, - executors=args.executor, - tags=args.tags, - name=args.name, exclude_tags=args.exclude_tags, + executors=args.executor, + executor_type=args.executor_type, filter_buildspecs=args.filter, - rebuild=args.rebuild, - validate=args.validate, - dry_run=args.dry_run, - testdir=args.testdir, - report_file=report_file, - maxpendtime=args.maxpendtime, - poll_interval=args.pollinterval, - remove_stagedir=args.remove_stagedir, - retry=args.retry, - account=args.account, helpfilter=args.helpfilter, - numprocs=args.procs, - numnodes=args.nodes, + limit=args.limit, + max_jobs=args.max_jobs, + maxpendtime=args.maxpendtime, modules=args.modules, modulepurge=args.module_purge, - unload_modules=args.unload_modules, + numnodes=args.nodes, + numprocs=args.procs, + name=args.name, + poll_interval=args.pollinterval, + profile=args.profile, + rebuild=args.rebuild, + remove_stagedir=args.remove_stagedir, + report_file=report_file, rerun=args.rerun, - executor_type=args.executor_type, - timeout=args.timeout, - limit=args.limit, + retry=args.retry, save_profile=args.save_profile, - profile=args.profile, - max_jobs=args.max_jobs, strict=args.strict, + tags=args.tags, + testdir=args.testdir, + timeout=args.timeout, + unload_modules=args.unload_modules, + validate=args.validate, verbose=args.verbose, write_config_file=args.write_config_file, ) diff --git a/buildtest/schemas/settings.schema.json b/buildtest/schemas/settings.schema.json index 030b9f8b6..174d9b02d 100644 --- a/buildtest/schemas/settings.schema.json +++ b/buildtest/schemas/settings.schema.json @@ -682,6 +682,7 @@ "unload-modules": {"type": "string"}, "module-purge": {"type": "boolean"}, "validate": {"type": "boolean", "description": "Validate given buildspecs and buildtest will stop after parse stage"}, + "display": {"$ref": "definitions.schema.json#/definitions/list_of_strings"}, "dry-run": {"type": "boolean", "description": "Show a list of tests that will potentially be run"}, "rebuild": {"type": "integer", "minimum": 1, "maximum": 50, "description": "Specify number of tests to rebuild"}, "limit": {"type": "integer", "minimum": 1, "description": "Limit number of tests to build"}, diff --git a/docs/gettingstarted/buildingtest.rst b/docs/gettingstarted/buildingtest.rst index 04cff5ef9..763814d80 100644 --- a/docs/gettingstarted/buildingtest.rst +++ b/docs/gettingstarted/buildingtest.rst @@ -618,8 +618,8 @@ then proceed to next test. .. command-output:: buildtest build -b tutorials/hello_world.yml --rebuild=5 --max-jobs=2 -Strict Mode ------------- +Strict Mode (``buildtest build --strict``) +------------------------------------------- Buildtest has an option to enable strict mode for test execution which can be enabled via ``--strict`` option. If this is set, buildtest will instead ``set -eo pipefail`` in the generated test which will cause test to exit immediately if any @@ -645,4 +645,19 @@ Now let's run the same test with strict mode enabled, we will see the test will We can see the generated test using **buildtest inspect query -t** and we will see the test script has **set -eo pipefail** in the generated test. - .. command-output:: buildtest inspect query -t linux_strict_test \ No newline at end of file + .. command-output:: buildtest inspect query -t linux_strict_test + +Display Mode (``buildtest build --display``) +--------------------------------------------- + +Buildtest can display output of test content and stream outout and error file to console. This can be useful +if you want to see how the test is generated for debugging purposes. + +In order to use this functionality, you can specify the ``--display`` option which takes either ``output`` or ``test``. +When ``output`` is specified, buildtest will display output and error files to console. When ``test`` +is specified, buildtest will display the content of the test and build script. You can append the ``--display`` option +if you want to specify both options. Shown below we run a test and display both output and test. + +.. dropdown:: ``buildtest build -b tutorials/vars.yml --display output --display test`` + + .. command-output:: buildtest build -b tutorials/vars.yml --display output --display test diff --git a/tests/cli/test_build.py b/tests/cli/test_build.py index 61c52882e..229d7ce05 100644 --- a/tests/cli/test_build.py +++ b/tests/cli/test_build.py @@ -416,6 +416,7 @@ def test_save_profile(self): BuildTest( configuration=buildtest_configuration, buildspecs=buildspecs, + display=["output", "test"], exclude_buildspecs=buildspecs, tags=["python"], executors=["generic.local.csh"], @@ -524,6 +525,15 @@ def test_retry(self): cmd = BuildTest(configuration=configuration, buildspecs=buildspecs, retry=2) cmd.build() + def test_display(self): + buildspecs = [os.path.join(BUILDTEST_ROOT, "tutorials", "post_run.yml")] + cmd = BuildTest( + configuration=configuration, + buildspecs=buildspecs, + display=["output", "test"], + ) + cmd.build() + def test_discover(): # test single buildspec file