Regenerate AWS tutorial examples

aws_tutorial/mpi_job_submission.yml

@@ -0,0 +1,11 @@
+buildspecs:
+  mpi_job_submission:
+    type: script
+    executor: generic.torque.e4spro-cluster
+    description: Run MPI Proc Name test
+    pbs: ["-l nodes=1:ppn=2,walltime=1:00"]
+    run: |
+      cd $HOME/examples/mpi-procname            
+      mpicc -o mpiprocname mpiprocname.c
+      mpirun -np 2 ./mpiprocname
+      rm mpiprocname

aws_tutorial/openmp_example_custom_compiler.yml

@@ -6,7 +6,7 @@ buildspecs:
     compilers:
       name: ["gcc"]
       config:
-        gcc_9.4.0:
+        gcc_12.3.0:
           env: 
             OMP_NUM_THREADS: "2"
           cflags: "-O1 -fopenmp"  

buildtest/cli/config.py

@@ -68,7 +68,7 @@ def handle_executors_command(command_args, configuration):
 
 def handle_validate_command(command_args, configuration, system):
     if command_args.config in ["validate", "val"]:
-        validate_config(configuration, system.system["moduletool"])
+        validate_config(configuration)
 
 
 def handle_systems_command(command_args, configuration):

buildtest/settings/aws.yml

@@ -21,6 +21,9 @@ system:
         bash:
           description: submit jobs on local machine using bash shell
           shell: bash
+      torque:
+        e4spro-cluster:
+          queue: e4spro-cluster
     compilers:
       find:
         gcc: ^(gcc)
@@ -30,10 +33,10 @@ system:
             cc: /usr/bin/gcc
             fc: /usr/bin/gfortran
             cxx: /usr/bin/g++
-          gcc_9.4.0:
-            cc: /usr/bin/gcc-9
-            fc: /usr/bin/gfortran-9
-            cxx: /usr/bin/g++-9
+          gcc_12.3.0:
+            cc: /usr/bin/gcc-12
+            fc: /usr/bin/gfortran-12
+            cxx: /usr/bin/g++-12
     cdash:
       url: https://my.cdash.org/
       project: buildtest

buildtest/tools/tutorialexamples.py

@@ -21,9 +21,10 @@ def generate_tutorial_examples(examples, dryrun=None, write=None, failfast=None)
         failfast (bool, optional): If set to True, we will exit on first failure. Default is None.
     """
 
-    settings_file = None
     autogen_examples_dir = None
-    moduletool = "none"
+
+    aws_settings = os.path.join(BUILDTEST_ROOT, "buildtest", "settings", "aws.yml")
+    settings_file = TUTORIALS_SETTINGS_FILE if examples == "spack" else aws_settings
 
     if examples == "spack":
 
@@ -35,24 +36,19 @@ def generate_tutorial_examples(examples, dryrun=None, write=None, failfast=None)
         autogen_examples_dir = os.path.join(
             BUILDTEST_ROOT, "docs", "buildtest_tutorial_examples"
         )
-        settings_file = TUTORIALS_SETTINGS_FILE
-        moduletool = "none"
 
     else:
 
-        if getpass.getuser() != "ubuntu" or os.getenv("HOME") != "/home/ubuntu":
+        if getpass.getuser() not in ["ubuntu", "lbladmin"]:
             sys.exit(
                 "This script can only be run in AWS instance using E4SPro image. Please check the AWS Market Place: https://aws.amazon.com/marketplace for the image "
             )
 
         autogen_examples_dir = os.path.join(BUILDTEST_ROOT, "docs", "aws_examples")
-        settings_file = os.path.join(BUILDTEST_ROOT, "buildtest", "settings", "aws.yml")
-
-        moduletool = "environment-modules"
 
     config = SiteConfiguration(settings_file=settings_file)
     config.detect_system()
-    config.validate(moduletool=moduletool)
+    config.validate()
 
     if write:
 
@@ -68,11 +64,19 @@ def generate_tutorial_examples(examples, dryrun=None, write=None, failfast=None)
 
     if examples == "spack":
         build_spack_examples(
-            autogen_examples_dir, dryrun=dryrun, write=write, failfast=failfast
+            autogen_dir=autogen_examples_dir,
+            settings_file=settings_file,
+            dryrun=dryrun,
+            write=write,
+            failfast=failfast,
         )
     else:
         build_aws_examples(
-            autogen_examples_dir, dryrun=dryrun, write=write, failfast=failfast
+            build_dir=autogen_examples_dir,
+            settings_file=settings_file,
+            dryrun=dryrun,
+            write=write,
+            failfast=failfast,
         )
 
 
@@ -85,9 +89,17 @@ def run(query):
     """
 
     console.print(f"Executing Command: {query}")
-    command = subprocess.run(
-        [query], shell=True, check=True, universal_newlines=True, capture_output=True
-    )
+    try:
+        command = subprocess.run(
+            [query],
+            shell=True,
+            check=True,
+            universal_newlines=True,
+            capture_output=True,
+        )
+    except subprocess.CalledProcessError as err:
+        console.print(f"[red]Error: {err}")
+        return (err.stdout, err.returncode)
 
     return (command.stdout, command.returncode)
 
@@ -118,11 +130,14 @@ def write_example(fname, command, content):
     console.print(firstNlines)
 
 
-def build_aws_examples(build_dir, dryrun=None, write=None, failfast=None):
+def build_aws_examples(
+    build_dir, settings_file, dryrun=None, write=None, failfast=None
+):
     """This method will build AWS examples for the tutorial
 
     Args:
         build_dir (str): Directory where auto generated documentation examples will be written.
+        settings_file (str): Path to settings file
         dryrun (bool, optional): If True we print commands to run and return. If False we execute commands. Defaults to None.
         write (bool, optional): If True we write output to file. Defaults to None.
         failfast (bool, optional): If True we exit on first failure. Defaults to None.
@@ -131,51 +146,37 @@ def build_aws_examples(build_dir, dryrun=None, write=None, failfast=None):
     AWS_EXAMPLE_DIR = os.path.join(BUILDTEST_ROOT, "aws_tutorial")
 
     commands_to_run = {
-        f"{build_dir}/hello_build.txt": f"buildtest build -b {AWS_EXAMPLE_DIR}/hello_world/hello.yml",
+        f"{build_dir}/hello_build.txt": f"buildtest -c {settings_file} build -b {AWS_EXAMPLE_DIR}/hello_world/hello.yml",
         f"{build_dir}/hello_inspect.txt": "buildtest inspect query -o -t hello_world_example",
-        f"{build_dir}/multi_compiler_hello_build.txt": f"buildtest build -b {AWS_EXAMPLE_DIR}/hello_world/multi_compiler_hello.yml",
+        f"{build_dir}/multi_compiler_hello_build.txt": f"buildtest -c {settings_file} build -b {AWS_EXAMPLE_DIR}/hello_world/multi_compiler_hello.yml",
         f"{build_dir}/multi_compiler_hello_inspect.txt": "buildtest inspect query -o -t hello_world_multi_compiler/",
-        f"{build_dir}/compiler_list_yaml.txt": "buildtest config compilers list --yaml",
-        f"{build_dir}/mpiproc_build.txt": f"buildtest build -b {AWS_EXAMPLE_DIR}/mpiproc.yml",
+        f"{build_dir}/compiler_list_yaml.txt": f"buildtest -c {settings_file} config compilers list --yaml",
+        f"{build_dir}/mpiproc_build.txt": f"buildtest -c {settings_file} build -b {AWS_EXAMPLE_DIR}/mpiproc.yml",
         f"{build_dir}/mpiproc_inspect.txt": "buildtest inspect query -o mpiprocname",
-        f"{build_dir}/osu_bandwidth_test_build.txt": f"buildtest build -b {AWS_EXAMPLE_DIR}/osu_bandwidth_test.yml",
+        f"{build_dir}/osu_bandwidth_test_build.txt": f"buildtest -c {settings_file} build -b {AWS_EXAMPLE_DIR}/osu_bandwidth_test.yml",
         f"{build_dir}/osu_bandwidth_test_inspect.txt": "buildtest inspect query -o osu_bandwidth osu_bandwidth_perf",
-        f"{build_dir}/openmp_example_build.txt": f"buildtest build -b {AWS_EXAMPLE_DIR}/openmp_example_custom_compiler.yml",
+        f"{build_dir}/openmp_example_build.txt": f"buildtest -c {settings_file} build -b {AWS_EXAMPLE_DIR}/openmp_example_custom_compiler.yml",
         f"{build_dir}/openmp_example_inspect.txt": "buildtest inspect query -o -t hello_world_openmp_custom_compiler/",
-        f"{build_dir}/docker_helloworld_build.txt": f"buildtest build -b {BUILDTEST_ROOT}/tutorials/containers/hello_world.yml",
+        f"{build_dir}/docker_helloworld_build.txt": f"buildtest -c {settings_file} build -b {BUILDTEST_ROOT}/tutorials/containers/hello_world.yml",
         f"{build_dir}/docker_helloworld_inspect.txt": "buildtest inspect query -o -t hello_world_docker",
-        f"{build_dir}/singularity_helloworld_build.txt": f"buildtest build -b {BUILDTEST_ROOT}/tutorials/containers/hello_world_singularity.yml",
+        f"{build_dir}/singularity_helloworld_build.txt": f"buildtest -c {settings_file} build -b {BUILDTEST_ROOT}/tutorials/containers/hello_world_singularity.yml",
         f"{build_dir}/singularity_helloworld_inspect.txt": "buildtest inspect query -o -t hello_world_singularity",
         f"{build_dir}/container_executor_list.txt": "buildtest -c $BUILDTEST_ROOT/buildtest/settings/container_executor.yml config executors list --yaml",
         f"{build_dir}/container_executor_build.txt": "buildtest -c $BUILDTEST_ROOT/buildtest/settings/container_executor.yml build -b $BUILDTEST_ROOT/tutorials/containers/container_executor/ubuntu.yml",
         f"{build_dir}/container_executor_inspect.txt": "buildtest inspect query -o -t -b ubuntu_container_example",
+        f"{build_dir}/mpi_job_submission_build.txt": f"buildtest -c {settings_file} build -b {AWS_EXAMPLE_DIR}/mpi_job_submission.yml --pollinterval=10 --display output --display test",
     }
+    execute_commands(commands_to_run, dryrun=dryrun, write=write, failfast=failfast)
 
-    if dryrun:
-        for command in commands_to_run.values():
-            console.print(command)
-        return
-
-    for fname, command in commands_to_run.items():
-        content, retcode = run(command)
-
-        # for non-negative returncode
-        if retcode != 0:
-            console.print(f"[red]Returncode: {retcode}")
-            if failfast:
-                sys.exit(1)
-
-        console.print(f"[green]Returncode: {retcode}")
-
-        if write:
-            write_example(fname=fname, command=command, content=content)
 
-
-def build_spack_examples(autogen_dir, dryrun=None, write=None, failfast=None):
+def build_spack_examples(
+    autogen_dir, settings_file, dryrun=None, write=None, failfast=None
+):
     """This method will build spack examples for the tutorial
 
     Args:
         autogen_dir (str): Directory where auto generated documentation examples will be written.
+        settings_file (str): Path to settings file
         dryrun (bool, optional): If True we print commands to run and return. If False we execute commands. Defaults to None.
         write (bool, optional): If True we write output to file. Defaults to None.
         failfast (bool, optional): If True we exit on first failure. Defaults to None.
@@ -218,6 +219,18 @@ def build_spack_examples(autogen_dir, dryrun=None, write=None, failfast=None):
         f"{inspect_dir}/clone_spack.txt": "buildtest inspect query -o -t clone_spack_automatically clone_spack_and_specify_root",
         f"{inspect_dir}/e4s_testsuite_mpich.txt": "buildtest inspect query -o -e -t mpich_e4s_testsuite",
     }
+    execute_commands(commands_to_run, dryrun=dryrun, write=write, failfast=failfast)
+
+
+def execute_commands(commands_to_run, dryrun=None, write=None, failfast=None):
+    """This method will execute a list of commands and handle the results.
+
+    Args:
+        commands_to_run (dict): A dictionary where keys are file names and values are commands to be executed.
+        dryrun (bool, optional): If True we print commands to run and return. If False we execute commands. Defaults to None.
+        write (bool, optional): If True we write output to file. Defaults to None.
+        failfast (bool, optional): If True we exit on first failure. Defaults to None.
+    """
 
     if dryrun:
         for command in commands_to_run.values():
@@ -232,6 +245,7 @@ def build_spack_examples(autogen_dir, dryrun=None, write=None, failfast=None):
             console.print(f"[red]Returncode: {retcode}")
             if failfast:
                 sys.exit(1)
+            continue
 
         console.print(f"[green]Returncode: {retcode}")
 

docs/aws_examples/compiler_list_yaml.txt

@@ -1,11 +1,11 @@
-$ buildtest config compilers list --yaml 
+$ buildtest -c /home/lbladmin/Documents/buildtest/buildtest/settings/aws.yml config compilers list --yaml 
 gcc:
   gcc_11.4.0:
     cc: /usr/bin/gcc
     cxx: /usr/bin/g++
     fc: /usr/bin/gfortran
-  gcc_9.4.0:
-    cc: /usr/bin/gcc-9
-    cxx: /usr/bin/g++-9
-    fc: /usr/bin/gfortran-9
+  gcc_12.3.0:
+    cc: /usr/bin/gcc-12
+    cxx: /usr/bin/g++-12
+    fc: /usr/bin/gfortran-12