From 8d29f65c15875b5f5f5d6301efbd28ce2919d756 Mon Sep 17 00:00:00 2001 From: bch0w Date: Fri, 9 Sep 2022 17:20:14 -0800 Subject: [PATCH] examples added MPI capabilities natively to base example, allow user to flag '--with_mpi' when running example, removed example 4 since we can just run example 3 with MPI which is the same thing --- seisflows/examples/ex4_multicore_fwd.py | 153 ------------------------ seisflows/examples/sfexample2d.py | 52 +++++++- seisflows/seisflows.py | 5 + 3 files changed, 52 insertions(+), 158 deletions(-) delete mode 100644 seisflows/examples/ex4_multicore_fwd.py diff --git a/seisflows/examples/ex4_multicore_fwd.py b/seisflows/examples/ex4_multicore_fwd.py deleted file mode 100644 index fd6d6809..00000000 --- a/seisflows/examples/ex4_multicore_fwd.py +++ /dev/null @@ -1,153 +0,0 @@ -#!/usr/bin/env python3 -""" - SEISFLOWS SPECFEM2D WORKSTATION EXAMPLE 4 - -This example mimics Example 3 (mass forward simulations) but uses the -parallelized version of SPECFEM2D to test out multi-core functionality. - -.. note:: - See Example 1 docstring for more information - -.. rubric:: - $ seisflows examples run 3 -""" -import os -import sys -import subprocess -from seisflows.tools import msg -from seisflows.tools.unix import cd, rm, ln -from seisflows.examples.sfexample2d import SFExample2D - - -class SFMultiCoreEx2D(SFExample2D): - """ - A class for running SeisFlows examples. Overloads Example 1 to take - advantage of the default SPECFEM2D stuff, onyl changes the generation of - MODEL TRUE, the number of stations, and the setup of the parameter file. - """ - def __init__(self, ntask=None, nsta=None, nproc=None, - method="run", specfem2d_repo=None, mpiexec="mpirun", **kwargs): - """ - Overloads init of the base problem - - :type ntask: int - :param ntask: number of events to use in inversion, between 1 and 25. - defaults to 3 - :type nsta: int - :param nsta: number of stations to include in inversion, between 1 and - 131 - :type nproc: int - :param nproc: number of processors to be sent to MPI executable - :type method: str - :param method: method for running the example problem, can be: - * 'run': setup and run the example problem - * 'setup': only setup the example problem, do not `submit` job - :type specfem2d_repo: str - :param specfem2d_repo: path to the SPECFEM2D directory which should - contain binary executables. If not given, SPECFEM2D will be - downloaded configured and compiled automatically. - """ - # We set defaults here because `seisflows examples` may input these - # values as NoneType which would override __init__ defaults. - super().__init__(ntask=ntask or 10, nsta=nsta or 25, nproc=nproc or 4, - niter=1, method=method, specfem2d_repo=specfem2d_repo) - - # Check that the MPI executable actually exists. Sometimes e.g., if we - # don't '$ module load mpi', we will get 'mpirun: command not found' - try: - subprocess.run(f"which {mpiexec}", check=True, shell=True, - stdout=subprocess.DEVNULL) - except subprocess.CalledProcessError: - print(msg.cli(f"MPI executable '{mpiexec}' not found on system. " - f"Please check that you have MPI installed/loaded. " - f"If '{mpiexec}' is not how you invoke MPI, use the " - "flag `--mpiexec {exc}` when running the example to " - "change the default exectable", header="missing mpi", - border="=") - ) - sys.exit(-1) - - self.mpiexec = mpiexec - self._parameters["nproc"] = self.nproc - self._parameters["mpiexec"] = self.mpiexec - - self._modules = { - "workflow": "forward", - "preprocess": "null", - "optimize": "null", - } - - self._parameters["export_traces"] = True - self._parameters["path_model_true"] = "null" # overload default par. - - # Overwrite configure cmd to get MPI - self._configure_cmd = \ - "./configure FC=gfortran CC=gcc MPIF90=mpif90 --with-mpi" - - def print_dialogue(self): - """ - Print help/system dialogue message that explains the setup of th - this workflow - """ - print(msg.ascii_logo_small) - print(msg.cli( - f"This is a [SPECFEM2D] [WORKSTATION] example, which will run " - f"forward simulations to generate synthetic seismograms through " - f"a homogeneous halfspace starting model. This example uses no " - f"preprocessing or optimization modules. This example uses MPI to " - f"run the external solver, SPECFEM2D." - f"[{self.ntask} events, {self.nsta} stations] " - f"The tasks involved include: ", - items=["1. (optional) Download, configure, compile " - "SPECFEM2D w/ MPI", - "2. [Setup] a SPECFEM2D working directory", - "3. [Setup] starting model from 'Tape2007' example", - "4. [Setup] a SeisFlows working directory", - "5. [Run] the forward simulation workflow"], - header="seisflows example 4", - border="=") - ) - - def run_xspecfem2d_binaries(self): - """ - Runs the xmeshfem2d and then xspecfem2d binaries using subprocess and then - do some cleanup to get files in the correct locations. Assumes that we - can run the binaries directly with './' - """ - cd(self.workdir_paths.workdir) - - mpicmd = f"{self.mpiexec} -n {self.nproc}" - cmd_mesh = f"{mpicmd} bin/xmeshfem2D > OUTPUT_FILES/mesher.log.txt" - cmd_spec = f"{mpicmd} bin/xspecfem2D > OUTPUT_FILES/solver.log.txt" - - for cmd in [cmd_mesh, cmd_spec]: - print(f"Running SPECFEM2D with command: {cmd}") - subprocess.run(cmd, shell=True, check=True, - stdout=subprocess.DEVNULL) - - def main(self): - """ - Setup the example and then optionally run the actual seisflows workflow - Mostly the same as Example 1 main() except it does not generate - MODEL_TRUE, and instead sets MODEL_TRUE as the starting model. - """ - print(msg.cli("EXAMPLE SETUP", border="=")) - - # Step 1: Download and configure SPECFEM2D, make binaries. Optional - self.download_specfem2d() - self.configure_specfem2d() - self.make_specfem2d_executables() - # Step 2: Create a working directory and generate initial/final models - self.create_specfem2d_working_directory() - # Step 2a: Generate MODEL_INIT, rearrange consequent directory structure - print(msg.cli("GENERATING INITIAL MODEL", border="=")) - self.setup_specfem2d_for_model_init() # setup SPECFEM run directory - self.run_xspecfem2d_binaries() - self.cleanup_xspecfem2d_run(choice="INIT") - # Step 3: Prepare Par_file and directory for MODEL_TRUE generation - self.setup_seisflows_working_directory() - self.finalize_specfem2d_par_file() - print(msg.cli("COMPLETE EXAMPLE SETUP", border="=")) - # Step 4: Run the workflwo - if self.run_example: - self.run_sf_example() diff --git a/seisflows/examples/sfexample2d.py b/seisflows/examples/sfexample2d.py index cbc2890c..112fa15f 100644 --- a/seisflows/examples/sfexample2d.py +++ b/seisflows/examples/sfexample2d.py @@ -45,7 +45,8 @@ class SFExample2D: multiple example runs can benefit from the code written here """ def __init__(self, ntask=None, event_id=None, niter=None, nsta=None, - nproc=None, method="run", specfem2d_repo=None, **kwargs): + nproc=None, method="run", specfem2d_repo=None, with_mpi=False, + mpiexec="mpirun", **kwargs): """ Set path structure which is used to navigate around SPECFEM repositories and the example working directory @@ -70,6 +71,13 @@ def __init__(self, ntask=None, event_id=None, niter=None, nsta=None, :param specfem2d_repo: path to the SPECFEM2D directory which should contain binary executables. If not given, SPECFEM2D will be downloaded configured and compiled automatically. + :type with_mpi: bool + :param with_mpi: run the example problem with MPI. That is, runs the + Solver with an MPI executable. All other tasks are run in serial. + :type mpiexec: str + :param mpiexec: MPI executable used to run MPI tasks. Defaults to + 'mpiexec' but User is allowed to choose incase their system has + different MPI run call. """ self.cwd = os.getcwd() self.sem2d_paths, self.workdir_paths = self.define_dir_structures( @@ -96,6 +104,7 @@ def __init__(self, ntask=None, event_id=None, niter=None, nsta=None, # -1 because it represents index but we need to talk in terms of count assert(1 <= self.nsta <= 132), \ f"number of stations must be between 1 and 131, not {self.nsta}" + self.nproc = nproc or 1 # must be 1 for Examples 1-3 # This bool information is provided by the User running 'setup' or 'run' @@ -132,8 +141,16 @@ def __init__(self, ntask=None, event_id=None, niter=None, nsta=None, "path_model_true": self.workdir_paths.model_true, } - # Used to configure SPECFEM2D binaries - self._configure_cmd = "./configure" + # Determine if we are running serial or parallel solver tasks + if with_mpi: + self.mpiexec = mpiexec + self._parameters["mpiexec"] = self.mpiexec # Pass on to workflow + self.check_mpi_executable() + self._configure_cmd = \ + "./configure FC=gfortran CC=gcc MPIF90=mpif90 --with-mpi" + else: + self.mpixec = None + self._configure_cmd = "./configure" def print_dialogue(self): """ @@ -157,6 +174,26 @@ def print_dialogue(self): border="=") ) + def check_mpi_executable(self): + """ + If User wants to run examples with MPI, checks that MPI executable is + available and can be used to run solver. Sometimes if we don't + '$ module load mpi', we will get 'mpirun: command not found' + """ + try: + subprocess.run(f"which {self.mpiexec}", check=True, shell=True, + stdout=subprocess.DEVNULL) + except subprocess.CalledProcessError: + print( + msg.cli(f"MPI executable '{self.mpiexec}' not found on " + f"system. Please check that you have MPI " + f"installed/loaded. If '{self.mpiexec}' is not how you " + f"invoke MPI, use the flag `--mpiexec $MPIFLAG` when " + f"running the example to change the default exectable", + header="missing mpi", border="=") + ) + sys.exit(-1) + @staticmethod def define_dir_structures(cwd, specfem2d_repo, ex="Tape2007"): """ @@ -335,8 +372,13 @@ def run_xspecfem2d_binaries(self): """ cd(self.workdir_paths.workdir) - cmd_mesh = f"./bin/xmeshfem2D > OUTPUT_FILES/mesher.log.txt" - cmd_spec = f"./bin/xspecfem2D > OUTPUT_FILES/solver.log.txt" + if self.mpiexec: + mpicmd = f"{self.mpiexec} -n {self.nproc}" + cmd_mesh = f"{mpicmd} bin/xmeshfem2D > OUTPUT_FILES/mesher.log.txt" + cmd_spec = f"{mpicmd} bin/xspecfem2D > OUTPUT_FILES/solver.log.txt" + else: + cmd_mesh = f"./bin/xmeshfem2D > OUTPUT_FILES/mesher.log.txt" + cmd_spec = f"./bin/xspecfem2D > OUTPUT_FILES/solver.log.txt" for cmd in [cmd_mesh, cmd_spec]: print(f"Running SPECFEM2D with command: {cmd}") diff --git a/seisflows/seisflows.py b/seisflows/seisflows.py index a979b7f7..07d8735c 100755 --- a/seisflows/seisflows.py +++ b/seisflows/seisflows.py @@ -338,6 +338,11 @@ def _format_action(self, action): "the Tape 2007 example (1 <= EVENT_ID <= 25). " "If not used, example will default to choosing " "sequential from 1 to NTASK") + examples.add_argument("--with_mpi", action="store_true", default=False, + help="Run Solver with MPI using MPI exectuable " + "`mpiexec` (defaults to 'mpirun'). Defaults to " + "False, in which case executables are run " + "in serial") examples.add_argument("--mpiexec", type=str, nargs="?", default="mpirun", help="Only for Example(s) 4. MPI executable to use " "when running SPECFEM2D with MPI. Defaults to "