abouteiller
diff --git a/‎README.FT.ULFM.md‎
Lines changed: 478 additions & 0 deletions b/‎README.FT.ULFM.md‎
Lines changed: 478 additions & 0 deletions
diff --git a/‎config/opal_setup_ft.m4‎
Lines changed: 43 additions & 7 deletions b/‎config/opal_setup_ft.m4‎
Lines changed: 43 additions & 7 deletions
diff --git a/‎configure.ac‎
Lines changed: 1 addition & 1 deletion b/‎configure.ac‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎contrib/Makefile.am‎
Lines changed: 4 additions & 0 deletions b/‎contrib/Makefile.am‎
Lines changed: 4 additions & 0 deletions
diff --git a/‎contrib/amca-param-sets/ft-mpi‎
Lines changed: 83 additions & 0 deletions b/‎contrib/amca-param-sets/ft-mpi‎
Lines changed: 83 additions & 0 deletions
diff --git a/‎contrib/platform/ft_mpi_ulfm‎
Lines changed: 19 additions & 0 deletions b/‎contrib/platform/ft_mpi_ulfm‎
Lines changed: 19 additions & 0 deletions
diff --git a/‎ompi/attribute/attribute_predefined.c‎
Lines changed: 18 additions & 2 deletions b/‎ompi/attribute/attribute_predefined.c‎
Lines changed: 18 additions & 2 deletions
diff --git a/‎ompi/communicator/Makefile.am‎
Lines changed: 6 additions & 1 deletion b/‎ompi/communicator/Makefile.am‎
Lines changed: 6 additions & 1 deletion
@@ -1,4 +1,8 @@
 dnl
+dnl Copyright (c) 2004-2018 The University of Tennessee and The University
+dnl                         of Tennessee Research Foundation.  All rights
+dnl                         reserved.
+dnl Copyright (c) 2009-2012 Oak Ridge National Labs.  All rights reserved.
 dnl Copyright (c) 2013-2014 Intel, Inc.  All rights reserved.
 dnl Copyright (c) 2015      Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
@@ -12,11 +16,15 @@ dnl
 #
 # --with-ft=TYPE
 #  TYPE:
+#    - mpi (synonym for 'ulfm')
 #    - LAM (synonym for 'cr' currently)
 #    - cr
 #  /* General FT sections */
 #  #if OPAL_ENABLE_FT == 0 /* FT Disabled globaly */
 #  #if OPAL_ENABLE_FT == 1 /* FT Enabled globaly */
+#  /* ULFM Specific sections */
+#  #if OPAL_ENABLE_FT_MPI == 0 /* FT ULFM Disabled */
+#  #if OPAL_ENABLE_FT_MPI == 1 /* FT ULFM Enabled */
 #  /* CR Specific sections */
 #  #if OPAL_ENABLE_FT_CR == 0 /* FT Ckpt/Restart Disabled */
 #  #if OPAL_ENABLE_FT_CR == 1 /* FT Ckpt/Restart Enabled */
@@ -33,9 +41,9 @@ AC_DEFUN([OPAL_SETUP_FT_OPTIONS],[
     opal_setup_ft_options="yes"
     AC_ARG_WITH(ft,
                 [AC_HELP_STRING([--with-ft=TYPE],
-                [Specify the type of fault tolerance to enable. Options: LAM (LAM/MPI-like), cr (Checkpoint/Restart), (default: disabled)])],
-                [opal_want_ft=1],
-                [opal_want_ft=0])
+                [Specify the type of fault tolerance to enable. Options: mpi (ULFM), LAM (LAM/MPI-like), cr (Checkpoint/Restart) (default: mpi)])],
+                [],
+                [with_ft=mpi]) # If not specified act as if --with-ft=mpi
 
     #
     # Checkpoint/restart enabled debugging
@@ -63,9 +71,11 @@ AC_DEFUN([OPAL_SETUP_FT],[
     if test "$opal_setup_ft_options" = "yes"; then
         AC_MSG_CHECKING([if want fault tolerance])
     fi
-    if test "x$with_ft" != "x" || test "$opal_want_ft" = "1"; then
+
+    if test x"$with_ft" != "xno"; then
         opal_want_ft=1
         opal_want_ft_cr=0
+        opal_want_ft_mpi=0
         opal_want_ft_type=none
 
         as_save_IFS=$IFS
@@ -74,8 +84,16 @@ AC_DEFUN([OPAL_SETUP_FT],[
             IFS=$as_save_IFS
 
             # Default value
-            if test "$opt" = "" || test "$opt" = "yes"; then
-                opal_want_ft_cr=1
+            if test "$opt" = "yes"; then
+                opal_want_ft_mpi=1
+            elif test "$opt" = "ULFM"; then
+                opal_want_ft_mpi=1
+            elif test "$opt" = "ulfm"; then
+                opal_want_ft_mpi=1
+            elif test "$opt" = "MPI"; then
+                opal_want_ft_mpi=1
+            elif test "$opt" = "mpi"; then
+                opal_want_ft_mpi=1
             elif test "$opt" = "LAM"; then
                 opal_want_ft_cr=1
             elif test "$opt" = "lam"; then
@@ -89,7 +107,9 @@ AC_DEFUN([OPAL_SETUP_FT],[
                 AC_MSG_ERROR([Cannot continue])
             fi
         done
-        if test "$opal_want_ft_cr" = 1; then
+        if test "$opal_want_ft_mpi" = 1; then
+            opal_want_ft_type="mpi"
+        elif test "$opal_want_ft_cr" = 1; then
             opal_want_ft_type="cr"
         fi
 
@@ -101,18 +121,33 @@ AC_DEFUN([OPAL_SETUP_FT],[
         AC_MSG_WARN([**************************************************])
     else
         opal_want_ft=0
+        opal_want_ft_mpi=0
         opal_want_ft_cr=0
         if test "$opal_setup_ft_options" = "yes"; then
             AC_MSG_RESULT([Disabled fault tolerance])
         fi
     fi
     AC_DEFINE_UNQUOTED([OPAL_ENABLE_FT], [$opal_want_ft],
                        [Enable fault tolerance general components and logic])
+    AC_DEFINE_UNQUOTED([OPAL_ENABLE_FT_MPI], [$opal_want_ft_mpi],
+                       [Enable fault tolerance MPI ULFM components and logic])
     AC_DEFINE_UNQUOTED([OPAL_ENABLE_FT_CR], [$opal_want_ft_cr],
                        [Enable fault tolerance checkpoint/restart components and logic])
     AM_CONDITIONAL(WANT_FT, test "$opal_want_ft" = "1")
+    AM_CONDITIONAL(WANT_FT_MPI, test "$opal_want_ft_mpi" = "1")
     AM_CONDITIONAL(WANT_FT_CR,  test "$opal_want_ft_cr" = "1")
 
+    if test "$opal_want_ft_mpi" = "1"; then
+        AC_MSG_CHECKING([loading UFLM FT MPI platform file additions])
+        if test -r "${srcdir}/contrib/platform/ft_mpi_ulfm" ; then
+            . ${srcdir}/contrib/platform/ft_mpi_ulfm
+            AC_MSG_RESULT([Loaded contrib/platform/ft_mpi_ulfm])
+            enable_mca_no_build="$enable_mca_no_build_ft,$enable_mca_no_build"
+        else
+            AC_MSG_RESULT([Not found])
+        fi
+    fi
+
     if test "$opal_setup_ft_options" = "yes"; then
         AC_MSG_CHECKING([if want checkpoint/restart enabled debugging option])
     fi
@@ -175,4 +210,5 @@ AC_DEFUN([OPAL_SETUP_FT],[
     AC_DEFINE_UNQUOTED([OPAL_ENABLE_FT_THREAD], [$opal_want_ft_thread],
                        [Enable fault tolerance thread in Open PAL])
     AM_CONDITIONAL(WANT_FT_THREAD, test "$opal_want_ft_thread" = "1")
+    OPAL_SUMMARY_ADD([[Miscellaneous]],[[Fault Tolerance support]],[unnecessary], [$with_ft (disabling components ${enable_mca_no_build_ft//,/ })])
 ])
@@ -1085,7 +1085,7 @@ AC_CACHE_SAVE
 # visible again
 #
 ###########################################################
-dnl OPAL_SETUP_FT_OPTIONS
+OPAL_SETUP_FT_OPTIONS
 ###########################################################
 # The following line is always required as it contains the
 # AC_DEFINE and AM_CONDITIONAL calls that set variables used
 
@@ -27,6 +27,10 @@
 amca_paramdir = $(AMCA_PARAM_SETS_DIR)
 dist_amca_param_DATA = amca-param-sets/example.conf
 
+if WANT_FT_MPI
+dist_amca_param_DATA += amca-param-sets/ft-mpi
+endif # WANT_FT_MPI
+
 if WANT_FT_CR
 dist_amca_param_DATA += \
 	amca-param-sets/ft-enable-cr \
 
@@ -0,0 +1,83 @@
+#
+# Copyright (c) 2020     The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+# An Aggregate MCA Parameter Set to setup an environment that can support
+# User-Level Failure Mitigation (ULFM) fault tolerance (must also be 
+# compiled in with --with-ft=mpi).
+#
+# Usage:
+#   shell$ mpirun --tune ft-mpi ./app
+#
+
+mpi_ft_enable=true
+
+#
+# Performance tuning parameters (default shown)
+#mpi_ft_detector=true
+#mpi_ft_detector_thread=false
+#mpi_ft_detector_rdma_heartbeat=false
+#mpi_ft_detector_period=3.
+#mpi_ft_detector_timeout=10.
+#
+
+
+#
+# Select only ULFM ready components
+# disabling non-tested and known broken components in FT-MPI builds
+#
+
+#
+# The following frameworks/components are TESTED
+# They handle faults amd should be prefered when running with FT.
+#   pml     ob1
+#   btl     tcp, self, sm(+xpmem,+cma), ugni, uct
+#   coll    base/basic, tuned, ftagree, libnbc
+pml=ob1
+threads=pthreads
+
+#
+# The following frameworks/components are UNTESTED, but **may** work.
+# They should run without faults, and **may** work with faults.
+# You may try and report if successfull.
+#   btl     ofi, portals4, smcuda, usnic, sm(+knem)
+#   coll    inter, sm, sync, cuda, monitoring
+#   pml     monitoring, v/vprotocol
+# We will disable only the components for which good components are known to exist.
+btl=^usnic
+# older versions of xpmem generate bus errors when the other end is dead.
+#btl_sm_single_copy_mechanism=cma
+
+
+#
+# The following frameworks/components are UNTESTED, and probably won't work.
+# They should run without faults, and will probably crash/deadlock after a fault.
+# You may try at your own risk.
+#   coll    hcoll, portals4
+#   topo    (all)
+#   osc     (all)
+#   io      (all)
+#   fcoll   (all)
+#   fbtl    (all)
+# We will disable only the components for which good components are known to exist.
+# Other untested components are selectable but will issue a runtime warning at
+# initiation if FT is enabled.
+coll=^hcoll,portals4
+
+#
+# The following frameworks/components are NOT WORKING. Do not enable these with FT.
+# They should run without faults, but will completely bypass FT capabilities.
+#   mtl     (all)
+#   pml     cm, crcpw, ucx
+mtl=^ofi,portals4,psm2
+# allready enforced by pml=ob1 above
+#pml=^cm,crcpw,ucx
+# allready enforced by threads=pthreads above
+#threads=^argobots,qthreads
+
@@ -0,0 +1,19 @@
+# configure time selection of components.
+
+# we have small modifications to libevent, better use the internal one.
+with_libevent=internal
+# at the moment using any PMIx4 should work.
+#with_pmix=internal
+
+# older versions of xpmem generate bus errors when the other end is dead.
+#with_cray_xpmem=no
+
+# These components are runtime-disabled when using
+#   `mpirun --tune ft-mpi`
+# You may also want to disable compilation alltogether.
+#enable_mca_no_build_ft=\
+#mtl,pml-cm,pml-crcpw,pml-yalla,pml-ucx,\
+#coll-cuda,coll-fca,coll-hcoll,coll-portals4,\
+#btl-usnic,btl-portals4,btl-scif,btl-smcuda,\
+#pml-monitoring,pml-v,vprotocol,crcp\
+
@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2019 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -52,6 +52,9 @@
  *
  * MPI_WTIME_IS_GLOBAL is set to 0 (a conservative answer).
  *
+ * MPI_FT is set to 0 or 1 (according to OPAL_ENABLE_FT_MPI and
+ * ompi_ftmpi_enabled)
+ *
  * MPI_APPNUM is set as the result of a GPR subscription.
  *
  * MPI_LASTUSEDCODE is set to an initial value and is reset every time
@@ -123,7 +126,11 @@ int ompi_attr_create_predefined(void)
         OMPI_SUCCESS != (ret = create_win(MPI_WIN_SIZE)) ||
         OMPI_SUCCESS != (ret = create_win(MPI_WIN_DISP_UNIT)) ||
         OMPI_SUCCESS != (ret = create_win(MPI_WIN_CREATE_FLAVOR)) ||
-        OMPI_SUCCESS != (ret = create_win(MPI_WIN_MODEL))) {
+        OMPI_SUCCESS != (ret = create_win(MPI_WIN_MODEL)) ||
+#if OPAL_ENABLE_FT_MPI
+        OMPI_SUCCESS != (ret = create_comm(MPI_FT, true)) ||
+#endif /* OPAL_ENABLE_FT_MPI */
+        0) {
         return ret;
     }
 
@@ -133,6 +140,12 @@ int ompi_attr_create_predefined(void)
         OMPI_SUCCESS != (ret = set_f(MPI_HOST, MPI_PROC_NULL)) ||
         OMPI_SUCCESS != (ret = set_f(MPI_IO, MPI_ANY_SOURCE)) ||
         OMPI_SUCCESS != (ret = set_f(MPI_WTIME_IS_GLOBAL, 0)) ||
+#if OPAL_ENABLE_FT_MPI
+        /* Although we always define the key to ease fortran integration,
+         * lets not set a default value to the attribute if we do not 
+         * have fault tolerance built in. */
+        OMPI_SUCCESS != (ret = set_f(MPI_FT, ompi_ftmpi_enabled)) ||
+#endif /* OPAL_ENABLE_FT_MPI */
         OMPI_SUCCESS != (ret = set_f(MPI_LASTUSEDCODE,
                                      ompi_mpi_errcode_lastused))) {
         return ret;
@@ -161,6 +174,9 @@ int ompi_attr_free_predefined(void)
         OMPI_SUCCESS != (ret = free_comm(MPI_APPNUM)) ||
         OMPI_SUCCESS != (ret = free_comm(MPI_LASTUSEDCODE)) ||
         OMPI_SUCCESS != (ret = free_comm(MPI_UNIVERSE_SIZE)) ||
+#if OPAL_ENABLE_FT_MPI
+        OMPI_SUCCESS != (ret = free_comm(MPI_FT)) ||
+#endif /* OPAL_ENABLE_FT_MPI */
         OMPI_SUCCESS != (ret = free_win(MPI_WIN_BASE)) ||
         OMPI_SUCCESS != (ret = free_win(MPI_WIN_SIZE)) ||
         OMPI_SUCCESS != (ret = free_win(MPI_WIN_DISP_UNIT)) ||
 
@@ -3,7 +3,7 @@
 # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
+# Copyright (c) 2004-2020 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
 # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -34,3 +34,8 @@ lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
 	communicator/comm_cid.c \
 	communicator/comm_request.c
 
+if WANT_FT_MPI
+lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
+	communicator/ft/comm_ft.c communicator/ft/comm_ft_reliable_bcast.c communicator/ft/comm_ft_propagator.c communicator/ft/comm_ft_detector.c communicator/ft/comm_ft_revoke.c
+endif # WANT_FT_MPI
+
Original file line number	Diff line number	Diff line change
`@@ -1085,7 +1085,7 @@ AC_CACHE_SAVE`
`1085`	`1085`	`# visible again`
`1086`	`1086`	`#`
`1087`	`1087`	`###########################################################`
`1088`		`-dnl OPAL_SETUP_FT_OPTIONS`
	`1088`	`+OPAL_SETUP_FT_OPTIONS`
`1089`	`1089`	`###########################################################`
`1090`	`1090`	`# The following line is always required as it contains the`
`1091`	`1091`	`# AC_DEFINE and AM_CONDITIONAL calls that set variables used`