Remove dynamic CCPP build (#103)

* Remove CCPP dynamic build from framework
* Bug fix in ccpp/driver/CCPP_driver.F90

CMakeLists.txt

@@ -8,9 +8,6 @@ endif()
 
 if(CCPP)
   add_definitions(-DCCPP)
-  if(STATIC)
-    add_definitions(-DSTATIC)
-  endif()
 endif()
 
 add_subdirectory(cpl)

ccpp/CMakeLists.txt

@@ -171,27 +171,13 @@ if (MPI)
   ADD_DEFINITIONS(-DMPI)
 endif (MPI)
 
-#------------------------------------------------------------------------------
-# Set flags for static building/linking of CCPP
-if (STATIC)
-  ADD_DEFINITIONS(-DSTATIC)
-endif (STATIC)
-
 #------------------------------------------------------------------------------
 # Set flag for 32bit dynamics build
 if (DYN32)
   message(STATUS "Dynamics compiled with 32-bit option, adjust fv_sat_adj types")
   ADD_DEFINITIONS(-DOVERLOAD_R4)
 endif (DYN32)
 
-#------------------------------------------------------------------------------
-# If a dynamic build is requested, set TARGET_SUPPORTS_SHARED_LIBS to TRUE; on
-# some systems, cmake doesn't set this even though the system supports shared
-# libraries (e.g. Cray systems)
-if(NOT STATIC)
-SET_PROPERTY(GLOBAL PROPERTY TARGET_SUPPORTS_SHARED_LIBS TRUE)
-endif(NOT STATIC)
-
 #------------------------------------------------------------------------------
 # Add model-specific flags for C/C++/Fortran preprocessor
 ADD_DEFINITIONS(-DMOIST_CAPPA -DUSE_COND -DNEMS_GSM)

ccpp/build_ccpp.sh

@@ -25,7 +25,6 @@ function usage   {
   echo "                                   REPRO=Y/N       (default N)"
   echo "                                   OPENMP=Y/N      (default Y)"
   echo "                                   32BIT=Y/N       (default N, affects dynamics/fast physics only)"
-  echo "                                   STATIC=Y/N      (default N, STATIC=Y requires SUITES=...)"
   echo "                                   SUITES=ABC,XYZ  (comma-separated list of CCPP suites; "
   echo "                                                    corresponding filenames: suite_ABC.xml. ...)"
   echo "                                   MULTI_GASES=Y/N (default N)"
@@ -125,18 +124,6 @@ if [[ "${MAKE_OPT}" == *"32BIT=Y"* ]]; then
 else
   CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DDYN32=OFF"
 fi
-if [[ "${MAKE_OPT}" == *"STATIC=Y"* ]]; then
-  CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DSTATIC=ON"
-else
-  # hera.gnu uses the NCEPLIBS-external/NCEPLIBS umbrella build libraries,
-  # which cannot be linked dynamically at this point (missing -fPIC flag)
-  if [[ "${MACHINE_ID}" == "hera.gnu" ]]; then
-    echo "Dynamic CCPP build not supported on hera.gnu at this time."
-    exit 1
-  fi
-  # Dynamic builds require linking the NCEPlibs, provide path to them
-  CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DSTATIC=OFF -DBACIO_LIB4=${BACIO_LIB4} -DSP_LIBd=${SP_LIBd} -DW3NCO_LIBd=${W3NCO_LIBd}"
-fi
 if [[ "${MAKE_OPT}" == *"MULTI_GASES=Y"* ]]; then
   CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DMULTI_GASES=ON"
 else
@@ -170,20 +157,6 @@ elif [[ "${MACHINE_ID}" == "gaea.intel" || "${MACHINE_ID}" == "wcoss_cray" ]]; t
   if [[ "${MACHINE_ID}" == "gaea.intel" ]]; then
     CCPP_CMAKE_FLAGS="${CCPP_CMAKE_FLAGS} -DLIBXML2_LIB_DIR=${LIBXML2_LIB_DIR} -DLIBXML2_INCLUDE_DIR=${LIBXML2_INCLUDE_DIR}"
   fi
-  # DH* At this time, it is not possible to use the dynamic CCPP
-  # build on gaea/wcoss_cray. While compiling/linking works, the model
-  # crashes immediately. This may be related to 64bit/32bit mismatches
-  # in the MPI libraries (missing "-fPIC" flags when the MPI libraries
-  # were compiled on the system?) - to be investigated.
-  if [[ "${MAKE_OPT}" == *"STATIC=Y"* ]]; then
-    :
-  else
-    ## FOR DYNAMIC BUILD, SET ENVIRONMENT VARIABLE CRAYPE_LINK_TYPE
-    #export CRAYPE_LINK_TYPE=dynamic
-    echo "Dynamic CCPP build not supported on gaea/wcoss_cray at this time."
-    exit 1
-  fi
-  # *DH
 fi
 
 CCPP_CMAKE_FLAGS=$(trim "${CCPP_CMAKE_FLAGS}")
@@ -212,19 +185,10 @@ make ${CCPP_MAKE_FLAGS} install
 # Generate ESMF makefile fragment
 
 set -u
-if ( echo "${MAKE_OPT}" | grep STATIC=Y ) ; then
-  # Set linker flags for static build
-  CCPP_LINK_OBJS="-L${PATH_CCPP_LIB} -lccpp -lccppphys"
-else
-  # Explicitly append libxml2, with or without path
-  CCPP_XML2_LIB="${LIBXML2_LIB_DIR:+-L${LIBXML2_LIB_DIR} }-lxml2"
-  # Set link objects
-  if ( echo "$MACHINE_ID" | grep gaea ) ; then
-    CCPP_LINK_OBJS="-dynamic -L${PATH_CCPP_LIB} -lccpp -lccppphys ${CCPP_XML2_LIB} ${CRAY_PMI_POST_LINK_OPTS} -lpmi"
-  else
-    CCPP_LINK_OBJS="-L${PATH_CCPP_LIB} -lccpp -lccppphys ${CCPP_XML2_LIB}"
-  fi
-fi
+
+# Set linker flags
+CCPP_LINK_OBJS="-L${PATH_CCPP_LIB} -lccpp -lccppphys"
+
 echo "ESMF_DEP_INCPATH=${PATH_CCPP_INC} ${PATH_CCPP_BUILD}/physics" > ${CCPP_MK}
 echo "ESMF_DEP_LINK_OBJS=${CCPP_LINK_OBJS}" >> ${CCPP_MK}
 

ccpp/driver/CCPP_driver.F90

@@ -1,41 +1,17 @@
 module CCPP_driver
 
-#ifdef STATIC
-! For static builds, the ccpp_physics_{init,run,finalize} calls
-! are not pointing to code in the CCPP framework, but to auto-generated
-! ccpp_suite_cap and ccpp_group_*_cap modules behind a ccpp_static_api
-  use ccpp_api,           only: ccpp_t,                              &
-                                ccpp_init,                           &
-                                ccpp_finalize,                       &
-                                ccpp_initialized
+  use ccpp_api,           only: ccpp_t
+
   use ccpp_static_api,    only: ccpp_physics_init,                   &
                                 ccpp_physics_run,                    &
                                 ccpp_physics_finalize
-#else
-  use ccpp_api,           only: ccpp_t,                              &
-                                ccpp_init,                           &
-                                ccpp_finalize,                       &
-                                ccpp_physics_init,                   &
-                                ccpp_physics_run,                    &
-                                ccpp_physics_finalize,               &
-                                ccpp_field_add,                      &
-                                ccpp_initialized,                    &
-                                ccpp_error
-#endif
 
   use CCPP_data,          only: cdata_tile,                          &
                                 cdata_domain,                        &
                                 cdata_block,                         &
                                 ccpp_suite,                          &
                                 GFS_control
 
-#ifndef STATIC
-! Begin include auto-generated list of modules for ccpp
-#include "ccpp_modules_slow_physics.inc"
-! End include auto-generated list of modules for ccpp
-  use iso_c_binding,      only: c_loc
-#endif
-
   implicit none
 
 !--------------------------------------------------------!
@@ -100,69 +76,20 @@ subroutine CCPP_step (step, nblks, ierr)
         nthrdsX = nthrds
       end if
 
-      !--- Initialize CCPP framework, if cdata_tile for fast physics
-      !    is already initialized, use its suite to avoid reading the
-      !    SDF multiple times. Initialize cdata for the entire domain:
-      if (ccpp_initialized(cdata_tile)) then
-        call ccpp_init(trim(ccpp_suite), cdata_domain, ierr=ierr, cdata_target=cdata_tile)
-      else
-        call ccpp_init(trim(ccpp_suite), cdata_domain, ierr=ierr)
-      end if
-      if (ierr/=0) then
-        write(0,*) 'An error occurred in ccpp_init'
-        return
-      end if
-
       ! For physics running over the entire domain, block and thread
       ! number are not used; set to safe values
       cdata_domain%blk_no = 1
       cdata_domain%thrd_no = 1
 
-#ifndef STATIC
-      ! Associate cdata with the global/domain cdata structure;
-      ! this is needed because ccpp_fields.inc uses 'cdata' in
-      ! its ccpp_field_add statements.
-      associate_domain: associate (cdata => cdata_domain)
-! Begin include auto-generated list of calls to ccpp_field_add
-#include "ccpp_fields_slow_physics.inc"
-! End include auto-generated list of calls to ccpp_field_add
-      end associate associate_domain
-#endif
-
       ! Allocate cdata structures for blocks and threads
-      allocate(cdata_block(1:nblks,1:nthrdsX))
+      if (.not.allocated(cdata_block)) allocate(cdata_block(1:nblks,1:nthrdsX))
 
       ! Loop over all blocks and threads
       do nt=1,nthrdsX
         do nb=1,nblks
-          !--- Initialize CCPP framework for blocks/threads, use suite from scalar cdata
-          !    to avoid reading the SDF multiple times. If cdata_tile is initialized, use
-          !    this version (since cdata_domain is just a copy), otherwise use cdata_domain
-          if (ccpp_initialized(cdata_tile)) then
-            call ccpp_init(trim(ccpp_suite), cdata_block(nb,nt), ierr=ierr, cdata_target=cdata_tile)
-          else
-            call ccpp_init(trim(ccpp_suite), cdata_block(nb,nt), ierr=ierr, cdata_target=cdata_domain)
-          end if
-          if (ierr/=0) then
-            write(0,'(2(a,i4))') "An error occurred in ccpp_init for block ", nb, " and thread ", nt
-            return
-          end if
-
           ! Assign the correct block and thread numbers
           cdata_block(nb,nt)%blk_no = nb
           cdata_block(nb,nt)%thrd_no = nt
-
-#ifndef STATIC
-          ! Associate cdata with the cdata structure for this block
-          ! and thread; this is needed because ccpp_fields.inc uses
-          ! 'cdata' in its ccpp_field_add statements.
-          associate_block: associate (cdata => cdata_block(nb,nt))
-! Begin include auto-generated list of calls to ccpp_field_add
-#include "ccpp_fields_slow_physics.inc"
-! End include auto-generated list of calls to ccpp_field_add
-          end associate associate_block
-#endif
-
         end do
       end do
 
@@ -173,11 +100,7 @@ subroutine CCPP_step (step, nblks, ierr)
       ! we can allow threading inside the time_vary routines.
       GFS_control%nthreads = nthrds
 
-#ifdef STATIC
       call ccpp_physics_init(cdata_domain, suite_name=trim(ccpp_suite), ierr=ierr)
-#else
-      call ccpp_physics_init(cdata_domain, ierr=ierr)
-#endif
       if (ierr/=0) then
         write(0,'(a)') "An error occurred in ccpp_physics_init"
         write(0,'(a)') trim(cdata_domain%errmsg)
@@ -193,11 +116,7 @@ subroutine CCPP_step (step, nblks, ierr)
       ! inside the time_vary routines.
       GFS_control%nthreads = nthrds
 
-#ifdef STATIC
       call ccpp_physics_run(cdata_domain, suite_name=trim(ccpp_suite), group_name="time_vary", ierr=ierr)
-#else
-      call ccpp_physics_run(cdata_domain, group_name="time_vary", ierr=ierr)
-#endif
       if (ierr/=0) then
         write(0,'(a)') "An error occurred in ccpp_physics_run for group time_vary"
         write(0,'(a)') trim(cdata_domain%errmsg)
@@ -230,11 +149,7 @@ subroutine CCPP_step (step, nblks, ierr)
             ntX = nt
         end if
         !--- Call CCPP radiation/physics/stochastics group
-#ifdef STATIC
         call ccpp_physics_run(cdata_block(nb,ntX), suite_name=trim(ccpp_suite), group_name=trim(step), ierr=ierr2)
-#else
-        call ccpp_physics_run(cdata_block(nb,ntX), group_name=trim(step), ierr=ierr2)
-#endif
         if (ierr2/=0) then
            write(0,'(2a,3(a,i4),a)') "An error occurred in ccpp_physics_run for group ", trim(step), &
                                      ", block ", nb, " and thread ", nt, " (ntX=", ntX, ")"
@@ -259,43 +174,17 @@ subroutine CCPP_step (step, nblks, ierr)
       do nt=1,nthrdsX
         do nb=1,nblks
           !--- Finalize CCPP physics
-#ifdef STATIC
           call ccpp_physics_finalize(cdata_block(nb,nt), suite_name=trim(ccpp_suite), ierr=ierr)
-#else
-          call ccpp_physics_finalize(cdata_block(nb,nt), ierr=ierr)
-#endif
           if (ierr/=0) then
             write(0,'(a,i4,a,i4)') "An error occurred in ccpp_physics_finalize for block ", nb, " and thread ", nt
             write(0,'(a)') trim(cdata_block(nb,nt)%errmsg)
             return
           end if
-          !--- Finalize CCPP framework for blocks/threads
-          call ccpp_finalize(cdata_block(nb,nt), ierr=ierr)
-          if (ierr/=0) then
-            write(0,'(a,i4,a,i4)') "An error occurred in ccpp_finalize for block ", nb, " and thread ", nt
-            return
-          end if
         end do
       end do
 
       ! Deallocate cdata structure for blocks and threads
-      deallocate(cdata_block)
-
-      !--- Finalize CCPP framework for domain
-      call ccpp_finalize(cdata_domain, ierr=ierr)
-      if (ierr/=0) then
-         write(0,'(a)') "An error occurred in ccpp_finalize"
-         return
-      end if
-
-      !--- Finalize CCPP framework for fast physics last (all other frameworks point to cdata_tile's suite)
-      if (ccpp_initialized(cdata_tile)) then
-         call ccpp_finalize(cdata_tile, ierr)
-         if (ierr/=0) then
-            write(0,'(a)') "An error occurred in ccpp_finalize"
-            return
-         end if
-      end if
+      if (allocated(cdata_block)) deallocate(cdata_block)
 
     else
 

ccpp/suites/suite_FV3_GFS_rasmgshoc.xml

@@ -56,8 +56,8 @@
       <scheme>moninshoc</scheme>
       <scheme>GFS_PBL_generic_post</scheme>
       <scheme>GFS_GWD_generic_pre</scheme>
-      <scheme>gwdps</scheme>
-      <scheme>gwdps_post</scheme>
+      <scheme>cires_ugwp</scheme>
+      <scheme>cires_ugwp_post</scheme>
       <scheme>GFS_GWD_generic_post</scheme>
       <scheme>rayleigh_damp</scheme>
       <scheme>GFS_suite_stateout_update</scheme>

ccpp/suites/suite_FV3_GFS_v15_ras.xml

@@ -75,7 +75,6 @@
       <scheme>GFS_DCNV_generic_post</scheme>
       <scheme>GFS_SCNV_generic_pre</scheme>
       <scheme>samfshalcnv</scheme>
-      <scheme>samfshalcnv_post</scheme>
       <scheme>GFS_SCNV_generic_post</scheme>
       <scheme>GFS_suite_interstitial_4</scheme>
       <scheme>cnvc90</scheme>

ccpp/suites/suite_FV3_GFS_v15plusras.xml

@@ -75,7 +75,6 @@
       <scheme>GFS_DCNV_generic_post</scheme>
       <scheme>GFS_SCNV_generic_pre</scheme>
       <scheme>samfshalcnv</scheme>
-      <scheme>samfshalcnv_post</scheme>
       <scheme>GFS_SCNV_generic_post</scheme>
       <scheme>GFS_suite_interstitial_4</scheme>
       <scheme>cnvc90</scheme>

gfsphysics/makefile

@@ -21,18 +21,14 @@ FFLAGS   += -I$(FMS_DIR) -I../cpl
 
 CPPDEFS += -DNEW_TAUCTMAX -DSMALL_PE -DNEMS_GSM -DINTERNAL_FILE_NML
 
-# CCPP dynamic and static builds
+# CCPP build
 ifneq (,$(findstring CCPP,$(CPPDEFS)))
 # Set flags for 32-bit dynamics build
 ifeq ($(DYN32),Y)
 CPPDEFS += -DOVERLOAD_R4
 endif
-# Set CCPP static api for static build
-ifneq (,$(findstring STATIC,$(CPPDEFS)))
-    CCPP_STATIC_API = ../ccpp/physics/ccpp_static_api.F90
-else
-    CCPP_STATIC_API = 
-endif
+CCPP_STATIC_API = ../ccpp/physics/ccpp_static_api.F90
+
 # Set physics source files
 # DH* 20190916 - can remove namelist_soilveg and set_soilveg
 # once calculation of snocvr is moved from FV3GFS_io.F90/GFS_driver.F90