Port to gust intel and cray (CICE-Consortium#781)

cicecore/cicedynB/dynamics/ice_transport_remap.F90

@@ -633,6 +633,7 @@ subroutine horizontal_remap (dt,             ntrace,   &
 
       endif  ! nghost
 
+      ! tcraig, this OMP loop sometimes fails with cce/14.0.3, compiler bug??
       !$OMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,this_block,n,  &
       !$OMP                     edgearea_e,edgearea_n,edge,iflux,jflux, &
       !$OMP                     xp,yp,indxing,indxjng,mflxe,mflxn, &

configuration/scripts/cice.batch.csh

@@ -33,6 +33,19 @@ cat >> ${jobfile} << EOFB
 #PBS -l walltime=${batchtime}
 EOFB
 
+else if (${ICE_MACHINE} =~ gust*) then
+cat >> ${jobfile} << EOFB
+#PBS -q ${queue}
+#PBS -l job_priority=regular
+#PBS -N ${ICE_CASENAME}
+#PBS -A ${acct}
+#PBS -l select=${nnodes}:ncpus=${corespernode}:mpiprocs=${taskpernodelimit}:ompthreads=${nthrds}
+#PBS -l walltime=${batchtime}
+#PBS -j oe 
+#PBS -W umask=022
+#PBS -o ${ICE_CASEDIR}
+EOFB
+
 else if (${ICE_MACHINE} =~ hobart*) then
 cat >> ${jobfile} << EOFB
 #PBS -j oe 

configuration/scripts/cice.launch.csh

@@ -22,6 +22,18 @@ mpiexec_mpt -np ${ntasks} omplace ./cice >&! \$ICE_RUNLOG_FILE
 EOFR
 endif
 
+#=======
+else if (${ICE_MACHCOMP} =~ gust*) then
+if (${ICE_COMMDIR} =~ serial*) then
+cat >> ${jobfile} << EOFR
+./cice >&! \$ICE_RUNLOG_FILE
+EOFR
+else
+cat >> ${jobfile} << EOFR
+mpiexec --cpu-bind depth -n ${ntasks} -ppn ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLOG_FILE
+EOFR
+endif
+
 #=======
 else if (${ICE_MACHCOMP} =~ hobart* || ${ICE_MACHCOMP} =~ izumi*) then
 if (${ICE_COMMDIR} =~ serial*) then

configuration/scripts/machines/Macros.gust_cray

@@ -0,0 +1,71 @@
+#==============================================================================
+# Makefile macros for NCAR cheyenne, intel compiler
+#==============================================================================
+
+CPP        := ftn -e P
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c -O2
+
+FIXEDFLAGS := -132
+FREEFLAGS  :=
+FFLAGS     := -hbyteswapio
+FFLAGS_NOOPT:= -O0
+LDFLAGS    := -hbyteswapio
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -hfp0 -g -Rbcdps -Ktrap=fp
+else
+  FFLAGS     += -O2 -hfp0
+endif
+
+SCC   := cc 
+SFC   := ftn
+MPICC := cc
+MPIFC := ftn
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD:= $(FC)
+
+NETCDF_PATH := $(NETCDF)
+
+PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs 
+
+PNETCDF_PATH := $(PNETCDF)
+#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib
+
+INCLDIR := $(INCLDIR)
+
+LIB_NETCDF := $(NETCDF)/lib
+#LIB_PNETCDF := $(PNETCDF_PATH)/lib
+#LIB_MPI := $(IMPILIBDIR)
+
+#SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -L$(LIB_PNETCDF) -lpnetcdf -lgptl
+SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -fopenmp 
+   CFLAGS += -fopenmp 
+   FFLAGS += -fopenmp 
+else
+   LDFLAGS += -hnoomp
+#   CFLAGS += -hnoomp
+   FFLAGS += -hnoomp
+endif
+
+#ifeq ($(ICE_IOTYPE), pio1)
+#   LIB_PIO := $(PIO_LIBDIR)
+#   SLIBS   := $(SLIBS) -L$(LIB_PIO) -lpio
+#endif
+
+ifeq ($(ICE_IOTYPE), pio2)
+   CPPDEFS := $(CPPDEFS) -DGPTL
+#   LIB_PIO := $(PIO_LIBDIR)
+#   SLIBS   := $(SLIBS) -L$(LIB_PIO) -lpiof -lpioc -lgptl
+endif
+

configuration/scripts/machines/Macros.gust_intel

@@ -0,0 +1,68 @@
+#==============================================================================
+# Makefile macros for NCAR cheyenne, intel compiler
+#==============================================================================
+
+CPP        := fpp
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c -O2 -fp-model precise
+
+FIXEDFLAGS := -132
+FREEFLAGS  := -FR
+FFLAGS     := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg
+#  FFLAGS     += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -link_mpi=dbg -stand f08
+#  FFLAGS     += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays -link_mpi=dbg
+else
+  FFLAGS     += -O2
+endif
+
+SCC   := cc 
+SFC   := ftn
+MPICC := cc
+MPIFC := ftn
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD:= $(FC)
+
+NETCDF_PATH := $(NETCDF)
+
+PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs 
+
+PNETCDF_PATH := $(PNETCDF)
+#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib
+
+INCLDIR := $(INCLDIR)
+
+LIB_NETCDF := $(NETCDF)/lib
+#LIB_PNETCDF := $(PNETCDF_PATH)/lib
+#LIB_MPI := $(IMPILIBDIR)
+
+#SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -L$(LIB_PNETCDF) -lpnetcdf -lgptl
+SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -qopenmp 
+   CFLAGS += -qopenmp 
+   FFLAGS += -qopenmp 
+endif
+
+#ifeq ($(ICE_IOTYPE), pio1)
+#   LIB_PIO := $(PIO_LIBDIR)
+#   SLIBS   := $(SLIBS) -L$(LIB_PIO) -lpio
+#endif
+
+ifeq ($(ICE_IOTYPE), pio2)
+   CPPDEFS := $(CPPDEFS) -DGPTL
+#   LIB_PIO := $(PIO_LIBDIR)
+#   SLIBS   := $(SLIBS) -L$(LIB_PIO) -lpiof -lpioc -lgptl
+endif
+

configuration/scripts/machines/env.gust_cray

@@ -0,0 +1,72 @@
+#!/bin/csh -f
+
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+source ${MODULESHOME}/init/csh
+
+module --force purge
+module load ncarenv/22.10
+module load craype/2.7.17
+module load cce/14.0.3
+module load ncarcompilers/0.7.1
+module load cray-mpich/8.1.19
+module load hdf5/1.12.2
+module load netcdf/4.8.1
+module load cray-libsci/22.08.1.1
+
+if ($?ICE_IOTYPE) then
+if ($ICE_IOTYPE =~ pio*) then
+  module unload netcdf
+  module load netcdf-mpi/4.8.1
+  module load pnetcdf/1.12.2
+  if ($ICE_IOTYPE == "pio1") then
+    module load pio/1.10.1
+  else
+    module load pio/2.5.9
+  endif
+endif
+endif
+
+if ($?ICE_BFBTYPE) then
+if ($ICE_BFBTYPE =~ qcchk*) then
+  module load conda
+# conda env create -f ../../configuration/scripts/tests/qctest.yml
+  conda activate qctest
+endif
+endif
+
+# For perftools with mpiexec
+# module load perftools-base
+# module load perftools
+#setenv PALS_TRANSFER FALSE
+
+endif
+
+limit coredumpsize unlimited
+limit stacksize unlimited
+setenv PALS_QUIET TRUE
+
+# May be needed for OpenMP memory
+setenv OMP_STACKSIZE 64M
+# OMP runtime diagnostics
+#setenv OMP_DISPLAY_ENV TRUE
+
+setenv ICE_MACHINE_MACHNAME gust
+setenv ICE_MACHINE_MACHINFO "Cray XE Milan Slingshot 11"
+setenv ICE_MACHINE_ENVNAME crayg
+setenv ICE_MACHINE_ENVINFO "cce 14.0.3, cray-mpich 2.19, netcdf4.8.1, pnetcdf1.12.2, pio2.5.9"
+setenv ICE_MACHINE_MAKE gmake
+setenv ICE_MACHINE_WKDIR /glade/gust/scratch/$user/CICE_RUNS
+setenv ICE_MACHINE_INPUTDATA /glade/work/tcraig/cice-consortium-data
+setenv ICE_MACHINE_BASELINE /glade/gust/scratch/$user/CICE_BASELINE
+setenv ICE_MACHINE_SUBMIT "qsub"
+setenv ICE_MACHINE_ACCT P00000000
+setenv ICE_MACHINE_QUEUE "main"
+setenv ICE_MACHINE_TPNODE 128
+setenv ICE_MACHINE_BLDTHRDS 8
+setenv ICE_MACHINE_QSTAT "qstat "

configuration/scripts/machines/env.gust_intel

@@ -0,0 +1,72 @@
+#!/bin/csh -f
+
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+source ${MODULESHOME}/init/csh
+
+module --force purge
+module load ncarenv/22.10
+module load craype/2.7.17
+module load intel/2021.6.0
+module load ncarcompilers/0.7.1
+module load cray-mpich/8.1.19
+module load hdf5/1.12.2
+module load netcdf/4.8.1
+module load cray-libsci/22.08.1.1
+
+if ($?ICE_IOTYPE) then
+if ($ICE_IOTYPE =~ pio*) then
+  module unload netcdf
+  module load netcdf-mpi/4.8.1
+  module load pnetcdf/1.12.2
+  if ($ICE_IOTYPE == "pio1") then
+    module load pio/1.10.1
+  else
+    module load pio/2.5.9
+  endif
+endif
+endif
+
+if ($?ICE_BFBTYPE) then
+if ($ICE_BFBTYPE =~ qcchk*) then
+  module load conda
+# conda env create -f ../../configuration/scripts/tests/qctest.yml
+  conda activate qctest
+endif
+endif
+
+# For perftools with mpiexec
+# module load perftools-base
+# module load perftools
+#setenv PALS_TRANSFER FALSE
+
+endif
+
+limit coredumpsize unlimited
+limit stacksize unlimited
+setenv PALS_QUIET TRUE
+
+# May be needed for OpenMP memory
+setenv OMP_STACKSIZE 64M
+# OMP runtime diagnostics
+#setenv OMP_DISPLAY_ENV TRUE
+
+setenv ICE_MACHINE_MACHNAME gust
+setenv ICE_MACHINE_MACHINFO "Cray XE Milan Slingshot 11"
+setenv ICE_MACHINE_ENVNAME intel
+setenv ICE_MACHINE_ENVINFO "ifort 2021.6.0 20220226, cray-mpich 2.19, netcdf4.8.1, pnetcdf1.12.2, pio2.5.9"
+setenv ICE_MACHINE_MAKE gmake
+setenv ICE_MACHINE_WKDIR /glade/gust/scratch/$user/CICE_RUNS
+setenv ICE_MACHINE_INPUTDATA /glade/work/tcraig/cice-consortium-data
+setenv ICE_MACHINE_BASELINE /glade/gust/scratch/$user/CICE_BASELINE
+setenv ICE_MACHINE_SUBMIT "qsub"
+setenv ICE_MACHINE_ACCT P00000000
+setenv ICE_MACHINE_QUEUE "main"
+setenv ICE_MACHINE_TPNODE 128
+setenv ICE_MACHINE_BLDTHRDS 8
+setenv ICE_MACHINE_QSTAT "qstat "