CICE-Consortium · eclare108213 · Oct 12, 2023 · Oct 12, 2023
diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh
@@ -168,6 +168,23 @@ cat >> ${jobfile} << EOFB
 ###SBATCH --mail-user username@domain.com
 EOFB
 
+else if (${ICE_MACHINE} =~ perlmutter*) then
+@ nthrds2 = ${nthrds} * 2
+cat >> ${jobfile} << EOFB
+#SBATCH -J ${ICE_CASENAME}
+#SBATCH -A ${acct}
+#SBATCH --qos=${queue}
+#SBATCH --time=${batchtime}
+#SBATCH --nodes=${nnodes}
+#SBATCH --ntasks=${ntasks}
+#SBATCH --cpus-per-task=${nthrds2}
+#SBATCH --constraint cpu
+###SBATCH -e filename
+###SBATCH -o filename
+###SBATCH --mail-type FAIL
+###SBATCH --mail-user username@domain.com
+EOFB
+
 else if (${ICE_MACHINE} =~ compy*) then
 if (${runlength} <= 2) set queue = "short"
 cat >> ${jobfile} <<EOFB

diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh
@@ -95,10 +95,11 @@ aprun -q -n ${ntasks} -N ${taskpernodelimit} -d ${nthrds} ./cice >&! \$ICE_RUNLO
 EOFR
 
 #=======
-else if (${ICE_MACHCOMP} =~ cori*) then
+else if (${ICE_MACHCOMP} =~ cori* || ${ICE_MACHCOMP} =~ perlmutter*) then
 if (${ICE_COMMDIR} =~ serial*) then
 cat >> ${jobfile} << EOFR
-./cice >&! \$ICE_RUNLOG_FILE
+#./cice >&! \$ICE_RUNLOG_FILE
+srun --cpu-bind=cores ./cice >&! \$ICE_RUNLOG_FILE
 EOFR
 else
 cat >> ${jobfile} << EOFR

diff --git a/configuration/scripts/machines/Macros.perlmutter_cray b/configuration/scripts/machines/Macros.perlmutter_cray
@@ -0,0 +1,56 @@
+#==============================================================================
+# Macros file for NERSC perlmutter, cray compiler
+#==============================================================================
+
+CPP        := ftn -e P
+CPPDEFS    := -DFORTRANUNDERSCORE -DNO_R16 ${ICE_CPPDEFS}
+CFLAGS     := -c -O2
+
+FIXEDFLAGS := -132
+FREEFLAGS  := 
+FFLAGS     := -hbyteswapio
+FFLAGS_NOOPT:= -O0
+LDFLAGS    := -hbyteswapio
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -hfp0 -g -Rbcdps -Ktrap=fp
+else
+  FFLAGS     += -O2 -hfp0     # -eo
+endif
+
+SCC   := cc
+SFC   := ftn
+MPICC := cc
+MPIFC := ftn
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD:= $(FC)
+
+# defined by module
+#NETCDF_PATH := $(NETCDF)
+#PNETCDF_PATH := $(PNETCDF)
+#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default
+#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib
+
+#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs 
+
+INCLDIR := $(INCLDIR)
+#INCLDIR += -I$(NETCDF_PATH)/include
+
+#LIB_NETCDF := $(NETCDF_PATH)/lib
+#LIB_PNETCDF := $(PNETCDF_PATH)/lib
+#LIB_MPI := $(IMPILIBDIR)
+#SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -fopenmp
+   CFLAGS += -fopenmp
+   FFLAGS += -fopenmp
+endif
+
diff --git a/configuration/scripts/machines/Macros.perlmutter_gnu b/configuration/scripts/machines/Macros.perlmutter_gnu
@@ -0,0 +1,57 @@
+#==============================================================================
+# Macros file for NERSC perlmutter, gnu compiler
+#==============================================================================
+
+CPP        := ftn -E
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c
+
+FIXEDFLAGS := -ffixed-line-length-132
+FREEFLAGS  := -ffree-form
+FFLAGS     := -fconvert=big-endian -fbacktrace -ffree-line-length-none -fallow-argument-mismatch
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -g -fcheck=bounds -finit-real=nan -fimplicit-none -ffpe-trap=invalid,zero,overflow
+  CFLAGS     += -O0
+else
+  FFLAGS   += -O2
+  CFLAGS   += -O2
+endif
+
+SCC   := cc
+SFC   := ftn
+MPICC := cc
+MPIFC := ftn
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD:= $(FC)
+
+# defined by module
+#NETCDF_PATH := $(NETCDF)
+#PNETCDF_PATH := $(PNETCDF)
+#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default
+#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib
+
+#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs 
+
+INCLDIR := $(INCLDIR)
+#INCLDIR += -I$(NETCDF_PATH)/include
+
+#LIB_NETCDF := $(NETCDF_PATH)/lib
+#LIB_PNETCDF := $(PNETCDF_PATH)/lib
+#LIB_MPI := $(IMPILIBDIR)
+#SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -fopenmp
+   CFLAGS += -fopenmp
+   FFLAGS += -fopenmp
+endif
+
diff --git a/configuration/scripts/machines/Macros.perlmutter_intel b/configuration/scripts/machines/Macros.perlmutter_intel
@@ -0,0 +1,57 @@
+#==============================================================================
+# Macros file for NERSC perlmutter, intel compiler
+#==============================================================================
+
+CPP        := fpp
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c -O2 -fp-model precise -march=core-avx2
+
+FIXEDFLAGS := -fixed -132
+FREEFLAGS  := -free
+FFLAGS     := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -march=core-avx2
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg
+#  FFLAGS     += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -link_mpi=dbg -stand f08
+#  FFLAGS     += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays -link_mpi=dbg
+else
+  FFLAGS     += -O2
+endif
+
+SCC   := cc
+SFC   := ftn
+MPICC := cc
+MPIFC := ftn
+
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD:= $(FC)
+
+# defined by module
+#NETCDF_PATH := $(NETCDF)
+#PNETCDF_PATH := $(PNETCDF)
+#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default
+#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib
+
+#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs 
+
+INCLDIR := $(INCLDIR)
+#INCLDIR += -I$(NETCDF_PATH)/include
+
+#LIB_NETCDF := $(NETCDF_PATH)/lib
+#LIB_PNETCDF := $(PNETCDF_PATH)/lib
+#LIB_MPI := $(IMPILIBDIR)
+#SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -qopenmp
+   CFLAGS += -qopenmp
+   FFLAGS += -qopenmp
+endif
+
diff --git a/configuration/scripts/machines/env.perlmutter_cray b/configuration/scripts/machines/env.perlmutter_cray
@@ -0,0 +1,51 @@
+#!/bin/csh -f
+
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+source ${MODULESHOME}/init/csh
+
+#module unload PrgEnv-aocc
+#module unload PrgEnv-cray
+#module unload PrgEnv-gnu
+#module unload PrgEnv-intel
+#module unload PrgEnv-nvidia
+#module unload gpu
+module load cpu
+module load PrgEnv-cray
+module unload cce
+module load cce/15.0.1
+module unload cray-mpich
+module load cray-mpich/8.1.25
+
+module unload cray-netcdf
+module unload cray-hdf5
+module load cray-hdf5/1.12.2.3
+module load cray-netcdf/4.9.0.3
+
+setenv NETCDF_PATH ${NETCDF_DIR}
+limit coredumpsize unlimited
+limit stacksize unlimited
+setenv OMP_STACKSIZE 128M
+setenv OMP_WAIT_POLICY PASSIVE
+
+endif
+
+setenv ICE_MACHINE_MACHNAME perlmutter
+setenv ICE_MACHINE_MACHINFO "HPE Cray EX AMD EPYC 7763 Milan, Slingshot-11 Interconnect"
+setenv ICE_MACHINE_ENVNAME cray
+setenv ICE_MACHINE_ENVINFO "Cray clang/Fortran 15.0.1, cray-mpich/8.1.25, netcdf/4.9.0.3"
+setenv ICE_MACHINE_MAKE gmake
+setenv ICE_MACHINE_WKDIR $SCRATCH/CICE_RUNS
+setenv ICE_MACHINE_INPUTDATA /global/cfs/cdirs/e3sm/tcraig/cice-consortium
+setenv ICE_MACHINE_BASELINE $SCRATCH/CICE_BASELINE
+setenv ICE_MACHINE_SUBMIT "sbatch"
+setenv ICE_MACHINE_ACCT P00000000
+setenv ICE_MACHINE_QUEUE "regular"
+setenv ICE_MACHINE_TPNODE 128
+setenv ICE_MACHINE_BLDTHRDS 8
+setenv ICE_MACHINE_QSTAT "squeue --jobs= "
diff --git a/configuration/scripts/machines/env.perlmutter_gnu b/configuration/scripts/machines/env.perlmutter_gnu
@@ -0,0 +1,51 @@
+#!/bin/csh -f
+
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+source ${MODULESHOME}/init/csh
+
+#module unload PrgEnv-aocc
+#module unload PrgEnv-cray
+#module unload PrgEnv-gnu
+#module unload PrgEnv-intel
+#module unload PrgEnv-nvidia
+#module unload gpu
+module load cpu
+module load PrgEnv-gnu
+module unload gcc
+module load gcc/11.2.0
+module unload cray-mpich
+module load cray-mpich/8.1.25
+
+module unload cray-netcdf
+module unload cray-hdf5
+module load cray-hdf5/1.12.2.3
+module load cray-netcdf/4.9.0.3
+
+setenv NETCDF_PATH ${NETCDF_DIR}
+limit coredumpsize unlimited
+limit stacksize unlimited
+setenv OMP_STACKSIZE 128M
+setenv OMP_WAIT_POLICY PASSIVE
+
+endif
+
+setenv ICE_MACHINE_MACHNAME perlmutter
+setenv ICE_MACHINE_MACHINFO "HPE Cray EX AMD EPYC 7763 Milan, Slingshot-11 Interconnect"
+setenv ICE_MACHINE_ENVNAME gnu
+setenv ICE_MACHINE_ENVINFO "gnu c/fortran 11.2.0 20210728, cray-mpich/8.1.25, netcdf/4.9.0.3"
+setenv ICE_MACHINE_MAKE gmake
+setenv ICE_MACHINE_WKDIR $SCRATCH/CICE_RUNS
+setenv ICE_MACHINE_INPUTDATA /global/cfs/cdirs/e3sm/tcraig/cice-consortium
+setenv ICE_MACHINE_BASELINE $SCRATCH/CICE_BASELINE
+setenv ICE_MACHINE_SUBMIT "sbatch"
+setenv ICE_MACHINE_ACCT P00000000
+setenv ICE_MACHINE_QUEUE "regular"
+setenv ICE_MACHINE_TPNODE 128
+setenv ICE_MACHINE_BLDTHRDS 8
+setenv ICE_MACHINE_QSTAT "squeue --jobs= "
diff --git a/configuration/scripts/machines/env.perlmutter_intel b/configuration/scripts/machines/env.perlmutter_intel
@@ -0,0 +1,51 @@
+#!/bin/csh -f
+
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+source ${MODULESHOME}/init/csh
+
+#module unload PrgEnv-aocc
+#module unload PrgEnv-cray
+#module unload PrgEnv-gnu
+#module unload PrgEnv-intel
+#module unload PrgEnv-nvidia
+#module unload gpu
+module load cpu
+module load PrgEnv-intel
+module unload intel
+module load intel/2023.1.0
+module unload cray-mpich
+module load cray-mpich/8.1.25
+
+module unload cray-netcdf
+module unload cray-hdf5
+module load cray-hdf5/1.12.2.3
+module load cray-netcdf/4.9.0.3
+
+setenv NETCDF_PATH ${NETCDF_DIR}
+limit coredumpsize unlimited
+limit stacksize unlimited
+setenv OMP_STACKSIZE 128M
+setenv OMP_WAIT_POLICY PASSIVE
+
+endif
+
+setenv ICE_MACHINE_MACHNAME perlmutter
+setenv ICE_MACHINE_MACHINFO "HPE Cray EX AMD EPYC 7763 Milan, Slingshot-11 Interconnect"
+setenv ICE_MACHINE_ENVNAME intel
+setenv ICE_MACHINE_ENVINFO "ifort 2021.9.0 20230302, Intel oneAPI DPC++/C++ 2023.1.0 (2023.1.0.20230320), cray-mpich/8.1.25, netcdf/4.9.0.3"
+setenv ICE_MACHINE_MAKE gmake
+setenv ICE_MACHINE_WKDIR $SCRATCH/CICE_RUNS
+setenv ICE_MACHINE_INPUTDATA /global/cfs/cdirs/e3sm/tcraig/cice-consortium
+setenv ICE_MACHINE_BASELINE $SCRATCH/CICE_BASELINE
+setenv ICE_MACHINE_SUBMIT "sbatch"
+setenv ICE_MACHINE_ACCT P00000000
+setenv ICE_MACHINE_QUEUE "regular"
+setenv ICE_MACHINE_TPNODE 128
+setenv ICE_MACHINE_BLDTHRDS 8
+setenv ICE_MACHINE_QSTAT "squeue --jobs= "