Merge branch 'master' into 2027_effective_shape_bug

.github/workflows/compilation.yml

@@ -51,14 +51,14 @@ on:
   push
 
 env:
-  CUDA_VERSION: 12.6.0
+  CUDA_VERSION: 12.6.2
   GFORTRAN_VERSION: 14.2.0
-  HDF5_VERSION: 1.14.4.3
+  HDF5_VERSION: 1.14.5
   NETCDF_C_VERSION: 4.9.2
   NETCDF_FORTRAN_VERSION: 4.6.1
-  NVFORTRAN_VERSION: 24.7
+  NVFORTRAN_VERSION: 24.9
   OPENMPI_VERSION: 5.0.5
-  PYTHON_VERSION: 3.12.5
+  PYTHON_VERSION: 3.13.0
 
 jobs:
   run_if_on_mirror:

.github/workflows/lfric_test.yml

@@ -47,7 +47,7 @@ jobs:
     runs-on: self-hosted
     env:
       LFRIC_APPS_REV: 3269
-      PYTHON_VERSION: 3.12.5
+      PYTHON_VERSION: 3.13.0
 
     steps:
     - uses: actions/checkout@v3
@@ -75,6 +75,58 @@ jobs:
         pip install .[test]
         pip install jinja2
 
+    # PSyclone, compile and run MetOffice gungho_model on GPU
+    - name: LFRic GungHo with OpenMP offload
+      run: |
+        # Set up environment
+        source /apps/spack/psyclone-spack/spack-repo/share/spack/setup-env.sh
+        spack load lfric-build-environment%nvhpc
+        source .runner_venv/bin/activate
+        export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts
+        export PSYCLONE_CONFIG_FILE=${PSYCLONE_LFRIC_DIR}/KGOs/lfric_psyclone.cfg
+        # The LFRic source must be patched to workaround bugs in the NVIDIA
+        # compiler's namelist handling.
+        rm -rf ${HOME}/LFRic/gpu_build
+        mkdir -p ${HOME}/LFRic/gpu_build
+        cp -r ${HOME}/LFRic/lfric_apps_${LFRIC_APPS_REV} ${HOME}/LFRic/gpu_build/lfric_apps
+        cp -r ${HOME}/LFRic/lfric_core_50869 ${HOME}/LFRic/gpu_build/lfric
+        cd ${HOME}/LFRic/gpu_build
+        patch -p1 < ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_${LFRIC_APPS_REV}_nvidia.patch
+        # Update the compiler definitions to build for GPU
+        cp ${PSYCLONE_LFRIC_DIR}/KGOs/nvfortran_acc.mk lfric/infrastructure/build/fortran/nvfortran.mk
+        cp ${PSYCLONE_LFRIC_DIR}/KGOs/nvc++.mk lfric/infrastructure/build/cxx/.
+        # Update the PSyclone commands to ensure transformed kernels are written
+        # to working directory.
+        cp ${PSYCLONE_LFRIC_DIR}/KGOs/psyclone.mk lfric/infrastructure/build/psyclone/.
+        # Update dependencies.sh to point to our patched lfric core.
+        sed -i -e 's/export lfric_core_sources=.*$/export lfric_core_sources\=\/home\/gh_runner\/LFRic\/gpu_build\/lfric/' lfric_apps/dependencies.sh
+        export LFRIC_DIR=${HOME}/LFRic/gpu_build/lfric_apps
+        export OPT_DIR=${LFRIC_DIR}/applications/gungho_model/optimisation/psyclone-test
+        cd ${LFRIC_DIR}
+        # PSyclone scripts must now be under 'optimisation' and be called 'global.py'
+        mkdir -p ${OPT_DIR}
+        cp ${PSYCLONE_LFRIC_DIR}/gpu_offloading.py ${OPT_DIR}/global.py
+        # Clean previous version and compile again
+        rm -rf applications/gungho_model/working
+        LFRIC_OFFLOAD_DIRECTIVES=omp ./build/local_build.py -a gungho_model -p psyclone-test
+        cd applications/gungho_model/example
+        cp ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gungho_configuration_4its.nml configuration.nml
+        mpirun -n 1 ../bin/gungho_model configuration.nml |& tee output.txt
+        python ${PSYCLONE_LFRIC_DIR}/compare_ouput.py ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gungho_configuration_4its_checksums.txt gungho_model-checksums.txt
+        cat timer.txt
+        export VAR_TIME=$(grep "gungho_model" timer.txt | cut -d'|' -f5)
+        export VAR_HALOS=$(grep "gungho_model" halo_calls_counter.txt | cut -d'|' -f5)
+        echo $GITHUB_REF_NAME $GITHUB_SHA $VAR_TIME $VAR_HALOS >> ${HOME}/store_results/lfric_omp_performance_history
+        ${HOME}/mongosh-2.1.1-linux-x64/bin/mongosh \
+          "mongodb+srv://cluster0.x8ncpxi.mongodb.net/PerformanceMonitoring" \
+          --quiet --apiVersion 1 --username ${{ secrets.MONGODB_USERNAME }} \
+          --password ${{ secrets.MONGODB_PASSWORD }} \
+          --eval 'db.GitHub_CI.insertOne({branch_name: "'"$GITHUB_REF_NAME"'", commit: "'"$GITHUB_SHA"'",
+          github_job: "'"$GITHUB_RUN_ID"'"-"'"$GITHUB_RUN_ATTEMPT"'",
+          ci_test: "LFRic OpenMP offloading", lfric_apps_version: '"$LFRIC_APPS_REV"', system: "GlaDos",
+          compiler:"spack-nvhpc-24.5", date: new Date(), elapsed_time: '"$VAR_TIME"',
+          num_of_halo_exchanges: '"$VAR_HALOS"'})'
+
     # PSyclone, compile and run MetOffice gungho_model on GPU
     - name: LFRic GungHo with OpenACC offload
       run: |
@@ -105,10 +157,10 @@ jobs:
         cd ${LFRIC_DIR}
         # PSyclone scripts must now be under 'optimisation' and be called 'global.py'
         mkdir -p ${OPT_DIR}
-        cp ${PSYCLONE_LFRIC_DIR}/acc_parallel.py ${OPT_DIR}/global.py
+        cp ${PSYCLONE_LFRIC_DIR}/gpu_offloading.py ${OPT_DIR}/global.py
         # Clean previous version and compile again
         rm -rf applications/gungho_model/working
-        ./build/local_build.py -a gungho_model -p psyclone-test
+        LFRIC_OFFLOAD_DIRECTIVES=acc ./build/local_build.py -a gungho_model -p psyclone-test
         cd applications/gungho_model/example
         cp ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_gungho_configuration_4its.nml configuration.nml
         mpirun -n 1 ../bin/gungho_model configuration.nml |& tee output.txt

.github/workflows/nemo_tests.yml

@@ -46,13 +46,13 @@ jobs:
     if: ${{ github.repository == 'stfc/PSyclone-mirror' }}
     runs-on: self-hosted
     env:
-      HDF5_VERSION: 1.14.4.3
+      HDF5_VERSION: 1.14.5
       NETCDF_C_VERSION: 4.9.2
       NETCDF_FORTRAN_VERSION: 4.6.1
       NVFORTRAN_VERSION: 23.7
       ONEAPI_VERSION: 2024.2.1
       PERL_VERSION: 5.40.0
-      PYTHON_VERSION: 3.12.5
+      PYTHON_VERSION: 3.13.0
 
     steps:
     - uses: actions/checkout@v3

.github/workflows/python-package.yml

@@ -62,7 +62,7 @@ jobs:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
-        python-version: '3.12'
+        python-version: '3.13'
     - run: sudo apt-get install -y graphviz doxygen
     - run: python -m pip install --upgrade pip
     - run: pip install .[doc]
@@ -83,7 +83,7 @@ jobs:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
-        python-version: '3.12'
+        python-version: '3.13'
     - run: python -m pip install --upgrade pip
     - run: pip install .[doc]
     # Sphinx since version 7.2 (7.2.0/1/2) aborts with
@@ -99,7 +99,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.12]
+        python-version: [3.7, 3.8, 3.13]
     steps:
     - uses: actions/checkout@v4
       with:

changelog

@@ -234,6 +234,20 @@
 	80) PR #2596 for #2592 and #2582. Routine nodes manage their own
 	symbols from their parent scope.
 
+	81) PR #2725 to close #717. Removes some TODOs and associated utility code
+	from the fparser2 frontend that is now unused.
+
+	82) PR #2733 for #2730. Adds OpenMP offloading support for LFRic plus
+	associated integration test.
+
+	83) PR #2743 for #2742. Ensure generated code for function spaces in
+	LFRic is always in a consistent order.
+
+	84) PR #2746. Update minor versions of CUDA, HDF5, nvfortran and
+	Python used in the CI.
+
+	85) PR #2748. Revert Python used on RTD to 3.12 as 3.13 unsupported.
+
 release 2.5.0 14th of February 2024
 
 	1) PR #2199 for #2189. Fix bugs with missing maps in enter data

doc/developer_guide/working_practises.rst

@@ -448,13 +448,13 @@ computational cost (so that we 'fail fast'):
  3. All links within the Sphinx documentation (rst files) are checked (see
     note below);
 
- 4. All of the examples are tested (for Python versions 3.7, 3.8 and 3.12)
+ 4. All of the examples are tested (for Python versions 3.7, 3.8 and 3.13)
     using the ``Makefile`` in the ``examples`` directory. No compilation is
     performed; only the ``transform`` (performs the PSyclone transformations)
     and ``notebook`` (runs the various Jupyter notebooks) targets are used.
     The ``transform`` target is run 2-way parallel (``-j 2``).
 
- 5. The full test suite is run for Python versions 3.7, 3.8 and 3.12 but
+ 5. The full test suite is run for Python versions 3.7, 3.8 and 3.13 but
     without the compilation checks. ``pytest`` is passed the ``-n auto`` flag
     so that it will run the tests in parallel on as many cores as are
     available (currently 2 on GHA instances).

doc/user_guide/getting_going.rst

@@ -212,7 +212,7 @@ Dependencies
 ------------
 
 PSyclone is written in Python so needs Python 3 to be installed on the
-target machine. PSyclone is regularly tested with Python 3.7, 3.8 and 3.12
+target machine. PSyclone is regularly tested with Python 3.7, 3.8 and 3.13
 but should work with any version >= 3.6. (The last PSyclone release to
 support Python 2.7 was version 2.1.0.)
 

examples/gocean/eg5/extract/Makefile

@@ -122,13 +122,13 @@ $(NAME): $(INF_LIB) $(EXTRACT_DIR)/$(LIB_NAME) $(KERNELS) alg.o psy.o
 #TODO #1757: $(INF_LIB) is required because of the meta-data in the
 #            kernel - once this is fixed, $(INF_LIB) can be removed.
 $(DRIVER_INIT).$(TYPE):	$(KERNELS) $(DRIVER_INIT).o
-	$(F90) $(KERNELS) $(DRIVER_INIT).o -o $(DRIVER_INIT).$(TYPE) \
+	$(F90) $(F90FLAGS) $(KERNELS) $(DRIVER_INIT).o -o $(DRIVER_INIT).$(TYPE) \
 		$(INF_LIB) $(EXTRACT_DIR)/$(LIB_NAME) $(LDFLAGS)
 
 #TODO #1757: $(INF_LIB) is required because of the meta-data in the
 #            kernel - once this is fixed, $(INF_LIB) can be removed.
 $(DRIVER_UPDATE).$(TYPE): $(KERNELS) $(DRIVER_UPDATE).o
-	$(F90) $(KERNELS) $(DRIVER_UPDATE).o -o $(DRIVER_UPDATE).$(TYPE) \
+	$(F90) $(F90FLAGS) $(KERNELS) $(DRIVER_UPDATE).o -o $(DRIVER_UPDATE).$(TYPE) \
 		$(INF_LIB) $(EXTRACT_DIR)/$(LIB_NAME) $(LDFLAGS)
 
 # The dl_esm_inf library

examples/lfric/scripts/KGOs/lfric_3269_nvidia.patch

@@ -57,65 +57,6 @@ index 19c9cff9..b5cd3014 100644
  	$(call MESSAGE,Compiled,$<)
 
 
-diff --git a/lfric/infrastructure/build/cxx/nvc++.mk b/lfric/infrastructure/build/cxx/nvc++.mk
-new file mode 100644
-index 00000000..13b17a10
---- /dev/null
-+++ b/lfric/infrastructure/build/cxx/nvc++.mk
-@@ -0,0 +1,9 @@
-+##############################################################################
-+# (c) Crown copyright 2017 Met Office. All rights reserved.
-+# The file LICENCE, distributed with this code, contains details of the terms
-+# under which the code may be used.
-+##############################################################################
-+
-+$(info ** Chosen NVC++ compiler)
-+
-+CXX_RUNTIME_LIBRARY=stdc++
-diff --git a/lfric/infrastructure/build/fortran/nvfortran.mk b/lfric/infrastructure/build/fortran/nvfortran.mk
-new file mode 100644
-index 00000000..cfed52c1
---- /dev/null
-+++ b/lfric/infrastructure/build/fortran/nvfortran.mk
-@@ -0,0 +1,38 @@
-+##############################################################################
-+# Copyright (c) 2017,  Met Office, on behalf of HMSO and Queen's Printer
-+# For further details please refer to the file LICENCE.original which you
-+# should have received as part of this distribution.
-+##############################################################################
-+# Various things specific to the Portland Fortran compiler.
-+##############################################################################
-+#
-+# This macro is evaluated now (:= syntax) so it may be used as many times as
-+# desired without wasting time rerunning it.
-+#
-+F_MOD_DESTINATION_ARG = -module$(SPACE)
-+OPENMP_ARG            = -mp
-+
-+FFLAGS_COMPILER           =
-+FFLAGS_NO_OPTIMISATION    = -O0
-+FFLAGS_SAFE_OPTIMISATION  = -O2
-+FFLAGS_RISKY_OPTIMISATION = -O4
-+FFLAGS_DEBUG              = -g -traceback
-+FFLAGS_RUNTIME            = -Mchkptr -Mchkstk
-+# Option for checking code meets Fortran standard (not available for PGI)
-+FFLAGS_FORTRAN_STANDARD   =
-+
-+LDFLAGS_COMPILER = -g
-+
-+FPP = cpp -traditional-cpp
-+FPPFLAGS = -P
-+FC = mpif90
-+
-+# FS#34981 (nvbug 4648082)
-+science/src/um/src/atmosphere/large_scale_precipitation/ls_ppnc.o: private FFLAGS_RUNTIME = -Mchkstk
-+
-+# FS#35751
-+mesh/create_mesh_mod.o: private FFLAGS_RUNTIME = -Mchkstk
-+
-+# 24.3
-+science/src/socrates/src/cosp_github/subsample_and_optics_example/optics/quickbeam_optics/optics_lib.o: private FFLAGS_SAFE_OPTIMISATION = -O1
-+science/src/socrates/src/cosp_github/subsample_and_optics_example/optics/quickbeam_optics/optics_lib.o: private FFLAGS_RISKY_OPTIMISATION = -O1
 diff --git a/lfric/infrastructure/build/tools/DependencyRules b/lfric/infrastructure/build/tools/DependencyRules
 index 9d4db390..e37384fc 100755
 --- a/lfric/infrastructure/build/tools/DependencyRules

examples/lfric/scripts/KGOs/nvfortran_acc.mk

@@ -20,9 +20,20 @@ FFLAGS_DEBUG              = -g -traceback
 FFLAGS_RUNTIME            = -Mchkptr -Mchkstk
 # Option for checking code meets Fortran standard (not available for PGI)
 FFLAGS_FORTRAN_STANDARD   =
-OPENMP_ARG = -acc=gpu -gpu=managed -mp=multicore
 
-LDFLAGS_COMPILER = -g -acc=gpu -gpu=managed -mp=multicore -cuda
+# Flags for OpenMP threading / OpenMP offloading / OpenACC Offloading
+# The LFRIC_OFFLOAD_DIRECTIVES env_variable is also queried in the PSyclone
+# script to generate matching directives
+ifeq ("$(LFRIC_OFFLOAD_DIRECTIVES)", "omp")
+	OPENMP_ARG = -mp=gpu -gpu=managed
+	LDFLAGS_COMPILER = -mp=gpu -gpu=managed -cuda
+else ifeq ("$(LFRIC_OFFLOAD_DIRECTIVES)", "acc")
+	OPENMP_ARG = -acc=gpu -gpu=managed -mp=multicore
+	LDFLAGS_COMPILER = -acc=gpu -gpu=managed -mp=multicore -cuda
+else
+	OPENMP_ARG = -mp
+	LDFLAGS_COMPILER = -mp
+endif
 
 FPP = cpp -traditional-cpp
 FPPFLAGS = -P

examples/lfric/scripts/Makefile

@@ -45,7 +45,7 @@ transform: ${SCRIPTS}
 .PHONY: ${SCRIPTS}
 
 ${SCRIPTS}:
-	${PSYCLONE} -api lfric -s ./$@ ../eg3/solver_mod.x90 -oalg /dev/null -opsy /dev/null
+	LFRIC_OFFLOAD_DIRECTIVES=acc ${PSYCLONE} -api lfric -s ./$@ ../eg3/solver_mod.x90 -oalg /dev/null -opsy /dev/null
 
 compile: transform
 	@echo "No compilation supported for lfric/scripts examples"