JDBetteridge/python312 (#3546)

* Fix cython files for Cython 3.0

* Install with latest numpy

* Update documentation to Python3.9-3.12

---------

Co-authored-by: Connor Ward <c.ward20@imperial.ac.uk>

.github/workflows/build.yml

@@ -88,7 +88,7 @@ jobs:
           . ../firedrake_venv/bin/activate
           python "$(which firedrake-clean)"
           python -m pip install \
-            pytest-cov pytest-timeout pytest-xdist pytest-timeout ipympl
+            pytest-xdist pytest-timeout ipympl
           python -m pip list
       - name: Test Firedrake
         run: |
@@ -98,7 +98,6 @@ jobs:
           python -m pytest -v tests/test_0init.py
           python -m pytest \
             --durations=200 \
-            --cov firedrake \
             --timeout=1800 \
             --timeout-method=thread \
             -o faulthandler_timeout=1860 \

.github/workflows/zenodo-canary.yml

@@ -19,10 +19,10 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.12
       - name: Install deps
         run: |
-          pip install requests
+          pip install requests packaging
       - name: Zenodo API canary
         run: |
           python scripts/firedrake-install --test-doi-resolution

docker/Dockerfile.env

@@ -1,6 +1,6 @@
 # DockerFile for an environment into which firedrake can be installed.
 
-FROM ubuntu:22.04
+FROM ubuntu:24.04
 
 # Update and install required packages for Firedrake
 USER root
@@ -13,17 +13,21 @@ RUN apt-get update \
                  cmake gfortran git libopenblas-serial-dev \
                  libtool python3-dev python3-pip python3-tk python3-venv \
                  python3-requests zlib1g-dev libboost-dev sudo gmsh \
-                 bison flex \
-                 liboce-ocaf-dev \
+                 bison flex ninja-build \
+                 libocct-ocaf-dev libocct-data-exchange-dev \
                  swig graphviz \
                  libcurl4-openssl-dev libxml2-dev \
     && rm -rf /var/lib/apt/lists/*
 
 # Use a more sane locale
 ENV LC_ALL C.UTF-8
 
-# Set up user so that we do not run as root
-RUN useradd -m -s /bin/bash -G sudo firedrake && \
+# Change the `ubuntu` user to `firedrake`
+# and ensure that we do not run as root on self-hosted systems
+RUN usermod -d /home/firedrake -m ubuntu && \
+    usermod -l firedrake ubuntu && \
+    groupmod -n firedrake ubuntu && \
+    usermod -aG sudo firedrake && \
     echo "firedrake:docker" | chpasswd && \
     echo "firedrake ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers && \
     ldconfig

docs/source/download.rst

@@ -107,18 +107,15 @@ packages can be installed into an existing Firedrake installation using
 System requirements
 -------------------
 
-Firedrake requires Python 3.8.x to 3.11.x. On MacOS Arm (M1 or M2) Python 3.9.x
-to 3.11.x is required. Many externally managed dependencies such as VTK
-have yet to create binary wheels for 3.11.x, but we have generated these
-for the major supported platforms.
-The installation script is tested on Ubuntu and MacOS X. On Ubuntu 22.04
-or later, the system installed Python 3 is supported and tested. On
-MacOS, the homebrew_ installed Python 3 is supported::
+Firedrake requires Python 3.9 to 3.12. The installation script is
+tested by CI on Ubuntu 24.04 LTS. On Ubuntu 22.04 or later, the system
+installed Python 3 is supported. On MacOS, the homebrew_ installed
+Python 3 is supported::
 
   brew install python3
 
 Installation is likely to work well on other Linux platforms, although
-the script may stop to ask you to install some dependency packages.
+the script may fail if dependency packages are not already installed.
 Installation on other Unix platforms may work but is untested. On Linux
 systems that do not use the Debian package management system, it will be
 necessary to pass the `--no-package-manager` option to the install
@@ -129,12 +126,13 @@ they have the system dependencies:
 * A Fortran compiler (for PETSc)
 * Blas and Lapack
 * Git, Mercurial
-* Python version 3.8.x-3.11.x (3.9.x-3.11.x on MacOS Arm)
+* Python version 3.9-3.12
 * The Python headers
 * autoconf, automake, libtool
 * CMake
 * zlib
 * flex, bison
+* Ninja
 
 Firedrake has been successfully installed on Windows 10 using the
 Windows Subsystem for Linux. There are more detailed instructions for

docs/source/install-debug.dot

@@ -8,15 +8,15 @@ digraph triage {
     venv_activated [label="venv activated?"];
     install_script_up_to_date [label="Install script\nup to date?"];
     using_anaconda [label="Using\nAnaconda?"];
-    python_version [label="Python <3.8?"];
+    python_version [label="Python <3.9?"];
     using_macos [label="Using\nMacOS?"];
     using_homebrew [label="Using\nHomebrew?"];
     url_error [label="URL Error with SSL\ncertificate failure?"];
     which_python [label="<which python3> points\nat <$(brew --prefix)/bin/python3>?"];
 
     activate_venv [label="Activate the\nvenv first."];
     uninstall_anaconda [label="Deactivate\nAnaconda."];
-    update_python [label="Get Python 3.8-3.11"];
+    update_python [label="Get Python 3.9-3.12"];
     update_install_script [label="Fetch new\ninstall script"];
     get_homebrew [label="Use Homebrew."];
     brew_doctor [label="brew doctor"];

firedrake/checkpointing.py

@@ -1359,7 +1359,7 @@ def _load_function_topology(self, tmesh, element, tf_name, idx=None):
         if element.family() == "Real":
             assert not isinstance(element, (finat.ufl.VectorElement, finat.ufl.TensorElement))
             value = self.get_attr(path, "_".join([PREFIX, "value" if idx is None else "value_" + str(idx)]))
-            tf.dat.data.itemset(value)
+            tf.dat.data[...] = value
         else:
             if path in self.h5pyfile:
                 timestepping = self.has_attr(os.path.join(path, tf.name()), "timestepping")

firedrake/cython/dmcommon.pyx

@@ -2,6 +2,7 @@
 
 # Utility functions common to all DMs used in Firedrake
 import functools
+import math
 import cython
 import numpy as np
 import firedrake
@@ -675,7 +676,7 @@ def closure_ordering(PETSc.DM dm,
                                 incident = 1
                                 break
                         if incident == 0:
-                            face_indices[nfaces] += v * 10**(1-fi)
+                            face_indices[nfaces] += v * <PetscInt> 10**(1-fi)
                             fi += 1
                     nfaces += 1
 
@@ -970,7 +971,7 @@ cdef inline PetscInt _compute_orientation_simplex(PetscInt *fiat_cone,
         coneSize1 -= 1
     assert n == coneSize
     for k in range(n):
-        o += np.math.factorial(n - 1 - k) * inds[k]
+        o += math.factorial(n - 1 - k) * inds[k]
     CHKERR(PetscFree(cone1))
     CHKERR(PetscFree(inds))
     return o
@@ -1019,10 +1020,10 @@ cdef inline PetscInt _compute_orientation_interval_tensor_product(PetscInt *fiat
                     # io += (2**(dim - 1 - i)) * 0
                     pass
                 elif plex_cone_copy[2 * j + 1] == fiat_cone[2 * i] and plex_cone_copy[2 * j] == fiat_cone[2 * i + 1]:
-                    io += (2**(dim - 1 - i)) * 1
+                    io += <PetscInt> (2**(dim - 1 - i)) * 1
                 else:
                     raise RuntimeError("Found inconsistent fiat_cone and plex_cone")
-                eo += np.math.factorial(dim - 1 - i) * j
+                eo += math.factorial(dim - 1 - i) * j
                 for k in range(j, dim1 - 1):
                     plex_cone_copy[2 * k] = plex_cone_copy[2 * k + 2]
                     plex_cone_copy[2 * k + 1] = plex_cone_copy[2 * k + 3]
@@ -1031,7 +1032,7 @@ cdef inline PetscInt _compute_orientation_interval_tensor_product(PetscInt *fiat
         else:
             raise RuntimeError("Found inconsistent fiat_cone and plex_cone")
     assert dim1 == 0
-    return (2**dim) * eo + io
+    return <PetscInt> (2**dim) * eo + io
 
 
 cdef inline PetscInt _compute_orientation(PETSc.DM dm,
@@ -2552,7 +2553,7 @@ cdef struct CommFacet:
     PetscInt global_u, global_v
     PetscInt local_facet
 
-cdef int CommFacet_cmp(const void *x_, const void *y_) nogil:
+cdef int CommFacet_cmp(const void *x_, const void *y_) noexcept nogil:
     """Three-way comparison C function for CommFacet structs."""
     cdef:
         CommFacet *x = <CommFacet *>x_
@@ -3334,7 +3335,7 @@ cdef int DMPlexGetAdjacency_Facet_Support(PETSc.PetscDM dm,
                                           PetscInt p,
                                           PetscInt *adjSize,
                                           PetscInt adj[],
-                                          void *ctx) nogil:
+                                          void *ctx) noexcept nogil:
     """Custom adjacency callback for halo growth.
 
     :arg dm: The DMPlex object.

firedrake/cython/mgimpl.pyx

@@ -254,7 +254,7 @@ def coarse_to_fine_cells(mc, mf, clgmaps, flgmaps):
     cdm = mc.topology_dm
     fdm = mf.topology_dm
     dim = cdm.getDimension()
-    nref = 2 ** dim
+    nref = <PetscInt> 2 ** dim
     ncoarse = mc.cell_set.size
     nfine = mf.cell_set.size
     co2n, _ = get_entity_renumbering(cdm, mc._cell_numbering, "cell")

firedrake/cython/petschdr.pxi

@@ -178,7 +178,7 @@ cdef inline int SETERR(int ierr) with gil:
         PyErr_SetObject(<object>PyExc_RuntimeError, <long>ierr)
     return ierr
 
-cdef inline int CHKERR(int ierr) nogil except -1:
+cdef inline int CHKERR(int ierr) except -1 nogil:
     if ierr == 0:
         return 0 # no error
     else:

firedrake/mesh.py

@@ -21,7 +21,7 @@
 from pyop2.mpi import (
     MPI, COMM_WORLD, internal_comm, is_pyop2_comm, temp_internal_comm
 )
-from pyop2.utils import as_tuple, tuplify
+from pyop2.utils import as_tuple
 
 import firedrake.cython.dmcommon as dmcommon
 import firedrake.cython.extrusion_numbering as extnum
@@ -1445,7 +1445,14 @@ def make_dofs_per_plex_entity(self, entity_dofs):
         dofs_per_entity = np.zeros((1 + self._base_mesh.cell_dimension(), 2), dtype=IntType)
         for (b, v), entities in entity_dofs.items():
             dofs_per_entity[b, v] += len(entities[0])
-        return tuplify(dofs_per_entity)
+
+        # Convert to a tuple of tuples with int (not numpy.intXX) values. This is
+        # to give us a string representation like ((0, 1), (2, 3)) instead of
+        # ((numpy.int32(0), numpy.int32(1)), (numpy.int32(2), numpy.int32(3))).
+        return tuple(
+            tuple(int(d_) for d_ in d)
+            for d in dofs_per_entity
+        )
 
     @PETSc.Log.EventDecorator()
     def node_classes(self, nodes_per_entity, real_tensorproduct=False):

firedrake/preconditioners/patch.py

@@ -816,10 +816,13 @@ def initialize(self, obj):
             is_snes = False
 
         if len(bcs) > 0:
-            ghost_bc_nodes = numpy.unique(numpy.concatenate([bcdofs(bc, ghost=True)
-                                                             for bc in bcs]))
-            global_bc_nodes = numpy.unique(numpy.concatenate([bcdofs(bc, ghost=False)
-                                                              for bc in bcs]))
+            ghost_bc_nodes = numpy.unique(
+                numpy.concatenate([bcdofs(bc, ghost=True) for bc in bcs],
+                                  dtype=PETSc.IntType)
+            )
+            global_bc_nodes = numpy.unique(
+                numpy.concatenate([bcdofs(bc, ghost=False) for bc in bcs],
+                                  dtype=PETSc.IntType))
         else:
             ghost_bc_nodes = numpy.empty(0, dtype=PETSc.IntType)
             global_bc_nodes = numpy.empty(0, dtype=PETSc.IntType)

scripts/firedrake-install

@@ -132,18 +132,14 @@ log = logging.getLogger()
 
 log.info("Running %s" % " ".join(sys.argv))
 
-if sys.version_info >= (3, 12):
-    print("""\nCan not install Firedrake with Python 3.12 at the moment:
-Some wheels are not yet available for Python 3.12 for some required package(s).
-Please install with Python 3.11 (or an earlier version >= 3.8).""")
+if sys.version_info >= (3, 13):
+    print("""\nCan not install Firedrake with Python 3.13 at the moment:
+Some wheels are not yet available for Python 3.13 for some required package(s).
+Please install with Python 3.12 (or an earlier version >= 3.9).""")
     sys.exit(1)
-elif sys.version_info < (3, 9) and osname == "Darwin" and arch == "arm64":
-    print("""
-Installing Firedrake on Mac Arm (M1 or M2) requires at least Python 3.9 since
-some required package(s) are not available for earlier Python versions.""")
-elif sys.version_info < (3, 8):
+elif sys.version_info < (3, 9):
     if mode == "install":
-        print("""\nInstalling Firedrake requires Python 3, at least version 3.8.
+        print("""\nInstalling Firedrake requires Python 3, at least version 3.9.
 You should run firedrake-install with python3.""")
     if mode == "update":
         if hasattr(sys, "real_prefix"):
@@ -688,7 +684,6 @@ def check_output(args):
         raise
 
 
-pyinstall = [python, "setup.py", "install"]
 if "PYTHONPATH" in os.environ and not args.honour_pythonpath:
     quit("""The PYTHONPATH environment variable is set. This is probably an error.
 If you really want to use your own Python packages, please run again with the
@@ -992,14 +987,10 @@ def run_pip_install(pipargs):
     # subprocesses wrote out.
     # Particularly important for debugging petsc fails.
     with environment(**blas):
-        pipargs = ["-vvv"] + pipargs
+        pipargs = ["-v"] + pipargs
         check_call(pipinstall + pipargs)
 
 
-def run_cmd(args):
-    check_call(args)
-
-
 def get_requirements(reqfname):
     with open(reqfname, "r") as f:
         reqs = f.readlines()
@@ -1012,14 +1003,6 @@ def run_pip_install_wrap(reqs, parallel_compiler_env):
         if package_name in parallel_packages:
             with environment(**parallel_compiler_env):
                 run_pip_install([req])
-        elif package_name == "numpy":
-            # Downgrade setuptools and wheel for numpy
-            run_pip(["install", "-U", "setuptools==59.2.0"])
-            run_pip(["install", "-U", "wheel==0.37.0"])
-            run_pip_install(["numpy==1.24"])
-            # Upgrade setuptools and wheel for everything else
-            run_pip(["install", "-U", "setuptools"])
-            run_pip(["install", "-U", "wheel"])
         else:
             run_pip_install([req])
 
@@ -1540,6 +1523,7 @@ if mode == "install" or not args.update_script:
                              "make",
                              "automake",
                              "cmake",
+                             "ninja",
                              "libtool",
                              "boost"]
         if args.with_blas is None and mode == "install":
@@ -1558,6 +1542,7 @@ if mode == "install" or not args.update_script:
                              "pkgconf",  # for p4est
                              "libtool",
                              "libxml2-dev",
+                             "ninja-build",  # for meson/numpy
                              "python3-dev",
                              "python3-pip",
                              "python3-tk",
@@ -1697,15 +1682,18 @@ if mode == "install":
     os.environ["VIRTUAL_ENV"] = firedrake_env
 
 # Ensure pip, setuptools, hatchling and wheel are at the latest version.
-# numpy requires setuptools==59.2.0 and wheel==0.37.0 until it moves to meson build
-run_pip(["install", "-U", "setuptools==59.2.0"])
+run_pip(["install", "-U", "setuptools"])
 run_pip(["install", "-U", "hatch"])
 run_pip(["install", "-U", "editables"])
 run_pip(["install", "-U", "pip"])
-run_pip(["install", "-U", "wheel==0.37.0"])
+run_pip(["install", "-U", "wheel"])
+
+# Extra numpy dependendencies, see
+# https://github.com/numpy/numpy/blob/main/pyproject.toml
+run_pip(["install", "-U", "meson-python>=0.15.0"])
+run_pip(["install", "-U", "Cython>=3.0.6"])
 
-# Pin Cython because it's causing multiple packages to fail to build
-run_pip(["install", "-U", "Cython==0.29.36"])
+run_pip(["install", "-U", "numpy"])
 
 # Loopy has additional build dependencies
 run_pip(["install", "-U", "scikit-build"])
@@ -1793,24 +1781,12 @@ if mode == "install":
     # recovery can be attempted if required.
     build_update_script()
 
-    # Force Cython to install first to work around pip dependency issues.
-    run_pip_install(["Cython>=0.22"])
-
     # Pre-install requested packages
     if args.pip_packages is not None:
         for package in args.pip_packages:
             log.info("Pip installing %s to venv" % package)
             run_pip_install_wrap(package.split(), {})
 
-    # numpy requires old setuptools (+wheel)
-    log.info("Installing numpy using setuptools==59.2.0 and wheel==0.37.0 and Cython==0.29.36")
-    log.info("Installing numpy==1.24 due to performance regression")
-    # https://github.com/inducer/pytential/issues/211
-    run_pip_install(["numpy==1.24"])
-    log.info("Updating setuptools and wheel to latest versions")
-    run_pip(["install", "-U", "setuptools"])
-    run_pip(["install", "-U", "wheel"])
-
     # Install VTK
     if not args.no_vtk:
         log.info("Pip installing VTK to venv")