firedrake-install

#! /usr/bin/env python3
import logging
import platform
import subprocess
import sys
import os
import shutil
from argparse import ArgumentParser, RawDescriptionHelpFormatter
import argparse
from collections import OrderedDict
import atexit
import json
import pprint
import shlex
from glob import iglob
from itertools import chain
import re
import importlib
import time
import warnings


warnings.warn(
    "firedrake-install/firedrake-update is no longer the recommended way to "
    "install/update Firedrake. Please use pip instead. For instructions on how "
    "to do this please visit https://www.firedrakeproject.org/install.html.",
    FutureWarning
)
# Give users time to see the warning
time.sleep(5)


class InstallError(Exception):
    # Exception for generic install problems.
    pass


try:
    from pkg_resources.extern.packaging.version import Version, InvalidVersion
except ModuleNotFoundError:
    try:
        from packaging.version import Version, InvalidVersion
    except ModuleNotFoundError:
        raise InstallError("Neither setuptools or packaging found. Please "
                           "install one of these packages before trying again.")

osname = platform.uname().system
arch = platform.uname().machine

# Packages which we wish to ensure are always recompiled
wheel_blacklist = ["mpi4py", "randomgen", "numpy"]
# Packages for which we set CC=mpicc, CXX=mpicxx, F90=mpif90.
parallel_packages = ["h5py", "petsc4py", "slepc", "slepc4py", "libsupermesh", "firedrake"]

# Firedrake application installation shortcuts.
firedrake_apps = {
    "gusto": ("""Atmospheric dynamical core library. http://firedrakeproject.org/gusto""",
              "git+ssh://github.com/firedrakeproject/gusto.git@main#egg=gusto"),
    "thetis": ("""Coastal ocean model. http://thetisproject.org""",
               "git+ssh://github.com/thetisproject/thetis#egg=thetis"),
    "icepack": ("""Glacier and ice sheet model. https://icepack.github.io""",
                "git+ssh://github.com/icepack/icepack.git#egg=icepack"),
    "irksome": ("""Implicit Runge-Kutta methods. https://github.com/firedrakeproject/Irksome/""",
                "git+ssh://github.com/firedrakeproject/Irksome.git#egg=Irksome"),
    "femlium": ("""Interactive visualization of finite element simulations on geographic maps with folium. https://femlium.github.io/""",
                "git+ssh://github.com/FEMlium/FEMlium.git@main#egg=FEMlium"),
    "fascd": ("""Full approximation scheme constraint decomposition solver for variational inequalities. https://bitbucket.org/pefarrell/fascd""",
              "git+ssh://bitbucket.org/pefarrell/fascd.git@master#egg=fascd"),
    "defcon": ("""Deflated continuation algorithm for bifurcation analysis. https://bitbucket.org/pefarrell/defcon""",
               "git+ssh://bitbucket.org/pefarrell/defcon.git@master#egg=defcon"),
    "gadopt": ("""Geodynamic Adjoint Optimization Platform. http://g-adopt.github.io""",
               "git+ssh://github.com/g-adopt/g-adopt.git@master#egg=gadopt"),
    "asQ": ("""ParaDiag parallel-in-time methods. http://firedrakeproject.org/asQ""",
            "git+ssh://github.com/firedrakeproject/asQ.git@master#egg=asQ"),
}


class FiredrakeConfiguration(dict):
    """A dictionary extended to facilitate the storage of Firedrake
    configuration information."""
    def __init__(self, args=None):
        super(FiredrakeConfiguration, self).__init__()

        '''A record of the persistent options in force.'''
        self["options"] = {}
        '''Relevant environment variables.'''
        self["environment"] = {}
        '''Additional packages installed via the plugin interface.'''
        self["additions"] = []

        if args:
            for o in self._persistent_options:
                if o in args.__dict__.keys():
                    self["options"][o] = args.__dict__[o]

    _persistent_options = ["package_manager",
                           "minimal_petsc", "mpicc", "mpicxx", "mpif90", "mpiexec", "mpihome", "disable_ssh",
                           "honour_petsc_dir", "with_parmetis",
                           "slepc", "packages", "honour_pythonpath",
                           "opencascade", "torch", "jax",
                           "petsc_int_type", "cache_dir", "complex", "remove_build_files", "with_blas", "netgen"]


def deep_update(this, that):
    from collections import abc
    for k, v in that.items():
        if isinstance(v, abc.Mapping) and k in this.keys():
            this[k] = deep_update(this.get(k, {}), v)
        else:
            this[k] = v
    return this


if os.path.basename(__file__) == "firedrake-install":
    mode = "install"
    logfile_directory = os.path.abspath(os.getcwd())
    logfile_mode = "w"

elif os.path.basename(__file__) == "firedrake-update":
    mode = "update"
    os.chdir(os.path.dirname(os.path.realpath(__file__)) + "/../..")
    try:
        logfile_directory = os.environ["VIRTUAL_ENV"]
    except KeyError:
        quit("Unable to retrieve venv name from the environment.\n Please ensure the venv is active before running firedrake-update.")
    logfile_mode = "a" if "--no-update-script" in sys.argv else "w"
else:
    sys.exit("Script must be invoked either as firedrake-install or firedrake-update")


# Set up logging
# Log to file at DEBUG level
if ("-h" in sys.argv) or ("--help" in sys.argv):
    # Don't log if help displayed to avoid overwriting an existing log
    logfile = os.devnull
else:
    logfile = os.path.join(logfile_directory, 'firedrake-%s.log' % mode)
logging.basicConfig(level=logging.DEBUG,
                    format='%(asctime)s %(levelname)-6s %(message)s',
                    filename=logfile,
                    filemode=logfile_mode)
# Log to console at INFO level
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter('%(message)s')
console.setFormatter(formatter)
logging.getLogger().addHandler(console)

log = logging.getLogger()

log.info("Running %s" % " ".join(sys.argv))

if sys.version_info >= (3, 14):
    print("""\nCan not install Firedrake with Python 3.14 at the moment:
Some wheels are not yet available for Python 3.14 for some required package(s).
Please install with Python 3.13 (or an earlier version >= 3.10).""")
    sys.exit(1)
elif sys.version_info < (3, 10):
    if mode == "install":
        print("""\nInstalling Firedrake requires Python 3, at least version 3.10.
You should run firedrake-install with python3.""")
    if mode == "update":
        if hasattr(sys, "real_prefix"):
            # sys.real_prefix exists iff we are in an active virtualenv.
            #
            # Existing install trying to update past the py2/py3 barrier
            print("""\nFiredrake is now Python 3 only.  You cannot upgrade your existing installation.
Please follow the instructions at http://www.firedrakeproject.org/download.html to reinstall.""")
            sys.exit(1)
        else:
            # Accidentally (?) running firedrake-update with python2.
            print("""\nfiredrake-update must be run with Python 3, did you accidentally use Python 2?""")
    sys.exit(1)

branches = {}


def resolve_doi_branches(doi):
    import requests
    import hashlib
    log.info("Installing Firedrake components specified by {}".format(doi))
    # Zenodo updated their API to use elastic search so simply querying with
    # ?q=doi:10.5281/zenodo.<id> now returns multiple results (not ideal)
    # Instead use
    # /<id>
    # as documented here https://developers.zenodo.org/#retrieve
    id_ = doi.split('.')[-1]
    response = requests.get("https://zenodo.org/api/records/{}".format(id_))
    if not response.ok:
        log.error("Unable to obtain Zenodo record for doi {}".format(doi))
        log.error("Response was {}".format(response.json()))
        sys.exit(1)
    record = response.json()
    files = record["files"]
    try:
        componentjson, = (f for f in files if f["key"] == "components.json")
    except ValueError:
        log.error("Expecting to find exactly one 'components.json' in record")
        sys.exit(1)
    download = requests.get(componentjson["links"]["self"])
    if download.status_code >= 400:
        log.error("Unable to download 'components.json'")
        log.error("Response was {}".format(download.json()))
        sys.exit(1)
    # component response has checksum as "md5:HEXDIGEST", strip the md5.
    if hashlib.md5(download.content).hexdigest() != componentjson["checksum"][4:]:
        log.error("Download failed checksum, expecting {expect}, got {got}".format(
            expect=componentjson["checksum"][4:],
            got=hashlib.md5(download.content).hexdigest()))
        sys.exit(1)
    componentjson = download.json()
    branches = {}
    for record in componentjson["components"]:
        commit = record["commit"]
        component = record["component"]
        package = component[component.find("/")+1:].lower()
        branches[package] = commit
        log.info("Using commit {commit} for component {comp}".format(
            commit=commit, comp=component))
    return branches


def honour_petsc_dir_get_petsc_dir():
    try:
        petsc_dir = os.environ["PETSC_DIR"]
    except KeyError:
        raise InstallError("Unable to find installed PETSc (did you forget to set PETSC_DIR?)")
    petsc_arch = os.environ.get("PETSC_ARCH", "")
    return petsc_dir, petsc_arch


def honour_petsc_dir_fetch_petscconf(name):
    # Return the line in "petscconf.h" that starts with name.
    petsc_dir, petsc_arch = honour_petsc_dir_get_petsc_dir()
    petscconf_h = os.path.join(petsc_dir, petsc_arch, "include", "petscconf.h")
    with open(petscconf_h) as fh:
        for line in fh.readlines():
            if line.startswith(name):
                return line


def honour_petsc_dir_get_petsc_int_type():
    line = honour_petsc_dir_fetch_petscconf("#define PETSC_USE_64BIT_INDICES")
    if line and line.split()[2] == '1':
        return "int64"
    else:
        return "int32"


def honour_petsc_dir_get_petsc_packages():
    line = honour_petsc_dir_fetch_petscconf("#define PETSC_HAVE_PACKAGES")
    return set(line.split()[2].strip().strip(':"').split(':'))


if mode == "install":
    # Handle command line arguments.
    parser = ArgumentParser(description="""Install firedrake and its dependencies.""",
                            epilog="""The install process has three steps.

1. Any required system packages are installed using brew (MacOS) or apt (Ubuntu
   and similar Linux systems). On a Linux system without apt, the installation
   will fail if a dependency is not found.

2. A set of standard and/or third party Python packages is installed to the
   specified install location.

3. The core set of Python packages is downloaded to ./firedrake/src/ and
   installed to the specified location.

The install creates a venv in ./firedrake (or in ./name where name is the
parameter given to --venv-name) and installs inside that venv.

The installer will ensure that the required configuration options are
passed to PETSc. In addition, any configure options which you provide
in the PETSC_CONFIGURE_OPTIONS environment variable will be
honoured.""",
                            formatter_class=RawDescriptionHelpFormatter)

    parser.add_argument("--slepc", action="store_true",
                        help="Install SLEPc along with PETSc.")
    parser.add_argument("--opencascade", action="store_true",
                        help="Install OpenCASCADE for CAD integration.")
    parser.add_argument("--torch", const="cpu", default=False, nargs='?', choices=["cpu", "cuda"],
                        help="Install PyTorch for a CPU or CUDA backend (default: CPU).")
    parser.add_argument("--jax", const="cpu", default=False, nargs='?', choices=["cpu", "cuda"],
                        help="Install JAX for a CPU or CUDA backend (default: CPU).")
    parser.add_argument("--disable-ssh", action="store_true",
                        help="Do not attempt to use ssh to clone git repositories: fall immediately back to https.")
    parser.add_argument("--no-package-manager", action='store_false', dest="package_manager",
                        help="Do not attempt to use apt or homebrew to install operating system packages on which we depend.")
    group = parser.add_mutually_exclusive_group()
    group.add_argument("--minimal-petsc", action="store_true",
                       help="Minimise the set of petsc dependencies installed. This creates faster build times (useful for build testing).")
    group.add_argument("--with-parmetis", action="store_true",
                       help="Install PETSc with ParMETIS? (Non-free license, see http://glaros.dtc.umn.edu/gkhome/metis/parmetis/download)")
    parser.add_argument("--honour-petsc-dir", action="store_true",
                        help="Usually it is best to let Firedrake build its own PETSc. If you wish to use another PETSc, set PETSC_DIR and pass this option.")
    parser.add_argument("--petsc-int-type", choices=["int32", "int64"],
                        default="int32", type=str,
                        help="The integer type used by PETSc.  Use int64 if you need to solve problems with more than 2 billion degrees of freedom.  Only takes effect if firedrake-install builds PETSc.")

    parser.add_argument("--honour-pythonpath", action="store_true",
                        help="Pointing to external Python packages is usually a user error. Set this option if you know that you want PYTHONPATH set.")
    parser.add_argument("--rebuild-script", action="store_true",
                        help="Only rebuild the firedrake-install script. Use this option if your firedrake-install script is broken and a fix has been released in upstream Firedrake. You will need to specify any other options which you wish to be honoured by your new update script.")
    parser.add_argument("--doi", type=str, nargs=1,
                        help="Install a set of components matching a particular Zenodo DOI. The record should have been created with firedrake-zenodo.")
    # Used for testing if Zenodo broke the API
    # Tries to resolve to a known DOI and immediately exits.
    parser.add_argument("--test-doi-resolution", action="store_true",
                        help=argparse.SUPPRESS)
    parser.add_argument("--package-branch", type=str, nargs=2, action="append", metavar=("PACKAGE", "BRANCH"),
                        help="Specify which branch of a package to use. This takes two arguments, the package name and the branch.")
    parser.add_argument("--verbose", "-v", action="store_true", help="Produce more verbose debugging output.")
    parser.add_argument("--mpicc", type=str,
                        action="store", default=None,
                        help="C compiler to use when building with MPI. If not set, MPICH will be downloaded and used.")
    parser.add_argument("--mpicxx", type=str,
                        action="store", default=None,
                        help="C++ compiler to use when building with MPI. If not set, MPICH will be downloaded and used.")
    parser.add_argument("--mpif90", type=str,
                        action="store", default=None,
                        help="Fortran compiler to use when building with MPI. If not set, MPICH will be downloaded and used.")
    parser.add_argument("--mpiexec", type=str,
                        action="store", default=None,
                        help="MPI launcher. If not set, MPICH will be downloaded and used.")
    parser.add_argument("--mpihome", type=str,
                        action="store", default=None,
                        help="Location of MPI files. If not set, MPICH will be downloaded and used.")
    parser.add_argument("--mpi4py-version", help="Specify an exact version of mpi4py to install")
    parser.add_argument("--show-petsc-configure-options", action="store_true",
                        help="Print out the configure options passed to PETSc and exit")
    parser.add_argument("--show-dependencies", action="store_true",
                        help="Print out the package manager used and packages installed/required.")
    parser.add_argument("--venv-name", default="firedrake",
                        type=str, action="store",
                        help="Name of the venv to create and the name of the install directory relative to this install script (default is 'firedrake' with directory ./firedrake)")
    parser.add_argument("--install", action="append", dest="packages",
                        help="Additional packages to be installed. The address should be in format vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir . Some additional packages have shortcut install options for more information see --install-help.")
    parser.add_argument("--pip-install", action="append", dest="pip_packages",
                        help="Pip install additional packages into the venv")
    parser.add_argument("--install-help", action="store_true",
                        help="Provide information on packages which can be installed using shortcut names.")
    parser.add_argument("--cache-dir", type=str,
                        action="store",
                        help="Directory to use for disk caches of compiled code (default is the .cache subdirectory of the Firedrake installation).")
    parser.add_argument("--complex", action="store_true",
                        help="Installs the complex version of Firedrake; rebuilds PETSc in complex mode.")
    parser.add_argument("--documentation-dependencies", action="store_true",
                        help="Install the dependencies required to build the documentation")
    parser.add_argument("--remove-build-files", action="store_true",
                        help="Cleans up build artefacts from Firedrake venv. Unless you really cannot spare the disk space, this option is not recommended")
    parser.add_argument("--with-blas", default=("download" if osname == "Darwin" else None),
                        help="Specify path to system BLAS directory. Use '--with-blas=download' to download openblas")
    parser.add_argument("--netgen", action="store_true",
                        help="Install NGSolve/Netgen and ngsPETSc.")
    parser.add_argument("--no-vtk", action="store_true",
                        help="Do not install VTK into the Firedrake virtualenv")

    args = parser.parse_args()

    # If the user has set any MPI info, they must set them all
    if args.mpicc or args.mpicxx or args.mpif90 or args.mpiexec or args.mpihome:
        if not (args.mpicc and args.mpicxx and args.mpif90 and args.mpiexec and args.mpihome):
            log.error("If you set any MPI information, you must set all of {mpicc, mpicxx, mpif90, mpiexec, mpihome}.")
            sys.exit(1)

    if args.package_branch:
        branches = {package.lower(): branch for package, branch in args.package_branch}
    if args.test_doi_resolution:
        actual = resolve_doi_branches("10.5281/zenodo.1322546")
        expect = {'coffee': '87e50785d3a05b111f5423a66d461cd44cc4bdc9',
                  'finat': 'aa74fd499304c8363a4520555fd62ef21e8e5e1f',
                  'fiat': '184601a46c24fb5cbf8fd7961d22b16dd26890e7',
                  'firedrake': '6a30b64da01eb587dcc0e04e8e6b84fe4839bdb7',
                  'petsc': '413f72f04f5cb0a010c85e03ed029573ff6d4c63',
                  'petsc4py': 'ac2690070a80211dfdbab04634bdb3496c14ca0a',
                  'tsfc': 'fe9973eaacaa205fd491cd1cc9b3743b93a3d076',
                  'ufl': 'c5eb7fbe89c1091132479c081e6fa9c182191dcc',
                  'pyop2': '741a21ba9a62cb67c0aa300a2e199436ea8cb61c'}
        if actual != expect:
            log.error("Unable to resolve DOI correctly.")
            log.error("You'll need to figure out how Zenodo have changed their API.")
            sys.exit(1)
        else:
            log.info("DOI resolution test passed.")
            sys.exit(0)
    if args.doi:
        branches = resolve_doi_branches(args.doi[0])
    if args.honour_petsc_dir:
        petsc_int_type = honour_petsc_dir_get_petsc_int_type()
        if petsc_int_type != args.petsc_int_type:
            petsc_dir, petsc_arch = honour_petsc_dir_get_petsc_dir()
            log.warning("Provided PETSc (PETSC_DIR=%s, PETSC_ARCH=%s) was compiled for %s, but given --petsc-int-type = %s: setting --petsc-int-type %s." % (petsc_dir, petsc_arch, petsc_int_type, args.petsc_int_type, petsc_int_type))
            args.petsc_int_type = petsc_int_type
    args.prefix = False  # Disabled as untested
    args.packages = args.packages or []

    config = FiredrakeConfiguration(args)
else:
    try:
        with open(os.path.join(os.environ["VIRTUAL_ENV"],
                               ".configuration.json"), "r") as f:
            config = json.load(f)
    except FileNotFoundError:
        raise InstallError("Failed to find existing Firedrake configuration")

    parser = ArgumentParser(description="""Update this firedrake install to the latest versions of all packages.""",
                            formatter_class=RawDescriptionHelpFormatter)
    parser.add_argument("--no-update-script", action="store_false", dest="update_script",
                        help="Do not update script before updating Firedrake.")
    parser.add_argument("--rebuild", action="store_true",
                        help="Rebuild all packages even if no new version is available. Usually petsc and petsc4py are only rebuilt if they change. All other packages are always rebuilt.")
    parser.add_argument("--rebuild-script", action="store_true",
                        help="Only rebuild the firedrake-install script. Use this option if your firedrake-install script is broken and a fix has been released in upstream Firedrake. You will need to specify any other options which you wish to be honoured by your new update script.")
    parser.add_argument("--slepc", action="store_true", dest="slepc", default=config["options"]["slepc"],
                        help="Install SLEPc along with PETSc")
    parser.add_argument("--opencascade", action="store_true", dest="opencascade", default=config["options"].get("opencascade", False),
                        help="Install OpenCASCADE for CAD integration.")
    parser.add_argument("--torch", const="cpu", nargs='?', choices=["cpu", "cuda"], default=config["options"].get("torch", False),
                        help="Install PyTorch for a CPU or CUDA backend (default: CPU).")
    parser.add_argument("--jax", const="cpu", nargs='?', choices=["cpu", "cuda"], default=config["options"].get("jax", False),
                        help="Install JAX for a CPU or CUDA backend (default: CPU).")
    parser.add_argument("--honour-petsc-dir", action="store_true",
                        default=config["options"]["honour_petsc_dir"],
                        help="Usually it is best to let Firedrake build its own PETSc. If you wish to use another PETSc, set PETSC_DIR and pass this option.")
    parser.add_argument("--petsc-int-type", choices=["int32", "int64"],
                        default=config["options"]["petsc_int_type"], type=str,
                        help="The integer type used by PETSc.  Use int64 if you need to solve problems with more than 2 billion degrees of freedom.  Only takes effect if firedrake-install builds PETSc.")

    parser.add_argument("--honour-pythonpath", action="store_true", default=config["options"].get("honour_pythonpath", False),
                        help="Pointing to external Python packages is usually a user error. Set this option if you know that you want PYTHONPATH set.")
    parser.add_argument("--clean", action='store_true',
                        help="Delete any remnants of obsolete Firedrake components.")
    parser.add_argument("--verbose", "-v", action="store_true", help="Produce more verbose debugging output.")
    parser.add_argument("--install", action="append", dest="packages", default=config["options"].get("packages", []),
                        help="Additional packages to be installed. The address should be in format vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir. Some additional packages have shortcut install options for more information see --install-help.")
    parser.add_argument("--pip-install", action="append", dest="pip_packages",
                        help="Pip install additional packages into the venv")
    parser.add_argument("--install-help", action="store_true",
                        help="Provide information on packages which can be installed using shortcut names.")
    parser.add_argument("--cache-dir", type=str,
                        action="store", default=config["options"].get("cache_dir", ""),
                        help="Directory to use for disk caches of compiled code (default is the .cache subdirectory of the Firedrake installation).")
    complex_group = parser.add_mutually_exclusive_group()
    complex_group.add_argument("--complex", action="store_true",
                               help="Updates to the complex version of Firedrake; rebuilds PETSc in complex mode.")
    complex_group.add_argument("--real", action="store_true",
                               help="Updates to the real version of Firedrake; rebuilds PETSc in real mode.")
    parser.add_argument("--documentation-dependencies", action="store_true",
                        help="Install the dependencies required to build the documentation")
    parser.add_argument("--show-dependencies", action="store_true",
                        help="Print out the package manager used and packages installed/required.")
    parser.add_argument("--remove-build-files", action="store_true",
                        help="Cleans up build artefacts from Firedrake venv. Unless you really cannot spare the disk space, this option is not recommended")
    parser.add_argument("--with-blas", default=("download" if osname == "Darwin" else None),
                        help="Specify path to system BLAS directory. Use '--with-blas=download' to download openblas")
    parser.add_argument("--netgen", action="store_true", dest="netgen", default=config["options"].get("netgen", False),
                        help="Install Netgen.")
    parser.add_argument("--mpi4py-version", help="Specify an exact version of mpi4py to install")

    args = parser.parse_args()

    args.packages = list(set(args.packages))  # remove duplicates

    if args.honour_petsc_dir != config["options"]["honour_petsc_dir"]:
        log.error("You installed Firedrake with --honour-petsc-dir=%s, but are trying to update it with --honour-petsc-dir=%s." % (config["options"]["honour_petsc_dir"], args.honour_petsc_dir))
        exit(1)
    if args.honour_petsc_dir:
        # Set int_type to the one found in petscconf.h
        petsc_int_type = honour_petsc_dir_get_petsc_int_type()
        if petsc_int_type != args.petsc_int_type:
            petsc_dir, petsc_arch = honour_petsc_dir_get_petsc_dir()
            log.warning("Provided PETSc (PETSC_DIR=%s, PETSC_ARCH=%s) was compiled for %s, but given --petsc-int-type = %s: setting --petsc-int-type %s." % (petsc_dir, petsc_arch, petsc_int_type, args.petsc_int_type, petsc_int_type))
            args.petsc_int_type = petsc_int_type
    petsc_int_type_changed = False
    if config["options"]["petsc_int_type"] != args.petsc_int_type:
        petsc_int_type_changed = True
        args.rebuild = True
    if config["options"].get("with_blas") is not None:
        if config["options"].get("with_blas") != args.with_blas:
            # Rebuild if BLAS library explicitly changed at command line
            args.rebuild = True
        else:
            # Prevent deep_update overwriting BLAS options if they exist in the config, but are unspecified at the command line
            args.with_blas = config["options"].get("with_blas")

    petsc_complex_type_changed = False
    previously_complex = config["options"].get("complex", False)
    now_complex = not args.real and (args.complex or previously_complex)
    if previously_complex != now_complex:
        petsc_complex_type_changed = True
        args.rebuild = True
    args.complex = now_complex

    config = deep_update(config, FiredrakeConfiguration(args))

if args.install_help:
    help_string = """
You can install the following packages by passing --install shortname
where shortname is one of the names given below:

"""
    componentformat = "|{:10}|{:70}|\n"
    header = componentformat.format("Name", "Description")
    line = "-" * (len(header) - 1) + "\n"
    help_string += line + header + line
    for package, d in firedrake_apps.items():
        help_string += componentformat.format(package, d[0])
    help_string += line
    print(help_string)
    sys.exit(0)


def sniff_compiler(exe):
    """Obtain the correct compiler class by calling the compiler executable.
    Attempt to determine the compiler version number.

        :arg cpp: If set to True will use the C++ compiler rather than
            the C compiler to determine the version number.
    :arg exe: String with name or path to compiler executable
    :returns: A compiler class
    """
    # NAME
    try:
        output = subprocess.run(
            [exe, "--version"],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            check=True,
            encoding="utf-8"
        ).stdout
    except (subprocess.CalledProcessError, UnicodeDecodeError):
        output = ""
    # Find the name of the compiler family
    if output.startswith("gcc") or output.startswith("g++") or output.startswith("GNU"):
        name = "GNU"
    elif output.startswith("clang"):
        name = "clang"
    elif output.startswith("Apple LLVM") or output.startswith("Apple clang"):
        name = "clang"
    elif output.startswith("icc"):
        name = "Intel"
    elif "Cray" in output.split("\n")[0]:
        # Cray is more awkward eg:
        # Cray clang version 11.0.4  (<some_hash>)
        # gcc (GCC) 9.3.0 20200312 (Cray Inc.)
        name = "Cray"
    else:
        name = "unknown"
    # VERSION
    version = None
    for dumpstring in ["-dumpfullversion", "-dumpversion"]:
        try:
            output = subprocess.run(
                [exe, dumpstring],
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                check=True,
                encoding="utf-8"
            ).stdout
            version = Version(output)
            break
        except (subprocess.CalledProcessError, UnicodeDecodeError, InvalidVersion):
            continue
    return name, version


# Temporary catch for tigerlake processors with gcc/g++/gfortran 9.3
if platform.system() == "Linux" and mode == "install":
    # Get the cpu information
    try:
        cpuinfostring = subprocess.run(
            ["lscpu"],
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            check=True,
            encoding="utf-8"
        ).stdout
    except (subprocess.CalledProcessError, UnicodeDecodeError):
        output = ""
    # Check if it is a tigerlake processor ie: "i3-11", "i5-11", "i7-11"
    if any(bad_cpu in cpuinfostring for bad_cpu in ["i3-11", "i5-11", "i7-11"]):
        # Get the compiler names and versions
        try:
            cc = sniff_compiler(args.mpicc or "gcc")
            cxx = sniff_compiler(args.mpicxx or "g++")
            fortran = sniff_compiler(args.mpif90 or "gfortran")
            # Check if the combination is a problem
            if any(map(lambda compiler: compiler[0] == "GNU" and compiler[1] < Version("9.4"), [cc, cxx, fortran])):
                log.info(
                    "\nUsing a Tiger Lake (or newer) CPU with gcc, g++ and gfortran version<=9.3 is not possible."
                    " Please install at least gcc, g++ and gfortran version 9.4.0 or later"
                )
                sys.exit(1)
        except (subprocess.CalledProcessError, FileNotFoundError):
            log.info(
                "\nFailed to sniff compiler version, perhaps there is no compiler installed.\n"
                "You can ignore this if you are not using a Tiger Lake (or newer) CPU with gcc, g++ or gfortran version<=9.3\n"
            )


@atexit.register
def print_log_location():
    log.info("\n\n%s log saved in %s" % (mode.capitalize(), logfile))


class directory(object):
    """Context manager that executes body in a given directory"""
    def __init__(self, dir):
        self.dir = os.path.abspath(dir)

    def __enter__(self):
        self.olddir = os.path.abspath(os.getcwd())
        log.debug("Old path '%s'" % self.olddir)
        log.debug("Pushing path '%s'" % self.dir)
        os.chdir(self.dir)

    def __exit__(self, *args):
        log.debug("Popping path '%s'" % self.dir)
        os.chdir(self.olddir)
        log.debug("New path '%s'" % self.olddir)


class environment(object):
    def __init__(self, **env):
        self.old = os.environ.copy()
        self.new = env

    def __enter__(self):
        os.environ.update(self.new)

    def __exit__(self, *args):
        os.environ = self.old


options = config["options"]

# Apply short cut package names. pyadjoint shoud no longer turn up the packages list because it is a hard dependency.
options["packages"] = [firedrake_apps.get(p, (None, p))[1] for p in options["packages"] if not p.endswith("pyadjoint")]


# Record of obsolete packages which --clean should remove from old installs.
old_git_packages = ["dolfin-adjoint", "libadjoint"]

if mode == "install":
    firedrake_env = os.path.abspath(args.venv_name)
else:
    firedrake_env = os.environ["VIRTUAL_ENV"]

if "cache_dir" not in config["options"] or not config["options"]["cache_dir"]:
    config["options"]["cache_dir"] = os.path.join(firedrake_env, ".cache")

# venv install
python = "%s/bin/python" % firedrake_env
# Use the pip from the venv
pip = [python, "-m", "pip"]
pipinstall = pip + ["install",
                    "--no-build-isolation",
                    "--no-binary", ",".join(wheel_blacklist)]

if mode == "install":
    # "petsc4py" is in `branches` when we are explicitly asking for a petsc4py
    # branch that lives in firedrake-project/petsc4py. This happens when we do:
    # python3 firedrake-install --doi some_old_zenodo_doi ...
    # or:
    # python3 firedrake-install --package-branch petsc4py some_old_branch ...
    use_petsc4py_in_petsc = "petsc4py" not in branches
    use_slepc4py_in_slepc = "slepc4py" not in branches
else:
    use_petsc4py_in_petsc = True
    use_slepc4py_in_slepc = True


# This context manager should be used whenever arguments should be temporarily added to pipinstall.
class pipargs(object):
    def __init__(self, *args):
        self.args = args

    def __enter__(self):
        self.save = pipinstall.copy()
        pipinstall.extend(self.args)

    def __exit__(self, *args, **kwargs):
        global pipinstall
        pipinstall = self.save


def check_call(arguments):
    try:
        log.debug("Running command '%s'", " ".join(arguments))
        log.debug(subprocess.check_output(arguments, stderr=subprocess.STDOUT, env=os.environ).decode())
    except subprocess.CalledProcessError as e:
        log.debug(e.output.decode())
        raise


def check_output(args):
    try:
        log.debug("Running command '%s'", " ".join(args))
        return subprocess.check_output(args, stderr=subprocess.STDOUT, env=os.environ).decode()
    except subprocess.CalledProcessError as e:
        log.debug(e.output.decode())
        raise


if "PYTHONPATH" in os.environ and not args.honour_pythonpath:
    quit("""The PYTHONPATH environment variable is set. This is probably an error.
If you really want to use your own Python packages, please run again with the
--honour-pythonpath option.
""")


def brew_gcc_libdir():
    brew_gcc_prefix = check_output(["brew", "--prefix", "gcc"])[:-1]
    brew_gcc_info = json.loads(check_output(["brew", "info", "--json", "gcc"])[:-1])
    gcc_major_version = brew_gcc_info[0]["installed"][0]["version"].split(".")[0]
    return brew_gcc_prefix + "/lib/gcc/" + gcc_major_version


# See .github/workflows/docker.yml to see the PETSc options used on CI.
# Edit these as appropriate if PETSc external packages are added or removed.
def get_minimal_petsc_packages():
    pkgs = set()
    # File format
    pkgs.add("hdf5")
    # Parallel mesh partitioner
    pkgs.add("ptscotch")
    # Sparse direct solver
    pkgs.add("scalapack")  # Needed for mumps
    pkgs.add("mumps")
    return pkgs


def get_petsc_options(minimal=False):
    # The logic in this function is getting out of hand...
    petsc_options = {"--with-fortran-bindings=0",
                     "--with-debugging=0",
                     "--with-shared-libraries=1",
                     "--with-c2html=0",
                     "--with-strict-petscerrorcode",
                     # Parser generator
                     "--download-bison"}
    for pkg in get_minimal_petsc_packages():
        petsc_options.add("--download-" + pkg)
    if osname == "Darwin":
        petsc_options.add("--with-x=0")
        # These three lines are used to inspect the MacOS Command Line Tools (CLT) version
        cmd = ["pkgutil", "--pkg-info=com.apple.pkg.CLTools_Executables"]
        output = subprocess.run(cmd, stdout=subprocess.PIPE, encoding="UTF-8")
        clt = {k.strip(): v.strip() for k, v in [line.split(':') for line in output.stdout.split('\n') if line]}
    elif osname == "Linux":
        # PETSc requires cmake version 3.18.1 or higher.
        petsc_options.add("--download-cmake")

    if (not options["minimal_petsc"]) and (not minimal):
        # Another sparse direct solver
        petsc_options.add("--download-superlu_dist")
        petsc_options.add("--with-zlib")
        # File formats
        petsc_options.add("--download-netcdf")
        petsc_options.add("--download-pnetcdf")
        # Sparse direct solvers
        petsc_options.add("--download-suitesparse")
        petsc_options.add("--download-pastix")
        # Required by pastix
        petsc_options.update({"--download-hwloc", "--download-netlib-lapack", "--with-netlib-lapack-c-bindings"})
        if osname == "Darwin":
            petsc_options.add("--download-hwloc-configure-arguments=--disable-opencl")
        # Serial mesh partitioner
        petsc_options.add("--download-metis")
        if options.get("with_parmetis", None):
            # Non-free license.
            petsc_options.add("--download-parmetis")

    if options["petsc_int_type"] != "int32":
        petsc_options.add("--with-64-bit-indices")

    if options["complex"]:
        petsc_options.add('--with-scalar-type=complex')
    else:
        # AMG
        petsc_options.add("--download-hypre")

    if options.get("mpiexec") is not None:
        petsc_options.add("--with-mpiexec={}".format(options["mpiexec"]))
        petsc_options.add("--with-cc={}".format(options["mpicc"]))
        petsc_options.add("--with-cxx={}".format(options["mpicxx"]))
        petsc_options.add("--with-fc={}".format(options["mpif90"]))
    else:
        # Download mpich if the user does not tell us about an MPI.
        petsc_options.add("--download-mpich")
        if osname == "Darwin":
            petsc_options.add("--download-mpich-configure-arguments=--disable-opencl")

    if options.get("with_blas") == "download":
        # Download openblas if the user specifies.
        petsc_options.add("--download-openblas")
        petsc_options.add("--download-openblas-make-options='USE_THREAD=0 USE_LOCKING=1 USE_OPENMP=0'")
        if osname == "Darwin":
            petsc_options.add("--CFLAGS=-Wno-implicit-function-declaration")
            if Version(clt["version"]) >= Version("15"):
                # CLT >= 15 requires legacy linking behaviour (-ld_classic flag)
                # The -dead_strip_dylibs flag is a workaround for MUMPS issues
                # on macOS with parallel solvers. Its necessity is unclear, so a
                # review in the future may be required.
                petsc_options.add("--LDFLAGS=-Wl,-ld_classic,-dead_strip_dylibs")
            if Version(clt["version"]) >= Version("16"):
                # Should revisit this flag when the the issue https://gitlab.com/petsc/petsc/-/issues/1692
                # is resolved.
                petsc_options.add("--with-cxx-dialect=17")
    elif options.get("with_blas") is not None:
        petsc_options.add("--with-blaslapack-dir={}".format(options["with_blas"]))
        if osname == "Darwin":
            petsc_options.add("--CFLAGS=-I{}/include -Wno-implicit-function-declaration".format(options["with_blas"]))
            if Version(clt["version"]) >= Version("15"):
                # CLT >= 15 requires legacy linking behaviour (-ld_classic flag)
                # The -dead_strip_dylibs flag is a workaround for MUMPS issues
                # on macOS with parallel solvers. Its necessity is unclear, so a
                # review in the future may be required.
                petsc_options.add("--LDFLAGS=-Wl,-ld_classic,-dead_strip_dylibs,-rpath,{0}/lib -L{0}/lib".format(options["with_blas"]))
        else:
            petsc_options.add("--CFLAGS=-I{}/include".format(options["with_blas"]))
            petsc_options.add("--LDFLAGS=-Wl,-rpath,{0}/lib -L{0}/lib".format(options["with_blas"]))
    elif osname == "Darwin":
        brew_blas_prefix = check_output(["brew", "--prefix", "openblas"])[:-1]
        petsc_options.add("--with-blaslapack-dir={}".format(brew_blas_prefix))
        petsc_options.add("--CFLAGS=-I{}/include -Wno-implicit-function-declaration".format(brew_blas_prefix))
        if Version(clt["version"]) >= Version("15"):
            # CLT >= 15 requires legacy linking behaviour (-ld_classic flag)
            # The -dead_strip_dylibs flag is a workaround for MUMPS issues
            # on macOS with parallel solvers. Its necessity is unclear, so a
            # review in the future may be required.
            petsc_options.add("--LDFLAGS=-Wl,-ld_classic,-dead_strip_dylibs,-rpath,{0}/lib -L{0}/lib -L{1}".format(brew_blas_prefix, brew_gcc_libdir()))
        else:
            petsc_options.add("--LDFLAGS=-Wl,-rpath,{0}/lib -L{0}/lib -L{1}".format(brew_blas_prefix, brew_gcc_libdir()))

    if not minimal:
        for option in shlex.split(os.environ.get("PETSC_CONFIGURE_OPTIONS", "")):
            if option.startswith("--with-hdf5-dir"):
                petsc_options.discard("--download-hdf5")
                os.environ["HDF5_DIR"] = option.replace("--with-hdf5-dir=", "")
            petsc_options.add(option)
    if "HDF5_DIR" in os.environ and "--download-hdf5" in petsc_options:
        del os.environ["HDF5_DIR"]
        log.info("\nWarning: HDF5_DIR environment variable set, but ignored; "
                 "use PETSC_CONFIGURE_OPTIONS=\"--with-hdf5-dir=$HDF5_DIR\" "
                 "to have PETSc built against $HDF5_DIR.")
    return list(petsc_options)


def get_petsc_config():
    petsc_dir, petsc_arch = get_petsc_dir()
    with open(os.path.join(petsc_dir, petsc_arch, "include", "petscconfiginfo.h")) as fh:
        for line in fh.readlines():
            if line.startswith("static const char *petscconfigureoptions"):
                info = line.split("=", maxsplit=1)[1].strip(' ";\n').split('--')[1:]
                break
    return ["--" + arg for arg in info]


if mode == "update" and petsc_int_type_changed:
    log.warning("""Force rebuilding all packages because PETSc int type changed""")

if mode == "update" and petsc_complex_type_changed:
    log.warning("""Force rebuilding all packages because PETSc scalar type changed between real and complex""")


def brew_install(name, options=None):
    arguments = [name]
    if options:
        arguments = options + arguments
    if args.verbose:
        arguments = ["--verbose"] + arguments
    check_call(["brew", "install"] + arguments)


def apt_check(name):
    log.info("Checking for presence of package %s..." % name)
    # Note that subprocess return codes have the opposite logical
    # meanings to those of Python variables.
    try:
        check_call(["dpkg-query", "-s", name])
        log.info("  installed.")
        return True
    except subprocess.CalledProcessError:
        log.info("  missing.")
        return False


def apt_install(names):
    log.info("Installing missing packages: %s." % ", ".join(names))
    if sys.stdin.isatty():
        subprocess.check_call(["sudo", "apt-get", "install"] + names)
    else:
        log.info("Non-interactive stdin detected; installing without prompts")
        subprocess.check_call(["sudo", "apt-get", "-y", "install"] + names)


def split_requirements_url(url):
    name = url.split(".git")[0].split("#")[0].split("/")[-1]
    spliturl = url.split("://")[1].split("#")[0].split("@")
    try:
        plain_url, branch = spliturl
    except ValueError:
        plain_url = spliturl[0]
        branch = "master"
    return name, plain_url, branch


def git_url(plain_url, protocol):
    if protocol == "ssh":
        return "git@%s:%s" % tuple(plain_url.split("/", 1))
    elif protocol == "https":
        return "https://%s" % plain_url
    else:
        raise ValueError("Unknown git protocol: %s" % protocol)


def git_clone(url):
    name, plain_url, branch = split_requirements_url(url)
    if name == "petsc" and args.honour_petsc_dir:
        log.info("Using existing PETSc installation\n")
        return name
    elif name == "petsc4py" and use_petsc4py_in_petsc:
        log.info("Not cloning petsc4py from firedrake-project repo: using petsc4py in PETSc source tree.\n")
        return name
    log.info("Cloning %s\n" % name)
    branch = branches.get(name.lower(), branch)
    try:
        if options["disable_ssh"]:
            raise InstallError("Skipping ssh clone because --disable-ssh")
        # note: so far only loopy requires submodule
        check_call(["git", "clone", "-q", "--recursive", git_url(plain_url, "ssh")])
        log.info("Successfully cloned repository %s" % name)
    except (subprocess.CalledProcessError, InstallError):
        if not options["disable_ssh"]:
            log.warning("Failed to clone %s using ssh, falling back to https." % name)
        try:
            check_call(["git", "clone", "-q", "--recursive", git_url(plain_url, "https")])
            log.info("Successfully cloned repository %s." % name)
        except subprocess.CalledProcessError:
            log.error("Failed to clone %s branch %s." % (name, branch))
            raise
    with directory(name):
        try:
            log.info("Checking out branch %s" % branch)
            check_call(["git", "checkout", "-q", branch])
            log.info("Successfully checked out branch %s" % branch)
        except subprocess.CalledProcessError:
            log.error("Failed to check out branch %s" % branch)
            raise
        try:
            log.info("Updating submodules.")
            check_call(["git", "submodule", "update", "--recursive"])
            log.info("Successfully updated submodules.")
        except subprocess.CalledProcessError:
            log.error("Failed to update submodules.")
            raise
    return name


def list_cloned_dependencies(name):
    log.info("Finding dependencies of %s\n" % name)
    deps = OrderedDict()
    try:
        for dep in open(name + "/requirements-git.txt", "r"):
            name = split_requirements_url(dep.strip())[0]
            deps[name] = dep.strip()
    except IOError:
        pass
    return deps


def clone_dependencies(name):
    log.info("Cloning the dependencies of %s" % name)
    deps = []
    try:
        for dep in open(name + "/requirements-git.txt", "r"):
            deps.append(git_clone(dep.strip()))
    except IOError:
        pass
    return deps


def git_update(name, url=None):
    # Update the named git repo and return true if the current branch actually changed.
    log.info("Updating the git repository for %s" % name)
    with directory(name):
        git_sha = check_output(["git", "rev-parse", "HEAD"])
        # Ensure remotes get updated if and when we move repositories.
        if url:
            _, plain_url, branch = split_requirements_url(url)
            current_url = check_output(["git", "remote", "-v"]).split()[1]
            current_branch = check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]).strip()
            protocol = "https" if current_url.startswith("https") else "ssh"
            new_url = git_url(plain_url, protocol)
            # Ensure we only change from bitbucket to github and not the reverse.
            if (new_url != current_url
                and ("bitbucket.org" in current_url)
                and ("github.com/firedrakeproject" in plain_url
                     or "github.com/dolfin-adjoint" in plain_url)):
                log.info("Updating git remote for %s" % name)
                check_call(["git", "remote", "set-url", "origin", new_url])
            # Ensure we only switch loopy branch if loopy is on firedrake and not on a feature branch.
            elif name == "loopy" and current_branch != branch and current_branch == "firedrake":
                log.info("Updating loopy branch to main")
                check_call(["git", "checkout", "-q", branch])

        check_call(["git", "pull", "--recurse-submodules"])
        git_sha_new = check_output(["git", "rev-parse", "HEAD"])
    return git_sha != git_sha_new


def run_pip(args):
    check_call(pip + args)


def run_pip_install(pipargs):
    # Make pip verbose when logging, so we see what the
    # subprocesses wrote out.
    # Particularly important for debugging petsc fails.
    with environment(**blas):
        pipargs = ["-v"] + pipargs
        check_call(pipinstall + pipargs)


def get_requirements(reqfname):
    with open(reqfname, "r") as f:
        reqs = f.readlines()
    return [req.strip() for req in reqs]


def run_pip_install_wrap(reqs, parallel_compiler_env):
    for req in reqs:
        package_name = re.split('<|>|=', req)[0]
        if package_name in parallel_packages:
            with environment(**parallel_compiler_env):
                run_pip_install([req])
        else:
            run_pip_install([req])


def pip_requirements(package, parallel_compiler_env):
    log.info("Installing pip dependencies for %s" % package)
    if os.path.isfile("%s/requirements-ext.txt" % package):
        reqs = get_requirements("%s/requirements-ext.txt" % package)
        run_pip_install_wrap(reqs, parallel_compiler_env)
    elif os.path.isfile("%s/requirements.txt" % package):
        if package == "COFFEE":
            # FIXME: Horrible hack to work around
            # https://github.com/coin-or/pulp/issues/123
            run_pip_install(["--no-deps", "-r", "%s/requirements.txt" % package])
        elif package == "pyadjoint":
            reqs = get_requirements("%s/requirements.txt" % package)
            for req in reqs:
                try:
                    run_pip_install([req])
                except subprocess.CalledProcessError as e:
                    if req == "tensorflow":
                        log.debug("Skipping failing install of optional pyadjoint dependency tensorflow")
                    else:
                        raise e
        elif package in {"loopy", "icepack"}:
            # dependencies are installed in setup.py
            return
        else:
            reqs = get_requirements("%s/requirements.txt" % package)
            run_pip_install_wrap(reqs, parallel_compiler_env)
    else:
        log.info("No dependencies found. Skipping.")


def install(package):
    log.info("Installing %s" % package)
    if package == "petsc/":
        build_and_install_petsc()
    elif package == "slepc/":
        build_and_install_slepc()
    elif package == "petsc4py/":
        build_and_install_x4py("petsc4py")
    elif package == "slepc4py/":
        build_and_install_x4py("slepc4py")
    elif package == "h5py/":
        # h5py tries to build against ancient version of numpy additional flag avoids this
        run_pip_install(["--no-build-isolation", package])
    elif package == "loopy/" and osname == "Darwin":
        # On MacOS loopy won't build unless additional flags are passed to the compiler
        with environment(CFLAGS="-Wno-implicit-function-declaration"):
            run_pip_install(["-e", package])
    else:
        run_pip_install(["-e", package])


def clean(package):
    log.info("Cleaning %s" % package)
    with directory(package):
        with environment(**blas):
            # loopy needs knowledge of the system BLAS just to run setup.py
            if os.path.isfile("setup.py"):
                check_call([python, "setup.py", "clean"])
            else:  # PEP 518 build system
                if os.path.exists("build") and os.path.isdir("build"):
                    shutil.rmtree("build")


def pip_uninstall(package):
    log.info("Removing existing %s installations\n" % package)
    # Uninstalling something with pip is an absolute disaster.  We
    # have to use pip freeze to list all available packages "locally"
    # and keep on removing the one we want until it is gone from this
    # list!  Yes, pip will happily have two different versions of the
    # same package co-existing.  Moreover, depending on the phase of
    # the moon, the order in which they are uninstalled is not the
    # same as the order in which they appear on sys.path!
    again = True
    uninstalled = False
    i = 0
    while again:
        # List installed packages, "locally".  In a venv,
        # this just tells me packages in the venv, otherwise it
        # gives me everything.
        lines = check_output(pip + ["freeze", "-l"])
        again = False
        for line in lines.split("\n"):
            # Do we have a locally installed package?
            if line.startswith(package+'=='):
                # Uninstall it.
                run_pip(["uninstall", "-y", line.strip()])
                uninstalled = True
                # Go round again, because THERE MIGHT BE ANOTHER ONE!
                again = True
        i += 1
        if i > 10:
            raise InstallError("pip claims it uninstalled %s more than 10 times.  Something is probably broken.", package)
    return uninstalled


def get_petsc_dir():
    if args.honour_petsc_dir:
        return honour_petsc_dir_get_petsc_dir()
    else:
        petsc_dir = os.path.join(os.environ["VIRTUAL_ENV"], "src", "petsc")
        petsc_arch = "default"
        return petsc_dir, petsc_arch


def get_petsc4py_dir():
    if use_petsc4py_in_petsc:
        petsc_dir, _ = get_petsc_dir()
        # petsc4py is currently here:
        return os.path.join(petsc_dir, "src/binding/petsc4py/")
    else:
        return "petsc4py/"


def get_slepc_dir():
    petsc_dir, petsc_arch = get_petsc_dir()
    if args.honour_petsc_dir:
        try:
            slepc_dir = os.environ["SLEPC_DIR"]
        except KeyError:
            raise InstallError("Need to set SLEPC_DIR for --slepc with --honour-petsc-dir")
    else:
        slepc_dir = os.path.join(os.environ["VIRTUAL_ENV"], "src", "slepc")
    return slepc_dir, petsc_arch


def get_slepc4py_dir():
    if use_slepc4py_in_slepc:
        slepc_dir, _ = get_slepc_dir()
        return os.path.join(slepc_dir, "src/binding/slepc4py")
    else:
        return "slepc4py/"


def build_and_install_petsc():
    log.info("Depending on your platform, PETSc may take an hour or more to build!")
    assert not args.honour_petsc_dir  # Belt and braces security before we start clobbering things.
    with directory("petsc"):
        petsc_dir, petsc_arch = get_petsc_dir()
        check_call(["rm", "-rf", os.path.join(petsc_dir, petsc_arch)])
        petsc_options = get_petsc_options()
        # Set MACOSX_DEPLOYMENT_TARGET to the current macos version
        # to workaround issues configuring PETSc on macos.
        # Choosing 12.0 as the oldest MacOS version supported as of 2024-02-08
        macosx_deployment_target = os.environ.get("MACOSX_DEPLOYMENT_TARGET", "12.0")
        with environment(MACOSX_DEPLOYMENT_TARGET=macosx_deployment_target):
            check_call([python, "./configure", "PETSC_DIR={}".format(petsc_dir), "PETSC_ARCH={}".format(petsc_arch)] + petsc_options)
            check_call(["make", "PETSC_DIR={}".format(petsc_dir), "PETSC_ARCH={}".format(petsc_arch), "all"])
        if args.remove_build_files and not args.honour_petsc_dir:
            log.info("Cleaning up PETSc build artifacts")
            check_call(["rm", "-rf", os.path.join(petsc_dir, petsc_arch, "externalpackages")])
            check_call(["rm", "-rf", os.path.join(petsc_dir, "src", "docs")])
            tutorial_output = os.path.join(petsc_dir, "src", "**", "tutorials", "output", "*")
            test_output = os.path.join(petsc_dir, "src", "**", "tests", "output", "*")
            for deletefile in chain(iglob(tutorial_output, recursive=True), iglob(test_output, recursive=True)):
                check_call(["rm", "-f", deletefile])


def build_and_install_x4py(package="petsc4py"):
    if package == "petsc4py":
        x4py_dir = get_petsc4py_dir()
    elif package == "slepc4py":
        x4py_dir = get_slepc4py_dir()
    else:
        raise ValueError("build_and_install_x4py() should be called with 'petsc4py' or 'slepc4py' only")
    with environment(**blas):
        if args.honour_petsc_dir:
            log.info("Copying {} to virtualenv to install".format(package))
            x4py_copy = os.path.join(os.environ['VIRTUAL_ENV'], 'src', package)
            if not os.path.exists(x4py_copy):
                shutil.copytree(x4py_dir, x4py_copy, dirs_exist_ok=True)
            pipargs = ["-vvv", "--ignore-installed", x4py_copy]
            try:
                check_output(pipinstall + pipargs)
            except subprocess.CalledProcessError:
                with open(os.path.join(x4py_copy, "README.md")) as fh:
                    fh.write("# Unable to install {}\n".format(package))
                    fh.write("This is just a copy of the {} source for debugging\n".format(package))
                    fh.write("See `firedrake-install.log` for the error")
                raise
            else:
                shutil.rmtree(x4py_copy)
        else:
            pipargs = ["-vvv", "--ignore-installed", x4py_dir]
            check_output(pipinstall + pipargs)


def build_and_install_h5py():
    log.info("Installing h5py")
    # Clean up old downloads
    if os.path.exists("h5py-2.5.0"):
        log.info("Removing old h5py-2.5.0 source")
        shutil.rmtree("h5py-2.5.0")
    if os.path.exists("h5py.tar.gz"):
        log.info("Removing old h5py.tar.gz")
        os.remove("h5py.tar.gz")

    url = "git+https://github.com/firedrakeproject/h5py.git@firedrake#egg=h5py"
    if os.path.exists("h5py"):
        changed = False
        with directory("h5py"):
            # Rewrite old h5py/h5py remote to firedrakeproject/h5py remote.
            plain_url = split_requirements_url(url)[1]
            current_remote = check_output(["git", "remote", "-v"]).split()[1]
            proto = "https" if current_remote.startswith("https") else "ssh"
            new_remote = git_url(plain_url, proto)
            if new_remote != current_remote and "h5py/h5py.git" in current_remote:
                log.info("Updating git remote for h5py from %s to %s", current_remote, new_remote)
                check_call(["git", "remote", "set-url", "origin", new_remote])
                check_call(["git", "fetch", "-p", "origin"])
                check_call(["git", "checkout", "firedrake"])
                changed = True
        changed |= git_update("h5py")
        # If h5py installation fails for some reason, we end up having a
        # situation where h5py is not installed, but h5py directory is
        # up-to-date, so we need to handle this here.
        changed |= not is_installed("h5py")
    else:
        git_clone(url)
        changed = True

    petsc_dir, petsc_arch = get_petsc_dir()
    hdf5_dir = os.environ.get("HDF5_DIR", "%s/%s" % (petsc_dir, petsc_arch))
    if changed or args.rebuild:
        log.info("Linking h5py against HDF5_DIR=%s\n" % hdf5_dir)
        with environment(HDF5_DIR=hdf5_dir,
                         HDF5_MPI="ON",
                         **blas):
            # Only uninstall if things changed.
            pip_uninstall("h5py")
            # Pip installing from dirty directory is potentially unsafe.
            with directory("h5py/"):
                check_call(["git", "clean", "-fdx"])
            install("h5py/")
    else:
        log.info("No need to rebuild h5py")


def build_and_install_pythonocc():
    log.info("Installing pythonocc-core")
    url = "git+https://github.com/tpaviot/pythonocc-core.git@595b0a4e8e60e8d6011bea0cdb54ac878efcfcd2"
    if not os.path.exists("pythonocc-core"):
        git_clone(url)
        with directory("pythonocc-core"):
            check_call(["git", "reset", "--hard"])
            check_call(["git", "clean", "-f", "-x", "-d"])
            check_call(["mkdir", "-p", "build"])
            with directory("build"):
                check_call(["cmake", "..",
                            "-DCMAKE_INSTALL_PREFIX=" + firedrake_env,
                            "-DPYTHON_EXECUTABLE=" + python,
                            "-DMPI_C_COMPILER=" + cc,
                            "-DMPI_CXX_COMPILER=" + cxx,
                            "-DMPI_Fortran_COMPILER=" + f90,
                            "-DCMAKE_Fortran_COMPILER=" + f90])
                check_call(["make", "install"])
    else:
        log.info("No need to rebuild pythonocc-core")


def build_and_install_torch():
    """Install PyTorch for a CPU or CUDA backend."""
    log.info("Installing PyTorch (backend: %s)" % args.torch)
    if osname == "Darwin" and args.torch == "cuda":
        raise InstallError("CUDA installation is not available on MacOS.")
    if sys.version_info >= (3, 10):
        run_pip_install(["typing_extensions"])
    extra_index_url = ["--extra-index-url", "https://download.pytorch.org/whl/cpu"] if args.torch == "cpu" else []
    run_pip_install(["torch"] + extra_index_url)


def build_and_install_jax():
    """Install JAX for a CPU or CUDA backend."""
    log.info("Installing JAX (backend: %s)" % args.jax)
    version_name = "jax" if args.jax == "cpu" else "jax[cuda12]"
    run_pip_install([version_name] + ["jaxlib"] + ["ml_dtypes"] + ["opt_einsum"])


def build_and_install_slepc():
    petsc_dir, petsc_arch = get_petsc_dir()
    slepc_dir, slepc_arch = get_slepc_dir()
    assert petsc_arch == slepc_arch

    slepc_changed = False
    if args.honour_petsc_dir:
        log.info("Using installed SLEPc from %s/%s", slepc_dir, petsc_arch)
    else:
        log.info("Installing SLEPc.")
        url = "git+https://github.com/firedrakeproject/slepc.git@firedrake"
        if os.path.exists("slepc"):
            slepc_changed |= git_update("slepc")
        else:
            git_clone(url)
            slepc_changed = True
        if slepc_changed:
            with directory("slepc"):
                check_call(["rm", "-rf", os.path.join(slepc_dir, petsc_arch)])
                with environment(PETSC_DIR=petsc_dir,
                                 SLEPC_DIR=slepc_dir,
                                 PETSC_ARCH=petsc_arch):
                    check_call([python, "./configure"])
                    check_call(["make", "all"])
                    check_call(["make", "check"])
        else:
            log.info("No need to rebuild SLEPc")

    url = "git+https://github.com/firedrakeproject/slepc4py.git@firedrake"
    if not use_slepc4py_in_slepc:
        if os.path.exists("slepc4py"):
            with directory("slepc4py"):
                plain_url = split_requirements_url(url)[1]
                current_remote = check_output(["git", "remote", "-v"]).split()[1]
                proto = "https" if current_remote.startswith("https") else "ssh"
                new_remote = git_url(plain_url, proto)
                if new_remote != current_remote:
                    log.info("Updating git remote for slepc4py from %s to %s", current_remote, new_remote)
                    check_call(["git", "remote", "set-url", "origin", new_remote])
                    check_call(["git", "fetch", "-p", "origin"])
                    check_call(["git", "checkout", "firedrake"])
        else:
            log.info("Cloning slepc4py from %s" % url)
            git_clone(url)


def install_documentation_dependencies():
    """Just install the required dependencies. There are no provenance
    issues here so no need to record this in the configuration dict."""
    log.info("Installing documentation dependencies")
    run_pip(["install", "-U", "sphinx"])
    run_pip(["install", "-U", "sphinx-autobuild"])
    run_pip(["install", "-U", "sphinxcontrib-bibtex"])
    run_pip(["install", "-U", "sphinxcontrib-svg2pdfconverter"])
    run_pip(["install", "-U", "sphinxcontrib-jquery"])
    run_pip(["install", "-U", "bibtexparser"])
    run_pip(["install", "-U", "sphinxcontrib-youtube"])
    run_pip(["install", "-U", "numpydoc"])


def clean_obsolete_packages():
    dirs = os.listdir(".")
    for package in old_git_packages:
        pip_uninstall(package)
        if package in dirs:
            shutil.rmtree(package)


def quit(message):
    log.error(message)
    sys.exit(1)


def build_update_script():
    log.info("Creating firedrake-update script.")
    with open("firedrake/scripts/firedrake-install", "r") as f:
        update_script = f.read()

    try:
        os.mkdir("../bin")
    except OSError:
        pass
    with open("../bin/firedrake-update", "w") as f:
        f.write(update_script)
    check_call(["chmod", "a+x", "../bin/firedrake-update"])


if args.rebuild_script:
    os.chdir(os.path.dirname(os.path.realpath(__file__)) + ("/../.."))

    build_update_script()

    log.info("Successfully rebuilt firedrake-update.\n")

    log.info("To upgrade firedrake, run firedrake-update")
    sys.exit(0)


def create_compiler_env(cc, cxx, f90, mpihome):
    env = dict()
    if cc:
        env["MPICC"] = cc
        env["MPI_C_COMPILER"] = cc
        env["CC"] = cc
        # Work around homebrew adding /usr/local/lib to the library search path.
        if osname == "Darwin":
            env["CFLAGS"] = " ".join((
                os.environ.get("CFLAGS", ""),
                "-L" + os.path.join(*get_petsc_dir(), "lib"),
                "-I" + os.path.join(*get_petsc_dir(), "include")
            ))
    if cxx:
        env["MPICXX"] = cxx
        env["MPI_CXX_COMPILER"] = cxx
        env["CXX"] = cxx
    if f90:
        env["MPIF90"] = f90
        env["MPI_Fortran_COMPILER"] = f90
        env["F90"] = f90
    if mpihome:
        env["MPI_HOME"] = mpihome

    return env


if mode == "install" and args.show_petsc_configure_options:
    log.info("*********************************************")
    log.info("Would build PETSc with the following options:")
    log.info("*********************************************\n")
    petsc_options = get_petsc_options()
    log.info("\n".join(petsc_options))
    sys.exit(0)

if "PETSC_DIR" in os.environ and not args.honour_petsc_dir:
    quit("""The PETSC_DIR environment variable is set. This is probably an error.
If you really want to use your own PETSc build, please run again with the
--honour-petsc-dir option.
""")

if "PETSC_DIR" not in os.environ and args.honour_petsc_dir:
    quit("""The --honour-petsc-dir is set, but PETSC_DIR environment variable is
not defined. If you have compiled PETSc manually, set PETSC_DIR
(and optionally PETSC_ARCH) variables to point to the build directory.
""")

if "SLEPC_DIR" not in os.environ and args.honour_petsc_dir and options["slepc"]:
    quit("""If you use --honour-petsc-dir, you must also build SLEPc manually
and set the SLEPC_DIR environment variable appropriately""")

if "SLEPC_DIR" in os.environ and not args.honour_petsc_dir and options["slepc"]:
    quit("""The SLEPC_DIR environment variable is set.  If you want to use
your own SLEPc version, you must also build your own PETSc and run
with --honour-petsc-dir.""")

if platform.uname()[0] == "Darwin" and args.opencascade:
    quit("""Sorry, automatically installing opencascade on OSX hasn't been
implemented yet. (It's not a supported package in brew.)
Please contact us to get this working.""")

log.debug("Installer running with: %s" % sys.executable)
log.debug("Python version: %s" % sys.version)

log.debug("*** Current environment (output of 'env') ***")
log.debug(check_output(["env"]))
log.debug("\n\n")

if mode == "install" and args.honour_petsc_dir:
    minimal_packages = get_minimal_petsc_packages()
    minimal_packages.update({"blaslapack", "mathlib", "mpi"})
    current_packages = honour_petsc_dir_get_petsc_packages()
    log.debug("*****************************************************")
    log.debug("Specified PETSc was built with the following options:")
    log.debug("*****************************************************\n")
    log.debug("PETSC_DIR={}  PETSC_ARCH={}".format(*get_petsc_dir()))
    log.debug("\n".join(sorted(get_petsc_config())) + '\n')
    log.debug("The following external packages were detected and are used by PETSc:")
    log.debug(", ".join(sorted(current_packages)))
    log.debug("\n")
    if not minimal_packages < current_packages:
        log.error("*********************************************************************")
        log.error("Specified PETSc is missing the following packages:")
        log.error(", ".join(sorted(minimal_packages - current_packages)) + '\n')
        log.error("You must ensure that PETSc has been built with these minimal options:")
        log.error("*********************************************************************\n")
        log.error("\n".join(sorted(get_petsc_options(minimal=True))))
        log.error("")
        log.error('Use --show-petsc-configure-options as an argument to the firedrake-install script to display options if additional functionality is required.\n ')
        log.error("FATAL: Unable to install with specified PETSc")
        exit(1)

if mode == "install" or not args.update_script:
    # Check operating system.
    required_packages = None
    package_manager = None
    if osname == "Darwin":
        package_manager = "brew"
        required_packages = ["gcc",
                             "pkg-config",
                             "autoconf",
                             "make",
                             "automake",
                             "cmake",
                             "ninja",
                             "libtool",
                             "boost"]
        if args.with_blas is None and mode == "install":
            required_packages.append("openblas")
    elif osname == "Linux":
        package_manager = "apt-get"
        required_packages = ["build-essential",
                             "autoconf",
                             "automake",
                             "bison",  # for ptscotch
                             "flex",   # for ptscotch
                             "cmake",
                             "gfortran",
                             "pkg-config",
                             "git",
                             "libcurl4-openssl-dev",
                             "pkgconf",  # for p4est
                             "libtool",
                             "libxml2-dev",
                             "ninja-build",  # for meson/numpy
                             "python3-dev",
                             "python3-pip",
                             "python3-tk",
                             "python3-venv",
                             "zlib1g-dev",
                             "libboost-dev"]  # For ROL in pyadjoint
        if args.opencascade:
            required_packages.append("liboce-ocaf-dev")
            required_packages.append("swig")

        if args.with_blas is None and mode == "install":
            required_packages.append("libopenblas-dev")

    else:
        log.warning("You do not appear to be running Linux or MacOS. Please do not be surprised if this install fails.")

    if args.documentation_dependencies:
        required_packages.append("graphviz")

    if args.show_dependencies:
        log.info("\n************************************************")
        log.info("The following dependencies need to be installed:")
        log.info("************************************************\n")
        log.info("* A C and C++ compiler (for example gcc/g++ or clang), GNU make")
        log.info("* A Fortran compiler (for PETSc)")
        log.info("* Blas and Lapack")
        log.info("* Boost")
        log.info("* Git")
        log.info("* Python version >=3.6")
        log.info("* The Python headers")
        log.info("* autoconf, automake, libtool")
        log.info("* CMake")
        if osname == "Linux":
            log.info("* zlib, bison, flex")
        if package_manager:
            log.info("\n********************************************")
            log.info("The following %s packages are recommended:" % (package_manager))
            log.info("********************************************\n")
            log.info("\n".join(required_packages))
            log.info("\nThey will be installed by %s unless --no-package-manager is used." % (package_manager))
            if osname == "Darwin":
                log.info("\nIn addition, the command line tools will be installed using xcode")
        sys.exit(0)

    blas_location = args.with_blas

    if osname == "Darwin":
        try:
            check_call(["brew", "--version"])
        except subprocess.CalledProcessError:
            quit("Installing Firedrake on MacOS requires Homebrew. Please install "
                 "it using the instructions at http://brew.sh and then try again.")

        if options["package_manager"]:
            log.info("Installing command line tools...")
            try:
                check_call(["xcode-select", "--install"])
            except subprocess.CalledProcessError:
                # expected failure if already installed
                pass

            log.info("Checking brew doctor...")
            try:
                check_call(["brew", "doctor"])
            except subprocess.CalledProcessError:
                pass
            log.info("Installing required packages via homebrew...")
            # Ensure a fortran compiler is available
            installed_packages = dict((p["name"], p)
                                      for p in json.loads(check_output(["brew", "info", "--installed", "--json"])))
            # Handle homebrew migration of default python == python3
            if installed_packages.get("python", {}).get("oldname") == "python3":
                installed_packages["python3"] = installed_packages["python"]
            for package in required_packages:
                if package in installed_packages:
                    log.info("Required package '{0}' is already installed via Homebrew.".format(package))
                    log.debug(pprint.pformat(installed_packages[package]))
                else:
                    log.info("Installing required package '{0}' via Homebrew.".format(package))
                    brew_install(package)
        else:
            log.info("Xcode and homebrew installation disabled. Proceeding on the rash assumption that packaged dependencies are in place.")
            log.info("Please use the --show-dependencies for more information.")

        # Unconditionally require openblas ("brew" is required above)
        if args.with_blas is None and mode == "install":
            config["options"]["with_blas"] = check_output(["brew", "--prefix", "openblas"])[:-1]
            blas_location = config["options"]["with_blas"]
            if not os.path.exists(blas_location):
                raise InstallError("Automatic determination of BLAS looked for openblas in %s but didn't find it\n"
                                   "Please provide location of a BLAS library with --with-blas (or use --with-blas=download)" % blas_location)
    elif osname == "Linux":
        # Check for apt.
        try:
            if not options["package_manager"]:
                raise InstallError
            check_call(["apt-get", "--version"])

            missing_packages = [p for p in required_packages if not apt_check(p)]
            if missing_packages:
                apt_install(missing_packages)

        except (subprocess.CalledProcessError, InstallError):
            log.info("apt-get not found or disabled. Proceeding on the rash assumption that your compiled dependencies are in place.")
            log.info("Please use the --show-dependencies for more information.")
else:
    blas_location = config["options"].get("with_blas")

if mode == "install":
    if os.path.exists(firedrake_env):
        log.warning("Specified venv '%s' already exists", firedrake_env)
        quit("Can't install into existing venv '%s'" % firedrake_env)

    log.info("Creating firedrake venv in '%s'." % firedrake_env)
    # Debian's Python3 is screwed, they don't ship ensurepip as part
    # of the base python package, so the default python -m venv
    # doesn't work.  Moreover, they have spiked the file such that it
    # calls sys.exit, which will kill any attempts to create a venv
    # with pip.
    try:
        import ensurepip        # noqa: F401
        with_pip = True
    except ImportError:
        with_pip = False
    import venv
    venv.EnvBuilder(with_pip=with_pip).create(firedrake_env)
    if not with_pip:
        import urllib.request
        log.debug("ensurepip unavailable, bootstrapping pip using get-pip.py")
        urllib.request.urlretrieve("https://bootstrap.pypa.io/get-pip.py", filename="get-pip.py")
        check_call([python, "get-pip.py"])
        log.debug("bootstrapping pip succeeded")
        log.debug("Removing get-pip.py")
        os.remove("get-pip.py")

    # We haven't activated the venv so we need to manually set the environment.
    os.environ["VIRTUAL_ENV"] = firedrake_env

# Ensure pip, setuptools, hatchling and wheel are at the latest version.
run_pip(["install", "-U", "setuptools"])
run_pip(["install", "cffi"])
run_pip(["install", "-U", "hatch"])
run_pip(["install", "-U", "editables"])
run_pip(["install", "-U", "pip"])
run_pip(["install", "-U", "wheel"])
run_pip(["install", "-U", "scikit-build-core"])
run_pip(["install", "-U", "rtree>=1.2"])

# Extra numpy dependendencies, see
# https://github.com/numpy/numpy/blob/main/pyproject.toml
run_pip(["install", "-U", "meson-python>=0.15.0"])
run_pip(["install", "-U", "Cython>=3.0.6"])

run_pip(["install", "-U", "numpy"])

# Loopy has additional build dependencies
run_pip(["install", "-U", "scikit-build"])
run_pip(["install", "-U", "nanobind"])
run_pip(["install", "-U", "make"])
run_pip(["install", "-U", "pcpp"])

petsc_dir, petsc_arch = get_petsc_dir()
if options["slepc"]:
    slepc_dir, _ = get_slepc_dir()

if mode == "install":
    should_link = not args.honour_petsc_dir or options.get("mpicc") is not None

    # Set up the MPI wrappers
    for opt in ["mpicc", "mpicxx", "mpif90", "mpiexec"]:
        if options.get(opt):
            name = options[opt]
            src = shutil.which(name)
        else:
            src = os.path.join(petsc_dir, petsc_arch, "bin", opt)
        dest = os.path.join(firedrake_env, "bin", opt)
        # Remove old symlinks
        if os.path.lexists(dest):
            os.remove(dest)
        elif os.path.exists(dest):
            os.remove(dest)
        if should_link:
            log.debug("Creating a wrapper for %s from %s to %s" % (opt, src, dest))
            # use low-level os.open to ensure permissions (incl. exec.) are set according to umask
            fd = os.open(dest, os.O_CREAT | os.O_WRONLY)
            with os.fdopen(fd, "w") as f:
                f.write('#!/bin/bash' + os.linesep + src + ' "$@"')

cc = options["mpicc"]
cxx = options["mpicxx"]
f90 = options["mpif90"]
mpiexec = options["mpiexec"]
try:
    mpihome = options["mpihome"]
except KeyError:
    if cc:
        try:
            mpihome = os.environ["MPI_HOME"]
        except KeyError:
            raise InstallError("MPI compilers specified but mpihome is missing from "
                               "configuration. Please set MPI_HOME to continue.")
    else:
        mpihome = None
compiler_env = create_compiler_env(cc, cxx, f90, mpihome)
os.chdir(firedrake_env)

# Dict to store BLAS and OPENBLAS environment variables
blas = dict()
if blas_location == "download":
    blas_location = os.path.join(petsc_dir, petsc_arch)
if blas_location is not None:
    blas["BLAS"] = blas_location
    blas["OPENBLAS"] = blas_location
if osname == "Darwin":
    blas["LDFLAGS"] = "-L" + brew_gcc_libdir()

log.info("BLAS config is %s" % blas)

# Configuration is now complete, and we have a venv so we dump the
# config file in order to maximise the chance that firedrake-update
# can be run on a partial install.
config_location = os.path.join(firedrake_env, ".configuration.json")
json_output = json.dumps(config)
with open(config_location, "w") as f:
    f.write(json_output)
log.info("Configuration saved to " + config_location)


def is_installed(name):
    # Return if package `name` is installed.
    try:
        importlib.import_module(name)
    except ModuleNotFoundError:
        return False
    else:
        return True


if mode == "install":
    os.mkdir("src")
    os.chdir("src")

    if os.environ.get("FIREDRAKE_CI_TESTS", ""):
        # When building Firedrake in CI, don't clone Firedrake again.
        check_call(["ln", "-s", "../../firedrake", "firedrake"])
    else:
        git_clone("git+https://github.com/firedrakeproject/firedrake.git")

    # Build the update script as soon as possible so that installation
    # recovery can be attempted if required.
    build_update_script()

    # Pre-install requested packages
    if args.pip_packages is not None:
        for package in args.pip_packages:
            log.info("Pip installing %s to venv" % package)
            run_pip_install_wrap(package.split(), {})

    # Install VTK
    if not args.no_vtk:
        log.info("Pip installing VTK to venv")
        run_pip_install_wrap(["vtk>=9.0.1"], {})

    if sys.version_info[:2] >= (3, 9):
        # Also lazy-object-proxy
        run_pip(["install", "lazy-object-proxy==1.4.*"])

    packages = clone_dependencies("firedrake")
    packages = ["petsc4py"] + packages
    packages += ["firedrake"]

    for p in options["packages"]:
        name = git_clone(p)
        packages.extend(clone_dependencies(name))
        packages += [name]

    if args.honour_petsc_dir:
        try:
            packages.remove("petsc")
        except ValueError:
            log.warning("PETSc wasn't in dependencies list")
    else:
        # Need to install petsc first in order to resolve hdf5 dependency.
        with environment(**compiler_env):
            with pipargs("--no-deps"):
                try:
                    packages.remove("petsc")
                except ValueError:
                    log.warning("PETSc wasn't in dependencies list")
                install("petsc/")
        os.environ["PETSC_DIR"] = petsc_dir
        os.environ["PETSC_ARCH"] = petsc_arch

    # If petsc built mpich then we now need to set the compiler environment variables.
    if not compiler_env:
        compilerbin = os.path.join(petsc_dir, petsc_arch, "bin")
        cc = os.path.join(compilerbin, "mpicc")
        cxx = os.path.join(compilerbin, "mpicxx")
        f90 = os.path.join(compilerbin, "mpif90")
        mpiexec = os.path.join(compilerbin, "mpiexec")
        mpihome = os.path.join(compilerbin, "..")
        compiler_env = create_compiler_env(cc, cxx, f90, mpihome)

    # Make sure that we link against the right MPI and PETSc shared libraries
    link_env = {
        "LDFLAGS": (
            "-Wl,-rpath,{petsc_dir}/{petsc_arch}/lib -L{petsc_dir}/{petsc_arch}/lib"
            .format(petsc_dir=petsc_dir, petsc_arch=petsc_arch)
        ),
    }

    # Install mpi4py
    with environment(**compiler_env, **link_env):
        if args.mpi4py_version:
            run_pip_install(["--no-cache-dir", "mpi4py==" + args.mpi4py_version])
        else:
            run_pip_install(["--no-cache-dir", "mpi4py"])

    for p in packages:
        pip_requirements(p, compiler_env)

    # FIAT requires recursivenodes, but does not include a requirements.txt
    run_pip_install(["recursivenodes"])
    # Pyadjoint requires checkpoint-schedules, but does not include a requirements.txt
    run_pip_install(["checkpoint_schedules"])

    with environment(**compiler_env):
        build_and_install_h5py()

    if "loopy" in packages:
        # We do want to install loopy's dependencies.
        install("loopy/")
        packages.remove("loopy")

    with pipargs("--no-deps"):
        try:
            packages.remove("firedrake")
        except ValueError:
            pass

        for p in packages:
            if p in parallel_packages:
                with environment(**compiler_env, **link_env):
                    install(p+"/")
            else:
                install(p+"/")
            sys.path.append(os.getcwd() + "/" + p)

    with pipargs("--no-deps"), environment(**compiler_env, **link_env):
        install("firedrake/")

    packages.remove("petsc4py")

else:
    # Update mode
    os.chdir("src")

    if args.update_script:
        # Pull firedrake, rebuild update script, launch new script
        git_update("firedrake")
        build_update_script()
        os.execv(sys.executable, [sys.executable, "../bin/firedrake-update", "--no-update-script"] + sys.argv[1:])

    # Pre-install requested packages
    if args.pip_packages is not None:
        for package in args.pip_packages:
            log.info("Pip installing %s to venv" % package)
            run_pip_install(package.split())

    deps = OrderedDict()
    deps.update(list_cloned_dependencies("firedrake"))
    for p in options["packages"]:
        name = split_requirements_url(p)[0]
        deps.update(list_cloned_dependencies(name))
        deps[name] = p
    packages = ["petsc4py"] + list(deps.keys())
    packages += ["firedrake"]

    # update packages.
    if not args.honour_petsc_dir:
        update_petsc = git_update("petsc", deps["petsc"])
    else:
        update_petsc = False
    if os.path.exists(os.path.join(firedrake_env, "src/petsc4py")):
        clean("petsc4py/")
        shutil.move("petsc4py", "petsc4py_old")
        update_petsc4py = True
    else:
        # Even if petsc has been installed, petsc4py might not, in
        # which case we need to make sure that we install it.
        # This situation occurs, e.g., when `firedrake-install`
        # fails somewhere after successfully installing petsc,
        # and the user continues installation with `firedrake-update`.
        update_petsc4py = not is_installed("petsc4py")
    if os.path.exists(os.path.join(firedrake_env, "src/slepc4py")):
        clean("slepc4py")
        shutil.move("slepc4py", "slepc4py_old")

    try:
        packages.remove("petsc")
    except ValueError:
        log.warning("PETSc wasn't in dependencies list")
    packages.remove("petsc4py")

    if args.clean:
        clean_obsolete_packages()
        for package in packages:
            pip_uninstall(package)
        if args.rebuild:
            pip_uninstall("petsc4py")

    # Handle archived repositories
    archived_repos = [("PyOP2", "firedrake", "PyOP2"),
                      ("tsfc", "firedrake+fiat", "tsfc"),
                      ("FInAT", "fiat", "FInAT")]
    for src_repo, dest_repo, pip_pkg_name in archived_repos:
        archived_path = os.path.join(firedrake_env, "src", src_repo)
        if os.path.exists(archived_path):
            log.warning("%s has been moved into %s, renaming to %s_old and pip uninstalling.\n"
                        % (src_repo, dest_repo, src_repo))
            pip_uninstall(pip_pkg_name)
            new_path = os.path.join(firedrake_env, "src", "%s_old" % src_repo)
            os.rename(archived_path, new_path)

    # If there is a petsc package to remove, then we're an old installation which will need to rebuild PETSc.
    update_petsc = pip_uninstall("petsc") or update_petsc

    for p in packages:
        try:
            git_update(p, deps.get(p, None))
        except OSError as e:
            if e.errno == 2:
                log.warning("%s missing, cloning anew.\n" % p)
                git_clone(deps[p])
            else:
                raise

    with pipargs("--no-deps"):
        with environment(**compiler_env):
            # Only rebuild petsc if it has changed.
            if not args.honour_petsc_dir and (args.rebuild or update_petsc):
                clean("petsc/")
                install("petsc/")
        if not args.honour_petsc_dir:
            os.environ["PETSC_DIR"] = petsc_dir
            os.environ["PETSC_ARCH"] = petsc_arch

        # If petsc built mpich then we now need to set the compiler environment variables.
        if not compiler_env:
            compilerbin = os.path.join(petsc_dir, petsc_arch, "bin")
            cc = os.path.join(compilerbin, "mpicc")
            cxx = os.path.join(compilerbin, "mpicxx")
            f90 = os.path.join(compilerbin, "mpif90")
            mpiexec = os.path.join(compilerbin, "mpiexec")
            mpihome = os.path.join(compilerbin, "..")
            compiler_env = create_compiler_env(cc, cxx, f90, mpihome)

    # Make sure that we link against the right MPI and PETSc shared libraries
    link_env = {
        "LDFLAGS": (
            "-Wl,-rpath,{petsc_dir}/{petsc_arch}/lib -L{petsc_dir}/{petsc_arch}/lib"
            .format(petsc_dir=petsc_dir, petsc_arch=petsc_arch)
        ),
    }

    # Install mpi4py
    with environment(**compiler_env, **link_env):
        if args.mpi4py_version:
            run_pip_install(["--no-cache-dir", "mpi4py==" + args.mpi4py_version])
        else:
            run_pip_install(["--no-cache-dir", "mpi4py"])

    # update dependencies.
    for p in packages:
        pip_requirements(p, compiler_env)

    # FIAT requires recursivenodes, but does not include a requirements.txt
    run_pip_install(["recursivenodes"])
    # Pyadjoint requires checkpoint-schedules, but does not include a requirements.txt
    run_pip_install(["checkpoint_schedules"])

    with pipargs("--no-deps"):
        if args.rebuild or update_petsc or update_petsc4py:
            # If not honour_petsc_dir, we have a version of petsc that contains
            # petsc4py in its source tree (old petsc must have been updated
            # in the above).
            # If honour_petsc_dir, we can get here (e.g., with --rebuild flag)
            # without obtaining the latest petsc in the above:
            # thus, we must check if the user installed petsc contains petsc4py.
            if args.honour_petsc_dir:
                if not os.path.exists(get_petsc4py_dir()):
                    raise InstallError("""\npetsc4py not found in your installation of PETSc,
which indicates that your PETSc is relatively old.
Please consider updating your PETSc manually.
""")
            clean(get_petsc4py_dir())
            with environment(**compiler_env):
                install("petsc4py/")
        # Always rebuild h5py.
        with environment(**compiler_env):
            build_and_install_h5py()

    if "loopy" in packages:
        # We do want to install loopy's dependencies.
        clean("loopy")
        install("loopy/")
        packages.remove("loopy")

    try:
        log.info("Attempting to install symengine")
        run_pip_install(["symengine"])
    except subprocess.CalledProcessError:
        log.info("Symengine installation failed, this is NOT FATAL.")

    with pipargs("--no-deps"):
        try:
            packages.remove("firedrake")
        except ValueError:
            pass

        for p in packages:
            clean(p)
            if p in parallel_packages:
                with environment(**compiler_env, **link_env):
                    install(p+"/")
            else:
                install(p+"/")

        with pipargs("--no-deps"), environment(**compiler_env, **link_env):
            install("firedrake/")

        # Ensure pytest is at the latest version
        run_pip(["install", "-U", "pytest"])

with environment(**compiler_env):
    with pipargs("--no-deps"):
        if options["slepc"]:
            build_and_install_slepc()
            with environment(PETSC_DIR=petsc_dir,
                             SLEPC_DIR=slepc_dir,
                             PETSC_ARCH=petsc_arch):
                install("slepc4py/")

    if options["netgen"]:
        packages += ["ngsPETSc"]
        run_pip(["install", "-U", "ngsPETSc"])

with pipargs("--no-deps"):
    if options["opencascade"]:
        build_and_install_pythonocc()

with pipargs("--no-deps"):
    if options["torch"]:
        build_and_install_torch()

with pipargs("--no-deps"):
    if options["jax"]:
        build_and_install_jax()

if args.documentation_dependencies:
    install_documentation_dependencies()

if mode == "update":
    try:
        from firedrake.configuration import setup_cache_dirs
        setup_cache_dirs()
        log.info("Clearing just in time compilation caches.")
        from firedrake.tsfc_interface import clear_cache, TSFCKernel
        from pyop2.compilation import clear_cache as pyop2_clear_cache
        print('Removing cached TSFC kernels from %s' % TSFCKernel._cachedir)
        clear_cache()
        pyop2_clear_cache()
    except:                     # noqa: E722
        # Unconditional except in order to avoid upgrade script failures.
        log.error("Failed to clear caches. Try running firedrake-clean.")


os.chdir("../..")

if mode == "install":
    log.info("\n\nSuccessfully installed Firedrake.\n")

    log.info("\nFiredrake has been installed in a python venv. You activate it with:\n")
    log.info("  . %s/bin/activate\n" % firedrake_env)
    log.info("The venv can be deactivated by running:\n")
    log.info("  deactivate\n\n")
    log.info("To upgrade Firedrake activate the venv and run firedrake-update\n")
else:
    log.info("\n\nSuccessfully updated Firedrake.\n")