Use stable URLs and ?full_index=1 for all github patches (spack#29239)

The number of commit characters in patch files fetched from GitHub can change,
so we should use `full_index=1` to enforce full commit hashes (and a stable
patch `sha256`).

Similarly, URLs for branches like `master` don't give us stable patch files,
because branches are moving targets. Use specific tags or commits for those.

- [x] update all github patch URLs to use `full_index=1`
- [x] don't use `master` or other branches for patches
- [x] add an audit check and a test for `?full_index=1`

Co-authored-by: Todd Gamblin <tgamblin@llnl.gov>

lib/spack/spack/audit.py

@@ -41,11 +41,14 @@ def _search_duplicate_compilers(error_cls):
 
 from six.moves.urllib.request import urlopen
 
-try:
-    from collections.abc import Sequence  # novm
-except ImportError:
-    from collections import Sequence
+import llnl.util.lang
+from llnl.util.compat import Sequence
 
+import spack.config
+import spack.patch
+import spack.repo
+import spack.spec
+import spack.variant
 
 #: Map an audit tag to a list of callables implementing checks
 CALLBACKS = {}
@@ -180,7 +183,6 @@ def run_check(tag, **kwargs):
 @config_compiler
 def _search_duplicate_compilers(error_cls):
     """Report compilers with the same spec and two different definitions"""
-    import spack.config
     errors = []
 
     compilers = list(sorted(
@@ -217,8 +219,6 @@ def _search_duplicate_compilers(error_cls):
 @config_packages
 def _search_duplicate_specs_in_externals(error_cls):
     """Search for duplicate specs declared as externals"""
-    import spack.config
-
     errors, externals = [], collections.defaultdict(list)
     packages_yaml = spack.config.get('packages')
 
@@ -265,6 +265,7 @@ def _search_duplicate_specs_in_externals(error_cls):
     kwargs=('pkgs',)
 )
 
+
 #: Sanity checks on linting
 # This can take some time, so it's run separately from packages
 package_https_directives = AuditClass(
@@ -275,15 +276,40 @@ def _search_duplicate_specs_in_externals(error_cls):
 )
 
 
+@package_directives
+def _check_patch_urls(pkgs, error_cls):
+    """Ensure that patches fetched from GitHub have stable sha256 hashes."""
+    github_patch_url_re = (
+        r"^https?://github\.com/.+/.+/(?:commit|pull)/[a-fA-F0-9]*.(?:patch|diff)"
+    )
+
+    errors = []
+    for pkg_name in pkgs:
+        pkg = spack.repo.get(pkg_name)
+        for condition, patches in pkg.patches.items():
+            for patch in patches:
+                if not isinstance(patch, spack.patch.UrlPatch):
+                    continue
+
+                if not re.match(github_patch_url_re, patch.url):
+                    continue
+
+                full_index_arg = "?full_index=1"
+                if not patch.url.endswith(full_index_arg):
+                    errors.append(error_cls(
+                        "patch URL in package {0} must end with {1}".format(
+                            pkg.name, full_index_arg,
+                        ),
+                        [patch.url],
+                    ))
+
+    return errors
+
+
 @package_https_directives
 def _linting_package_file(pkgs, error_cls):
     """Check for correctness of links
     """
-    import llnl.util.lang
-
-    import spack.repo
-    import spack.spec
-
     errors = []
     for pkg_name in pkgs:
         pkg = spack.repo.get(pkg_name)
@@ -308,11 +334,6 @@ def _linting_package_file(pkgs, error_cls):
 @package_directives
 def _unknown_variants_in_directives(pkgs, error_cls):
     """Report unknown or wrong variants in directives for this package"""
-    import llnl.util.lang
-
-    import spack.repo
-    import spack.spec
-
     errors = []
     for pkg_name in pkgs:
         pkg = spack.repo.get(pkg_name)
@@ -367,9 +388,6 @@ def _unknown_variants_in_directives(pkgs, error_cls):
 @package_directives
 def _unknown_variants_in_dependencies(pkgs, error_cls):
     """Report unknown dependencies and wrong variants for dependencies"""
-    import spack.repo
-    import spack.spec
-
     errors = []
     for pkg_name in pkgs:
         pkg = spack.repo.get(pkg_name)
@@ -417,8 +435,6 @@ def _unknown_variants_in_dependencies(pkgs, error_cls):
 @package_directives
 def _version_constraints_are_satisfiable_by_some_version_in_repo(pkgs, error_cls):
     """Report if version constraints used in directives are not satisfiable"""
-    import spack.repo
-
     errors = []
     for pkg_name in pkgs:
         pkg = spack.repo.get(pkg_name)
@@ -455,7 +471,6 @@ def _version_constraints_are_satisfiable_by_some_version_in_repo(pkgs, error_cls
 
 
 def _analyze_variants_in_directive(pkg, constraint, directive, error_cls):
-    import spack.variant
     variant_exceptions = (
         spack.variant.InconsistentValidationError,
         spack.variant.MultipleValuesInExclusiveVariantError,

lib/spack/spack/test/audit.py

@@ -8,30 +8,30 @@
 import spack.config
 
 
-@pytest.mark.parametrize('packages,failing_check', [
+@pytest.mark.parametrize('packages,expected_error', [
     # A non existing variant is used in a conflict directive
     (['wrong-variant-in-conflicts'], 'PKG-DIRECTIVES'),
     # The package declares a non-existing dependency
     (['missing-dependency'], 'PKG-DIRECTIVES'),
     # The package use a non existing variant in a depends_on directive
     (['wrong-variant-in-depends-on'], 'PKG-DIRECTIVES'),
+    # This package has a GitHub patch URL without full_index=1
+    (['invalid-github-patch-url'], 'PKG-DIRECTIVES'),
     # This package has no issues
     (['mpileaks'], None),
     # This package has a conflict with a trigger which cannot constrain the constraint
     # Should not raise an error
     (['unconstrainable-conflict'], None),
 ])
-def test_package_audits(packages, failing_check, mock_packages):
+def test_package_audits(packages, expected_error, mock_packages):
     reports = spack.audit.run_group('packages', pkgs=packages)
 
-    for check, errors in reports:
-        # Check that we have errors only if there is an expected failure
-        # and that the tag matches our expectations
-        if bool(failing_check):
-            assert check == failing_check
-            assert errors
-        else:
-            assert not errors
+    # Check that errors were reported only for the expected failure
+    actual_errors = [check for check, errors in reports if errors]
+    if expected_error:
+        assert [expected_error] == actual_errors
+    else:
+        assert not actual_errors
 
 
 # Data used in the test below to audit the double definition of a compiler

var/spack/repos/builtin.mock/packages/invalid-github-patch-url/package.py

@@ -0,0 +1,20 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class InvalidGithubPatchUrl(Package):
+    """Package that has a GitHub patch URL that fails auditing."""
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/patch-1.0.tar.gz"
+
+    version('1.0', '0123456789abcdef0123456789abcdef')
+
+    patch(
+        'https://github.com/spack/spack/commit/cc76c0f5f9f8021cfb7423a226bd431c00d791ce.patch',
+        sha256='6057c3a8d50a23e93e5642be5a78df1e45d7de85446c2d7a63e3d0d88712b369',
+    )

var/spack/repos/builtin/packages/adios/package.py

@@ -106,8 +106,8 @@ class Adios(AutotoolsPackage):
     patch('zfp051.patch', when='@1.11.0:1.13.1')
 
     # Fix a bug in configure.ac that causes automake issues on RHEL 7.7
-    patch('https://github.com/ornladios/ADIOS/pull/207.patch', when='@1.12.0: +mpi',
-          sha256='01113e9efb929d71c28bf33cc8b7f215d85195ec700e99cb41164e2f8f830640')
+    patch('https://github.com/ornladios/ADIOS/pull/207.patch?full_index=1', when='@1.12.0: +mpi',
+          sha256='aea47e56013b57c2d5d36e23e0ae6010541c3333a84003784437768c2e350b05')
 
     def validate(self, spec):
         """Checks if incompatible variants have been activated at the same time

var/spack/repos/builtin/packages/adios2/package.py

@@ -131,8 +131,8 @@ class Adios2(CMakePackage):
 
     # Add missing include <memory>
     # https://github.com/ornladios/adios2/pull/2710
-    patch('https://github.com/ornladios/adios2/pull/2710.patch', when='@:2.7.1',
-          sha256='8d301e8232baf4049b547f22bd73774309662017a62dac36360d2965907062bf')
+    patch('https://github.com/ornladios/adios2/pull/2710.patch?full_index=1', when='@:2.7.1',
+          sha256='8221073d1b2f8944395a88a5d60a15c7370646b62f5fc6309867bbb6a8c2096c')
 
     @when('%fj')
     def patch(self):

var/spack/repos/builtin/packages/bart/package.py

@@ -19,8 +19,8 @@ class Bart(MakefilePackage, CudaPackage):
     version('0.5.00', sha256='30eedcda0f0ef3808157542e0d67df5be49ee41e4f41487af5c850632788f643')
 
     # patch to fix build with MKL
-    patch('https://github.com/mrirecon/bart/commit/b62ca4972d5ac41a44217a5c27123c15daae74db.patch',
-          sha256='8fd1be181da928448da750b32d45ee6dce7ba6af0424617c4f8d653cf3f05445',
+    patch('https://github.com/mrirecon/bart/commit/b62ca4972d5ac41a44217a5c27123c15daae74db.patch?full_index=1',
+          sha256='501ca985df2324fc02e32cc4ffb32ce5c235fe83a59f5b853eb78ad0f2d6629c',
           when='@0.5.00')
 
     # patch to fix Makefile for openblas and cuda

var/spack/repos/builtin/packages/boost/package.py

@@ -308,14 +308,14 @@ def libs(self):
 
     # Fix float128 support when building with CUDA and Cray compiler
     # See https://github.com/boostorg/config/pull/378
-    patch("https://github.com/boostorg/config/commit/fee1ad07968386b6d547f089311b7a2c1bf7fa55.patch",
-          sha256="3b159d65a0d3d2df2a21c6bf56ffaba943fce92d2d41d628b2c4d2e924e0f421",
+    patch("https://github.com/boostorg/config/commit/fee1ad07968386b6d547f089311b7a2c1bf7fa55.patch?full_index=1",
+          sha256="666eec8cfb0f71a87443ab27d179a9771bda32bcb8ff5e16afa3767f7b7f1e70",
           when="@:1.76%cce",
           level=2)
 
     # Fix building with Intel compilers
-    patch("https://github.com/bfgroup/b2/commit/23212066f0f20358db54568bb16b3fe1d76f88ce.patch",
-          sha256="93f4aad8f88d1437e50d95a2d066390ef3753b99ef5de24f7a46bc083bd6df06",
+    patch("https://github.com/bfgroup/b2/commit/23212066f0f20358db54568bb16b3fe1d76f88ce.patch?full_index=1",
+          sha256="4849671f9df4b8f3c962130d7f6d44eba3b20d113e84f9faade75e6469e90310",
           when="@1.77.0",
           working_dir="tools/build")
 

var/spack/repos/builtin/packages/claw/package.py

@@ -36,14 +36,14 @@ class Claw(CMakePackage):
 
     # Enable parsing of source files with calls to TRACEBACKQQ from the Intel
     # Fortran run-time library:
-    patch('https://github.com/claw-project/claw-compiler/commit/e9fe6dbd291454ce34dd58f21d102f7f1bdff874.patch',
-          sha256='82033a576966143f3b1fd66f4d5b5604704b615b3e08afa4901fc1c29caefbe2',
+    patch('https://github.com/claw-project/claw-compiler/commit/e9fe6dbd291454ce34dd58f21d102f7f1bdff874.patch?full_index=1',
+          sha256='262799fde57cb32f1514db22a7757e994bd8b97090ce0a5f55249fd56d0e5c29',
           when='@:2.0.2%intel')
 
     # Fix the dependency preprocessing for compilers that cannot use
     # redirection > to save file (cce is currently the only known case):
-    patch('https://github.com/claw-project/claw-compiler/commit/4d8bc7a794af3651b8b61501388fc00096b23a85.patch',
-          sha256='0a55110c67d7755741e1c86c2f71341286e7502a81ac29958ce80273e87bc8e1',
+    patch('https://github.com/claw-project/claw-compiler/commit/4d8bc7a794af3651b8b61501388fc00096b23a85.patch?full_index=1',
+          sha256='a20427456560070e284ff44edb658383b635042be91d2ffbe7aeb7afbd8f02bc',
           when='@2.0.2%cce')
 
     # Cache ANT dependencies in the stage directory.

var/spack/repos/builtin/packages/clfft/package.py

@@ -26,8 +26,8 @@ class Clfft(CMakePackage):
     # See https://github.com/spack/spack/pull/22303 for reference
     depends_on(Boost.with_default_variants, when='+client')
 
-    patch('https://github.com/clMathLibraries/clFFT/commit/eea7dbc888367b8dbea602ba539eb1a9cbc118d9.patch',
-          sha256='3148d5937077def301b30b913bc2437df869204fca1de4385ccd46e3b98b13aa', when='@2.12.2')
+    patch('https://github.com/clMathLibraries/clFFT/commit/eea7dbc888367b8dbea602ba539eb1a9cbc118d9.patch?full_index=1',
+          sha256='9ee126955f0bb9469c8302d8b08672d60a493d3373dbad06bdfda0b5c7893488', when='@2.12.2')
 
     root_cmakelists_dir = 'src'
 

var/spack/repos/builtin/packages/conduit/package.py

@@ -173,8 +173,8 @@ class Conduit(CMakePackage):
 
     # Add missing include for numeric_limits
     # https://github.com/LLNL/conduit/pull/773
-    patch('https://github.com/LLNL/conduit/pull/773.patch', when='@:0.7.2',
-          sha256='89d1829ad52f503f6179e43efddf998c239a95c14ca1f248463a3f61ad7d5cf7')
+    patch('https://github.com/LLNL/conduit/pull/773.patch?full_index=1', when='@:0.7.2',
+          sha256='784d74942a63acf698c31b39848b46b4b755bf06faa6aa6fb81be61783ec0c30')
 
     ###################################
     # build phases used by this package

var/spack/repos/builtin/packages/cp2k/package.py

@@ -206,8 +206,8 @@ class Cp2k(MakefilePackage, CudaPackage):
     conflicts('+cuda', when='cuda_arch=none', msg=cuda_msg)
 
     # Fix 2- and 3-center integral calls to libint
-    patch("https://github.com/cp2k/cp2k/commit/5eaf864ed2bd21fb1b05a9173bb77a815ad4deda.patch",
-          sha256="18e58ba8fdde5c507bece48ec064f7f2b80e59d1b7cfe6b7a639e5f64f84d43f",
+    patch("https://github.com/cp2k/cp2k/commit/5eaf864ed2bd21fb1b05a9173bb77a815ad4deda.patch?full_index=1",
+          sha256="3617abb877812c4b933f601438c70f95e21c6161bea177277b1d4125fd1c0bf9",
           when="@8.2")
 
     @property

var/spack/repos/builtin/packages/dealii/package.py

@@ -231,25 +231,25 @@ class Dealii(CMakePackage, CudaPackage):
 
     # Explicitly provide a destructor in BlockVector,
     # otherwise deal.II may fail to build with Intel compilers.
-    patch('https://github.com/dealii/dealii/commit/a89d90f9993ee9ad39e492af466b3595c06c3e25.patch',
-          sha256='4282b32e96f2f5d376eb34f3fddcc4615fcd99b40004cca784eb874288d1b31c',
+    patch('https://github.com/dealii/dealii/commit/a89d90f9993ee9ad39e492af466b3595c06c3e25.patch?full_index=1',
+          sha256='72304bc6c3fb4549cf53ed533a00311d12827d48817e2038efd3a8ef6c43d149',
           when='@9.0.1')
 
     # https://github.com/dealii/dealii/pull/7935
-    patch('https://github.com/dealii/dealii/commit/f8de8c5c28c715717bf8a086e94f071e0fe9deab.patch',
-          sha256='61f217744b70f352965be265d2f06e8c1276685e2944ca0a88b7297dd55755da',
+    patch('https://github.com/dealii/dealii/commit/f8de8c5c28c715717bf8a086e94f071e0fe9deab.patch?full_index=1',
+          sha256='4aba56b01d816ca950b1625f436840df253f145650e3a3eba51e7f2696ec7dc0',
           when='@9.0.1 ^boost@1.70.0:')
 
     # Fix TBB version check
     # https://github.com/dealii/dealii/pull/9208
-    patch('https://github.com/dealii/dealii/commit/80b13fe5a2eaefc77fa8c9266566fa8a2de91edf.patch',
-          sha256='6f876dc8eadafe2c4ec2a6673864fb451c6627ca80511b6e16f3c401946fdf33',
+    patch('https://github.com/dealii/dealii/commit/80b13fe5a2eaefc77fa8c9266566fa8a2de91edf.patch?full_index=1',
+          sha256='3da530766050a0cea80106684347055bdb78528a1869ce99e8fbf8fc83074fd0',
           when='@9.0.0:9.1.1')
 
     # Explicitly include a boost header, otherwise deal.II fails to compile
     # https://github.com/dealii/dealii/pull/11438
-    patch('https://github.com/dealii/dealii/commit/3b815e21c4bfd82c792ba80e4d90314c8bb9edc9.patch',
-          sha256='5f9f411ab9336bf49d8293b9936344bad6e1cf720955b9d8e8b29883593b0ed9',
+    patch('https://github.com/dealii/dealii/commit/3b815e21c4bfd82c792ba80e4d90314c8bb9edc9.patch?full_index=1',
+          sha256='90ae9ddefe77fffd297bba6b070ab68d07306d4ef525ee994e8c49cef68f76f3',
           when='@9.2.0 ^boost@1.72.0:')
 
     # Check for sufficiently modern versions

var/spack/repos/builtin/packages/eccodes/package.py

@@ -137,8 +137,8 @@ class Eccodes(CMakePackage):
     patch('cmake_install_rpath.patch', when='@:2.10')
 
     # Fix a bug preventing cmake from finding NetCDF:
-    patch('https://github.com/ecmwf/ecbuild/commit/3916c7d22575c45166fcc89edcbe02a6e9b81aa2.patch',
-          sha256='857454528b666c52eb36ef3aa5d40ae018981b44e129bb8df3c2d3d560e3fa03',
+    patch('https://github.com/ecmwf/ecbuild/commit/3916c7d22575c45166fcc89edcbe02a6e9b81aa2.patch?full_index=1',
+          sha256='9dcc4affaaa850d4b7247baa939d0f9ffedea132369f1afc3f248dbf720386c9',
           when='@:2.4.0+netcdf')
 
     @when('%nag+fortran')

var/spack/repos/builtin/packages/enzo/package.py

@@ -32,7 +32,7 @@ class Enzo(MakefilePackage):
     patch('for_aarch64.patch', when='target=aarch64:')
 
     # https://github.com/enzo-project/enzo-dev/pull/158
-    patch('https://github.com/enzo-project/enzo-dev/commit/0191ff5ad9ad2c7639d44823e84cd0115e7a2970.patch', sha256='01328a5f5fe72ac5af31661deb6891ea160264b67a470d6ce91b71b001845810', when='@2.6.1 ^hdf5@1.12.0:')
+    patch('https://github.com/enzo-project/enzo-dev/commit/0191ff5ad9ad2c7639d44823e84cd0115e7a2970.patch?full_index=1', sha256='f6db2fef04d3ffe4f05ef589d0593b2ab7ab6d63088abf9b76c7bacf835625c0', when='@2.6.1 ^hdf5@1.12.0:')
 
     def flag_handler(self, name, flags):
         if name == 'fflags':

var/spack/repos/builtin/packages/fdb/package.py

@@ -47,8 +47,8 @@ class Fdb(CMakePackage):
     patch('metkit_1.7.0.patch', when='@:5.7.10+tools^metkit@1.7.0:')
 
     # Download test data before running a test:
-    patch('https://github.com/ecmwf/fdb/commit/86e06b60f9a2d76a389a5f49bedd566d4c2ad2b2.patch',
-          sha256='e2254577e6d84a61d394eddcf42f894582f5daaf58d8962c609e41be0e3471b3',
+    patch('https://github.com/ecmwf/fdb/commit/86e06b60f9a2d76a389a5f49bedd566d4c2ad2b2.patch?full_index=1',
+          sha256='8b4bf3a473ec86fd4d7672faa7d74292dde443719299f2ba59a2c8501d6f0906',
           when='@5.7.1:5.7.10+tools')
 
     def cmake_args(self):

var/spack/repos/builtin/packages/ferret/package.py

@@ -36,8 +36,8 @@ class Ferret(Package):
     depends_on("curl")
 
     # Make Java dependency optional with older versions of Ferret
-    patch('https://github.com/NOAA-PMEL/Ferret/commit/c7eb70a0b17045c8ca7207d586bfea77a5340668.patch',
-          sha256='5bd581db4578c013faed375844b206fbe71f93fe9ce60f8f9f41d64abc6a5972',
+    patch('https://github.com/NOAA-PMEL/Ferret/commit/c7eb70a0b17045c8ca7207d586bfea77a5340668.patch?full_index=1',
+          sha256='6dd0a6b11c103b0097fba3da06d6e655da9770e8f568a15968d9b64a0f3c2315',
           level=1, working_dir='FERRET', when='@:6')
 
     resource(name='datasets',

var/spack/repos/builtin/packages/flatbuffers/package.py

@@ -41,9 +41,8 @@ class Flatbuffers(CMakePackage):
     # Silences false positive "-Wstringop-overflow" on GCC 10+
     # https://github.com/google/flatbuffers/issues/5950
     # Possibly affects earlier releases but I haven't tried to apply it.
-    patch('https://patch-diff.githubusercontent.com/raw/google/flatbuffers/pull/'
-          '6020.patch',
-          sha256='4a9a18abc776407f3f97e02c40f349cfb24fe7ddb41df952271d894777a31c88',
+    patch('https://github.com/google/flatbuffers/pull/6020.patch?full_index=1',
+          sha256='579cb6fa4430d4304b93c7a1df7e922f3c3ec614c445032877ad328c209d5462',
           when='@1.12.0:%gcc@10:')
 
     @run_after('install')

var/spack/repos/builtin/packages/flex/package.py

@@ -49,7 +49,9 @@ class Flex(AutotoolsPackage):
     # - https://github.com/spack/spack/issues/8152
     # - https://github.com/spack/spack/issues/6942
     # - https://github.com/westes/flex/issues/241
-    patch('https://github.com/westes/flex/commit/24fd0551333e7eded87b64dd36062da3df2f6380.patch', sha256='09c22e5c6fef327d3e48eb23f0d610dcd3a35ab9207f12e0f875701c677978d3', when='@2.6.4')
+    patch('https://github.com/westes/flex/commit/24fd0551333e7eded87b64dd36062da3df2f6380.patch?full_index=1',
+          sha256='f8b85a00849bfb58c9b68e177b369f1e060ed8758253ff8daa57a873eae7b7a5',
+          when='@2.6.4')
 
     @classmethod
     def determine_version(cls, exe):