Release Manager · Release Manager · commit cd2796561638 · 2023-12-17T16:51:22.000+01:00
gh-36774: `build/pkgs/nauty`: Update to 2.8.8; use VERSION in all `upstream_url`s.   https://pallini.di.uniroma1.it/changes24-28.txt Also, motivated by a mistake I made while preparing this PR (#36774 (comment)), and previously in #35380 (see #36781), and countless other times, I am changing all `upstream_url`s to use `VERSION` instead of the hardcoded version number. This makes it less error-prone to use `sage -package update SPKG NEW_VERSION` and `sage -package update-latest`. These commands now also display the URL from which they are downloading the tarball, and they warn if the `upstream_url` field does not contain the `VERSION` variable. (Exceptions: `ipykernel`, which is fixed in #36129, and `libbraiding`, which is fixed in #36781) Because some of the `upstream_url` need the version in a different format, a mechanism to refer to the components of a version is added: The strings `VERSION_MAJOR`, `VERSION_MINOR`, `VERSION_MICRO` can be used. For added readability, instead of the naked strings, also the notation `${VERSION_MAJOR}` etc. can be used. For example, `nauty`, which previously had to use a hardcoded version number, now uses the upstream URL pattern `upstream_url=https://pallini.di.uniroma1.it/nauty${VERSION_MAJOR}_${VER SION_MINOR}_${VERSION_MICRO}.tar.gz` The syntax `${VERSION_MAJOR}`, borrowed from Bourne shell, matches what sage-bootstrap already uses in https://github.com/sagemath/sage/blob/develop/.upstream.d/10-SAGE_SERVER    ### 📝 Checklist     - [x] The title is concise, informative, and self-explanatory. - [x] The description explains in detail what this PR is about. - [ ] I have linked a relevant issue or discussion. - [ ] I have created tests covering the changes. - [ ] I have updated the documentation accordingly. ### ⌛ Dependencies  - Depends on #36771 (merged here)  URL: #36774 Reported by: Matthias Köppe Reviewer(s): David Coudert
diff --git a/build/bin/sage-spkg b/build/bin/sage-spkg
@@ -341,7 +341,7 @@ fi
 PKG_BASE_VER=`echo $PKG_VER | sed 's/\.p[0-9][0-9]*$//'`
 if [ -f "$PKG_SCRIPTS/checksums.ini" ]; then
     # Normal/wheel package
-    PKG_NAME_UPSTREAM=`lookup_param tarball "$PKG_SCRIPTS/checksums.ini" | sed "s/VERSION/$PKG_BASE_VER/"`
+    PKG_NAME_UPSTREAM=`lookup_param tarball "$PKG_SCRIPTS/checksums.ini"`
 fi
 
 # Set the $SAGE_DESTDIR variable to be passed to the spkg-install
@@ -396,11 +396,11 @@ ensure_pkg_src() { ###############################################
 if [ ! -f "$PKG_SRC" ]; then
     if [ -n "$PKG_NAME_UPSTREAM" ]; then
         # Normal or wheel package
-        if ! sage-download-file $SAGE_DOWNLOAD_FILE_OPTIONS "$PKG_NAME_UPSTREAM"; then
-            error_msg "Error downloading $PKG_NAME_UPSTREAM"
+        PKG_SRC=$(sage-package download $SAGE_DOWNLOAD_FILE_OPTIONS $PKG_BASE)
+        if [ $? != 0 ]; then
+            error_msg "Error downloading tarball of $PKG_BASE"
             exit 1
         fi
-        PKG_SRC="$SAGE_DISTFILES/$PKG_NAME_UPSTREAM"
         # Do a final check that PKG_SRC is a file with an absolute path
         cd /
         if [ ! -f "$PKG_SRC" ]; then
diff --git a/build/pkgs/4ti2/checksums.ini b/build/pkgs/4ti2/checksums.ini
@@ -1,5 +1,5 @@
-tarball=4ti2-VERSION.tar.gz
+tarball=4ti2-${VERSION}.tar.gz
 sha1=3d41f30ea3ef94c293eae30c087494269fc1a6b9
 md5=1215872325ddfc561865ecb22b2bccb2
 cksum=2439180289
-upstream_url=https://github.com/4ti2/4ti2/releases/download/Release_1_6_10/4ti2-1.6.10.tar.gz
+upstream_url=https://github.com/4ti2/4ti2/releases/download/Release_${VERSION_MAJOR}_${VERSION_MINOR}_${VERSION_MICRO}/4ti2-${VERSION}.tar.gz
diff --git a/build/pkgs/cunningham_tables/checksums.ini b/build/pkgs/cunningham_tables/checksums.ini
@@ -2,4 +2,4 @@ tarball=cunningham_tables-VERSION.tar.gz
 sha1=8bea1a113d85bb9c37d8f213dd19525d9d026f22
 md5=e71b32f12e9a46c1c86e275e8441a06b
 cksum=1990403877
-upstream_url=http://users.ox.ac.uk/~coml0531/sage/cunningham_tables-1.0.tar.gz
+upstream_url=http://users.ox.ac.uk/~coml0531/sage/cunningham_tables-VERSION.tar.gz
diff --git a/build/pkgs/gf2x/checksums.ini b/build/pkgs/gf2x/checksums.ini
@@ -2,4 +2,4 @@ tarball=gf2x-VERSION.tar.gz
 sha1=1b9c7e14031afc5488b9aa27f5501f78c90f00b4
 md5=842f087ce423c279dced26b85b0fd1d0
 cksum=3368093312
-upstream_url=https://gitlab.inria.fr/gf2x/gf2x/uploads/c46b1047ba841c20d1225ae73ad6e4cd/gf2x-1.3.0.tar.gz
+upstream_url=https://gitlab.inria.fr/gf2x/gf2x/uploads/c46b1047ba841c20d1225ae73ad6e4cd/gf2x-VERSION.tar.gz
diff --git a/build/pkgs/graphs/checksums.ini b/build/pkgs/graphs/checksums.ini
@@ -2,4 +2,4 @@ tarball=graphs-VERSION.tar.bz2
 sha1=c3b9fcbc92482efd6b7f6f3a33df5a78e1256aa1
 md5=4357919410e8ac2611c9fe643976c8ff
 cksum=2340933149
-upstream_url=http://users.ox.ac.uk/~coml0531/sage/graphs-20210214.tar.bz2
+upstream_url=http://users.ox.ac.uk/~coml0531/sage/graphs-VERSION.tar.bz2
diff --git a/build/pkgs/kenzo/checksums.ini b/build/pkgs/kenzo/checksums.ini
@@ -1,5 +1,5 @@
-tarball=kenzo-1.1.10.tar.gz
-upstream_url=https://github.com/miguelmarco/kenzo/releases/download/1.1.10/kenzo-1.1.10.tar.gz
+tarball=kenzo-VERSION.tar.gz
+upstream_url=https://github.com/miguelmarco/kenzo/releases/download/VERSION/kenzo-VERSION.tar.gz
 sha1=76115aae9972090d5d51fee18592fc7a79461474
 md5=3a3d5350fb17304f03e614713e585ed4
 cksum=2981306888
diff --git a/build/pkgs/nauty/checksums.ini b/build/pkgs/nauty/checksums.ini
@@ -1,5 +1,5 @@
-tarball=nautyVERSION.tar.gz
-sha1=10c39117c55c69c18c6a107110e7c08f3d873652
-md5=7a82f4209f5d552da3078c67e5af872e
-cksum=2164796643
-upstream_url=https://pallini.di.uniroma1.it/nauty2_8_6.tar.gz
+tarball=nauty${VERSION}.tar.gz
+sha1=672e9fc9dfd07201af37ee65807a9b493331ed92
+md5=16c6edc1a8747c9281041b7c7092135f
+cksum=2663136901
+upstream_url=https://pallini.di.uniroma1.it/nauty${VERSION_MAJOR}_${VERSION_MINOR}_${VERSION_MICRO}.tar.gz
diff --git a/build/pkgs/nauty/package-version.txt b/build/pkgs/nauty/package-version.txt
@@ -1 +1 @@
-2.8.6.p1
+2.8.8
diff --git a/build/pkgs/nauty/patches/nauty-2.8.6-gentreeg-gentourng.patch b/build/pkgs/nauty/patches/nauty-2.8.6-gentreeg-gentourng.patch
diff --git a/build/pkgs/planarity/checksums.ini b/build/pkgs/planarity/checksums.ini
@@ -2,4 +2,4 @@ tarball=planarity-VERSION.tar.gz
 sha1=8407bccf33c07bf0dae22d79b5e6ac7d89c62ea3
 md5=200116e6a67544c8e94f9de7c3ba1b1a
 cksum=4207261512
-upstream_url=http://users.ox.ac.uk/~coml0531/sage/planarity-3.0.1.0.tar.gz
+upstream_url=http://users.ox.ac.uk/~coml0531/sage/planarity-VERSION.tar.gz
diff --git a/build/pkgs/plantri/checksums.ini b/build/pkgs/plantri/checksums.ini
@@ -1,5 +1,5 @@
-tarball=plantri53.tar.gz
+tarball=plantri${VERSION_MAJOR}${VERSION_MINOR}.tar.gz
 sha1=a04aec2fa90c43f1c9bef59d041a54d8fa5bf562
 md5=ea765b3508dd56384f94ad1f032e2dd4
 cksum=3200215885
-upstream_url=https://users.cecs.anu.edu.au/~bdm/plantri/plantri53.tar.gz
+upstream_url=https://users.cecs.anu.edu.au/~bdm/plantri/plantri${VERSION_MAJOR}${VERSION_MINOR}.tar.gz
diff --git a/build/pkgs/polytopes_db_4d/checksums.ini b/build/pkgs/polytopes_db_4d/checksums.ini
@@ -2,4 +2,4 @@ tarball=polytopes_db_4d-VERSION.spkg
 sha1=c9779821e365df2d7f9bc684f9e2ec0e95fb8650
 md5=fe775a26fd7b2afc187e9bfabfb1b86a
 cksum=3415837678
-upstream_url=http://ftp.sparcs.org/sage/spkg/huge/polytopes_db_4d-1.0.spkg
+upstream_url=http://ftp.sparcs.org/sage/spkg/huge/polytopes_db_4d-VERSION.spkg
diff --git a/build/pkgs/sqlite/checksums.ini b/build/pkgs/sqlite/checksums.ini
@@ -1,5 +1,5 @@
-tarball=sqlite-autoconf-3360000.tar.gz
+tarball=sqlite-autoconf-${VERSION_MAJOR}${VERSION_MINOR}0${VERSION_MICRO}00.tar.gz
 sha1=a4bcf9e951bfb9745214241ba08476299fc2dc1e
 md5=f5752052fc5b8e1b539af86a3671eac7
 cksum=763219165
-upstream_url=https://www.sqlite.org/2021/sqlite-autoconf-3360000.tar.gz
+upstream_url=https://www.sqlite.org/2021/sqlite-autoconf-${VERSION_MAJOR}${VERSION_MINOR}0${VERSION_MICRO}00.tar.gz
diff --git a/build/pkgs/symmetrica/checksums.ini b/build/pkgs/symmetrica/checksums.ini
@@ -2,4 +2,4 @@ tarball=symmetrica-VERSION.tar.xz
 sha1=0044cc087ff04267c246e730c6570d89f6e593af
 md5=cd4716c26b5c625a012c22656113ef6f
 cksum=1186250347
-upstream_url=http://users.ox.ac.uk/~coml0531/sage/symmetrica-3.0.1.tar.xz
+upstream_url=http://users.ox.ac.uk/~coml0531/sage/symmetrica-VERSION.tar.xz
diff --git a/build/pkgs/sympow/checksums.ini b/build/pkgs/sympow/checksums.ini
@@ -2,4 +2,4 @@ tarball=sympow-vVERSION.tar.gz
 sha1=37a909c26009415197b5088a2f1b53dd3558f494
 md5=51f2c717c84ec9c2840af740751cf797
 cksum=1444149964
-upstream_url=https://github.com/mkoeppe/sympow/releases/download/v2.023.6/sympow-v2.023.6.tar.gz
+upstream_url=https://github.com/mkoeppe/sympow/releases/download/vVERSION/sympow-vVERSION.tar.gz
diff --git a/build/sage_bootstrap/cmdline.py b/build/sage_bootstrap/cmdline.py
@@ -292,6 +292,9 @@ def make_parser():
     parser_download.add_argument(
         '--on-error', choices=['stop', 'warn'], default='stop',
         help='What to do if the tarball cannot be downloaded')
+    parser.add_argument(
+        '--no-check-certificate', action='store_true',
+        help='Do not check SSL certificates for https connections')
 
     parser_upload = subparsers.add_parser(
         'upload', epilog=epilog_upload,
@@ -381,6 +384,12 @@ def run():
     elif args.subcommand == 'update-latest':
         app.update_latest_cls(args.package_name, commit=args.commit)
     elif args.subcommand == 'download':
+        if args.no_check_certificate:
+            try:
+                import ssl
+                ssl._create_default_https_context = ssl._create_unverified_context
+            except ImportError:
+                pass
         app.download_cls(args.package_name,
                          allow_upstream=args.allow_upstream,
                          on_error=args.on_error)
diff --git a/build/sage_bootstrap/package.py b/build/sage_bootstrap/package.py
@@ -121,6 +121,42 @@ def tarball(self):
             self.__tarball = Tarball(self.tarball_filename, package=self)
         return self.__tarball
 
+    def _substitute_variables_once(self, pattern):
+        """
+        Substitute (at most) one occurrence of variables in ``pattern`` by the values.
+
+        These variables are ``VERSION``, ``VERSION_MAJOR``, ``VERSION_MINOR``,
+        ``VERSION_MICRO``, either appearing like this or in the form ``${VERSION_MAJOR}``
+        etc.
+
+        Return a tuple:
+        - the string with the substitution done or the original string
+        - whether a substitution was done
+        """
+        for var in ('VERSION_MAJOR', 'VERSION_MINOR', 'VERSION_MICRO', 'VERSION'):
+            # As VERSION is a substring of the other three, it needs to be tested last.
+            dollar_brace_var = '${' + var + '}'
+            if dollar_brace_var in pattern:
+                value = getattr(self, var.lower())
+                return pattern.replace(dollar_brace_var, value, 1), True
+            elif var in pattern:
+                value = getattr(self, var.lower())
+                return pattern.replace(var, value, 1), True
+        return pattern, False
+
+    def _substitute_variables(self, pattern):
+        """
+        Substitute all occurrences of ``VERSION`` in ``pattern`` by the actual version.
+
+        Likewise for ``VERSION_MAJOR``, ``VERSION_MINOR``, ``VERSION_MICRO``,
+        either appearing like this or in the form ``${VERSION}``, ``${VERSION_MAJOR}``,
+        etc.
+        """
+        not_done = True
+        while not_done:
+            pattern, not_done = self._substitute_variables_once(pattern)
+        return pattern
+
     @property
     def tarball_pattern(self):
         """
@@ -150,7 +186,7 @@ def tarball_filename(self):
         """
         pattern = self.tarball_pattern
         if pattern:
-            return pattern.replace('VERSION', self.version)
+            return self._substitute_variables(pattern)
         else:
             return None
 
@@ -177,7 +213,7 @@ def tarball_upstream_url(self):
         """
         pattern = self.tarball_upstream_url_pattern
         if pattern:
-            return pattern.replace('VERSION', self.version)
+            return self._substitute_variables(pattern)
         else:
             return None
 
@@ -212,6 +248,39 @@ def version(self):
         """
         return self.__version
 
+    @property
+    def version_major(self):
+        """
+        Return the major version
+
+        OUTPUT:
+
+        String. The package's major version.
+        """
+        return self.version.split('.')[0]
+
+    @property
+    def version_minor(self):
+        """
+        Return the minor version
+
+        OUTPUT:
+
+        String. The package's minor version.
+        """
+        return self.version.split('.')[1]
+
+    @property
+    def version_micro(self):
+        """
+        Return the micro version
+
+        OUTPUT:
+
+        String. The package's micro version.
+        """
+        return self.version.split('.')[2]
+
     @property
     def patchlevel(self):
         """
diff --git a/build/sage_bootstrap/updater.py b/build/sage_bootstrap/updater.py
@@ -69,8 +69,11 @@ def _update_version(self, new_version):
     def download_upstream(self, download_url=None):
         tarball = self.package.tarball
         if download_url is None:
+            pattern = self.package.tarball_upstream_url_pattern
+            if pattern and 'VERSION' not in pattern:
+                print('Warning: upstream_url pattern does not use the VERSION variable')
             download_url = self.package.tarball_upstream_url
         if download_url is None:
             raise ValueError("package has no default upstream_url pattern, download_url needed")
-        print('Downloading tarball to {0}'.format(tarball.upstream_fqn))
+        print('Downloading tarball from {0} to {1}'.format(download_url, tarball.upstream_fqn))
         Download(download_url, tarball.upstream_fqn).run()
diff --git a/src/doc/en/developer/packaging.rst b/src/doc/en/developer/packaging.rst
