Release Manager · Release Manager · commit 795a40e97bbc · 2024-01-12T03:44:39.000+01:00
gh-36977: `sage --package metrics`: New tool to assist discussions of the Sage distribution   Discussions of the complexity of the Sage distribution pop up unexpectedly, as seen in #36982 (comment), #36982 (comment), #36982 (comment), #36982 (comment), #36982 (comment), #36982 (comment), #36982 (comment) #36726 (comment), #36726 (comment), #36726 (comment), #36726 (comment), #36726 (comment), #36726 (comment), #36726 (comment), #36726 (comment), #36726 (comment), #36726 (comment), #36726 (comment), #36726 (comment), #36726 (comment), #36726 (comment), #36726 (comment), #36726 (comment), #36726 (comment) #36777 (comment), #36777 (comment), #36777 (comment), #36777 (comment), #36777 (comment), #36777 (comment) To help put such discussions on a solid factual basis, we introduce the command `sage --package metrics`. ``` $ ./sage -package metrics :standard: has_file_distros_arch_txt=131 has_file_distros_conda_txt=216 has_file_distros_debian_txt=125 has_file_distros_fedora_txt=138 has_file_distros_gentoo_txt=181 has_file_distros_homebrew_txt=61 has_file_distros_macports_txt=129 has_file_distros_nix_txt=51 has_file_distros_opensuse_txt=146 has_file_distros_slackware_txt=25 has_file_distros_void_txt=184 has_file_patches=35 has_file_spkg_check=59 has_file_spkg_configure_m4=222 has_file_spkg_install=198 has_tarball_upstream_url=231 line_count_file_patches=22561 line_count_file_spkg_check=402 line_count_file_spkg_configure_m4=2792 line_count_file_spkg_install=2960 packages=272 type_standard=272 ``` Use `PATH=build/bin:$PATH SAGE_ROOT=some-other-worktree build/bin/sage- package metrics :standard:` to obtain the metrics of another version of Sage in some other worktree. We add computation and before/after comparison of the metrics to the CI Linux Incremental workflow. As an illustration, we change one Python package from "normal" to "wheel", removing an `spkg-install.in` file in the process. See https:// github.com/sagemath/sage/actions/runs/7342841283/job/19992606617?pr=3697 7#step:6:12 More metrics can be added after - #36740    ### 📝 Checklist     - [x] The title is concise, informative, and self-explanatory. - [x] The description explains in detail what this PR is about. - [ ] I have linked a relevant issue or discussion. - [ ] I have created tests covering the changes. - [ ] I have updated the documentation accordingly. ### ⌛ Dependencies   URL: #36977 Reported by: Matthias Köppe Reviewer(s): Kwankyu Lee, Matthias Köppe
diff --git a/.github/workflows/ci-linux-incremental.yml b/.github/workflows/ci-linux-incremental.yml
@@ -63,6 +63,21 @@ jobs:
           echo "uninstall_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.configures_all_changed_files }}; do echo $a | sed -E 's,build/pkgs/([a-z0-9][_.a-z0-9]*)/spkg-configure[.]m4 *,\1-uninstall,'; done | sort -u))" >> $GITHUB_OUTPUT
           echo "build_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.pkgs_all_changed_files }}; do SPKG=$(echo $a | sed -E 's,-,_,g;s,(build/)?pkgs/([a-z0-9][-_.a-z0-9]*)/[^ ]* *,\2,;'); if [ -f "build/pkgs/$SPKG/checksums.ini" -o -f "build/pkgs/$SPKG/requirements.txt" -o -f "build/pkgs/$SPKG/spkg-install" ]; then echo "$SPKG-ensure"; fi; done | sort -u))" >> $GITHUB_OUTPUT
           cat $GITHUB_OUTPUT
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.base_ref }}
+          path: worktree-base
+        if: github.base_ref
+      - name: Compute metrics
+        run: |
+          export PATH=build/bin:$PATH
+          if [ -d worktree-base ]; then
+            (echo "# $GITHUB_BASE_REF"; SAGE_ROOT=worktree-base sage-package metrics :all:) > base-metrics.txt
+            (echo "# $GITHUB_REF"; sage-package metrics :all:) > metrics.txt
+            diff --color=always --width=100 --side-by-side --left-column base-metrics.txt metrics.txt || true
+          else
+            sage-package metrics :all:
+          fi
 
   test:
     needs: [changed_files]
diff --git a/build/pkgs/tzdata/checksums.ini b/build/pkgs/tzdata/checksums.ini
@@ -1,5 +1,5 @@
-tarball=tzdata-VERSION.tar.gz
-sha1=e244bf1bde63515d3f8a452d3bbe9f97738e1e70
-md5=2ad4652fecc6ef4f6794726d3f367363
-cksum=2894139369
-upstream_url=https://pypi.io/packages/source/t/tzdata/tzdata-VERSION.tar.gz
+tarball=tzdata-VERSION-py2.py3-none-any.whl
+sha1=4686c7c91a01d5af9075903937c343afa05c141b
+md5=5e534124c0c916ab617421247649b193
+cksum=2929397850
+upstream_url=https://pypi.io/packages/py2.py3/t/tzdata/tzdata-VERSION-py2.py3-none-any.whl
diff --git a/build/pkgs/tzdata/package-version.txt b/build/pkgs/tzdata/package-version.txt
@@ -1 +1 @@
-2022.6
+2023.3
diff --git a/build/pkgs/tzdata/spkg-install.in b/build/pkgs/tzdata/spkg-install.in
diff --git a/build/sage_bootstrap/app.py b/build/sage_bootstrap/app.py
@@ -24,6 +24,8 @@
 import logging
 log = logging.getLogger()
 
+from collections import defaultdict
+
 from sage_bootstrap.package import Package
 from sage_bootstrap.tarball import Tarball, FileNotMirroredError
 from sage_bootstrap.updater import ChecksumUpdater, PackageUpdater
@@ -366,3 +368,63 @@ def clean(self):
                     os.remove(filepath)
                     count += 1
         print('{} files were removed from the {} directory'.format(count, SAGE_DISTFILES))
+
+    def metrics_cls(self, *package_classes):
+        """
+        Show the metrics of given packages
+
+        $ sage --package metrics :standard:
+        has_file_distros_arch_txt=131
+        has_file_distros_conda_txt=216
+        has_file_distros_debian_txt=125
+        has_file_distros_fedora_txt=138
+        has_file_distros_gentoo_txt=181
+        has_file_distros_homebrew_txt=61
+        has_file_distros_macports_txt=129
+        has_file_distros_nix_txt=51
+        has_file_distros_opensuse_txt=146
+        has_file_distros_slackware_txt=25
+        has_file_distros_void_txt=184
+        has_file_patches=35
+        has_file_spkg_check=59
+        has_file_spkg_configure_m4=222
+        has_file_spkg_install=198
+        has_tarball_upstream_url=231
+        line_count_file_patches=22561
+        line_count_file_spkg_check=402
+        line_count_file_spkg_configure_m4=2792
+        line_count_file_spkg_install=2960
+        packages=272
+        type_standard=272
+        """
+        log.debug('Computing metrics')
+        metrics = defaultdict(int)
+        pc = PackageClass(*package_classes)
+        for package_name in pc.names:
+            package = Package(package_name)
+            metrics['packages'] += 1
+            metrics['type_' + package.type] += 1
+            for filenames in [['spkg-configure.m4'],
+                              ['spkg-install', 'spkg-install.in'],
+                              ['spkg-check', 'spkg-check.in'],
+                              ['distros/arch.txt'],
+                              ['distros/conda.txt'],
+                              ['distros/debian.txt'],
+                              ['distros/fedora.txt'],
+                              ['distros/gentoo.txt'],
+                              ['distros/homebrew.txt'],
+                              ['distros/macports.txt'],
+                              ['distros/nix.txt'],
+                              ['distros/opensuse.txt'],
+                              ['distros/slackware.txt'],
+                              ['distros/void.txt'],
+                              ['patches']]:
+                key = filenames[0].replace('.', '_').replace('-', '_').replace('/', '_')
+                metrics['has_file_' + key] += int(any(package.has_file(filename)
+                                                      for filename in filenames))
+                if not key.startswith('distros_'):
+                    metrics['line_count_file_' + key] += sum(package.line_count_file(filename)
+                                                             for filename in filenames)
+            metrics['has_tarball_upstream_url'] += int(bool(package.tarball_upstream_url))
+        for key, value in sorted(metrics.items()):
+            print('{0}={1}'.format(key, value))
diff --git a/build/sage_bootstrap/cmdline.py b/build/sage_bootstrap/cmdline.py
@@ -189,6 +189,27 @@
     42 files were removed from the .../upstream directory
 """
 
+epilog_metrics = \
+"""
+Print metrics of given package.
+
+EXAMPLE:
+
+    $ sage --package metrics :standard:
+    has_file_distros_arch_txt=131
+    ...
+    has_file_distros_void_txt=184
+    has_file_patches=35
+    has_file_spkg_check=59
+    has_file_spkg_configure_m4=222
+    has_file_spkg_install=198
+    has_tarball_upstream_url=231
+    line_count_file_patches=22561
+    ...
+    packages=272
+    type_standard=272
+"""
+
 
 def make_parser():
     """
@@ -346,6 +367,16 @@ def make_parser():
         formatter_class=argparse.RawDescriptionHelpFormatter,
         help='Remove outdated source tarballs from the upstream/ directory')
 
+    parser_metrics = subparsers.add_parser(
+        'metrics', epilog=epilog_metrics,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        help='Print metrics of given packages')
+    parser_metrics.add_argument(
+        'package_class', metavar='[package_name|:package_type:]',
+        type=str, nargs='*', default=[':all:'],
+        help=('package name or designator for all packages of a given type '
+              '(one of :all:, :standard:, :optional:, and :experimental:; default: :all:)'))
+
     return parser
 
 
@@ -403,6 +434,8 @@ def run():
         app.fix_checksum_cls(*args.package_class)
     elif args.subcommand == 'clean':
         app.clean()
+    elif args.subcommand == 'metrics':
+        app.metrics_cls(*args.package_class)
     else:
         raise RuntimeError('unknown subcommand: {0}'.format(args))
 
diff --git a/build/sage_bootstrap/package.py b/build/sage_bootstrap/package.py
@@ -370,6 +370,28 @@ def has_file(self, filename):
         """
         return os.path.exists(os.path.join(self.path, filename))
 
+    def line_count_file(self, filename):
+        """
+        Return the number of lines of the file
+
+        Directories are traversed recursively.
+
+        OUTPUT:
+
+        integer; 0 if the file cannot be read, 1 if it is a symlink
+        """
+        path = os.path.join(self.path, filename)
+        if os.path.islink(path):
+            return 1
+        if os.path.isdir(path):
+            return sum(self.line_count_file(os.path.join(filename, entry))
+                       for entry in os.listdir(path))
+        try:
+            with open(path, "rb") as f:
+                return len(list(f))
+        except OSError:
+            return 0
+
     def _init_checksum(self):
         """
         Load the checksums from the appropriate ``checksums.ini`` file
diff --git a/src/doc/en/developer/packaging.rst b/src/doc/en/developer/packaging.rst
