Merge branch 'develop' into p/use-external-urls-in-jupyterlab

Author: kwankyu

.ci/merge-fixes.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+# Merge open PRs from sagemath/sage labeled "blocker".
+REPO="sagemath/sage"
+GH="gh -R $REPO"
+PRs="$($GH pr list --label "p: blocker / 1" --json number --jq '.[].number')"
+if [ -z "$PRs" ]; then
+    echo 'Nothing to do: Found no open PRs with "blocker" status.'
+else
+    echo "Found PRs: " $PRs
+    export GIT_AUTHOR_NAME="ci-sage workflow"
+    export GIT_AUTHOR_EMAIL="ci-sage@example.com"
+    export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"
+    export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
+    git tag -f test_base
+    git commit -q -m "Uncommitted changes" --no-allow-empty -a
+    for a in $PRs; do
+        echo "::group::Merging PR https://github.com/$REPO/pull/$a"
+        git tag -f test_head
+        $GH pr checkout -b pr-$a $a
+        git fetch --unshallow --all
+        git checkout -q test_head
+        if git merge --no-edit --squash -q pr-$a; then
+            echo "::endgroup::"
+            if git commit -q -m "Merge https://github.com/$REPO/pull/$a" -a --no-allow-empty; then
+                echo "Merged #$a"
+            else
+                echo "Empty, skipped"
+            fi
+        else
+            echo "::endgroup::"
+            echo "Failure merging #$a, resetting"
+            git reset --hard
+        fi
+    done
+    git log test_base..HEAD
+fi

.ci/retrofit-worktree.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+if [ $# != 2 ]; then
+    echo >&2 "usage: $0 WORKTREE_NAME WORKTREE_DIRECTORY"
+    echo >&2 "Ensures that the current working directory is a git repository,"
+    echo >&2 "then makes WORKTREE_DIRECTORY a git worktree named WORKTREE_NAME."
+fi
+WORKTREE_NAME="$1"
+WORKTREE_DIRECTORY="$2"
+
+export GIT_AUTHOR_NAME="ci-sage workflow"
+export GIT_AUTHOR_EMAIL="ci-sage@example.com"
+export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"
+export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
+
+set -ex
+
+# If actions/checkout downloaded our source tree using the GitHub REST API
+# instead of with git (because do not have git installed in our image),
+# we first make the source tree a repo.
+if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi
+
+# Tag this state of the source tree "new". This is what we want to build and test.
+git tag -f new
+
+# Our container image contains a source tree in $WORKTREE_DIRECTORY with a full build of Sage.
+# But $WORKTREE_DIRECTORY is not a git repository.
+# We make $WORKTREE_DIRECTORY a worktree whose index is at tag "new".
+# We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.)
+# Then we update worktree and index with "git reset --hard new".
+# (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.)
+# Finally we reset the index to "old". (This keeps all mtimes unchanged.)
+# The changed files now show up as uncommitted changes.
+# The final "git add -N" makes sure that files that were added in "new" do not show
+# as untracked files, which would be removed by "git clean -fx".
+git worktree add --detach $WORKTREE_NAME
+rm -rf $WORKTREE_DIRECTORY/.git && mv $WORKTREE_NAME/.git $WORKTREE_DIRECTORY/
+rm -rf $WORKTREE_NAME && ln -s $WORKTREE_DIRECTORY $WORKTREE_NAME
+if [ ! -f $WORKTREE_NAME/.gitignore ]; then cp .gitignore $WORKTREE_NAME/; fi
+(cd $WORKTREE_NAME && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status)

.devcontainer/onCreate-conda.sh

@@ -4,7 +4,7 @@ set -e
 # Create conda environment
 ./bootstrap-conda
 conda install mamba -n base -c conda-forge -y
-mamba env create --file src/environment-dev.yml || mamba env update --file src/environment-dev.yml
+mamba env create --file src/environment-dev-3.11.yml || mamba env update --file src/environment-dev-3.11.yml
 conda init bash
 
 # Build sage

.github/workflows/build.yml

@@ -2,8 +2,11 @@ name: Build & Test
 
 on:
   pull_request:
+  merge_group:
   push:
-    branches: ['**']
+    branches:
+      - master
+      - develop
     # Ignore pushes on tags to prevent two uploads of codecov reports
     tags-ignore: ['**']
   workflow_dispatch:
@@ -24,9 +27,32 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  get_ci_fixes:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        id: checkout
+        uses: actions/checkout@v4
+      - name: Merge CI fixes from sagemath/sage
+        run: |
+          .ci/merge-fixes.sh
+        env:
+          GH_TOKEN: ${{ github.token }}
+      - name: Store CI fixes in upstream artifact
+        run: |
+          mkdir -p upstream
+          if git format-patch --stdout test_base > ci_fixes.patch; then
+            cp ci_fixes.patch upstream/
+          fi
+      - uses: actions/upload-artifact@v3
+        with:
+          path: upstream
+          name: upstream
+
   build:
     runs-on: ubuntu-latest
     container: ghcr.io/sagemath/sage/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}}
+    needs: [get_ci_fixes]
     steps:
       - name: Checkout
         id: checkout
@@ -43,48 +69,38 @@ jobs:
         id: worktree
         run: |
           set -ex
-          git config --global user.email "ci-sage@example.com"
-          git config --global user.name "Build & Test workflow"
           git config --global --add safe.directory $(pwd)
-          # If actions/checkout downloaded our source tree using the GitHub REST API
-          # instead of with git (because do not have git installed in our image),
-          # we first make the source tree a repo.
-          if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi
-          # Tag this state of the source tree "new". This is what we want to build and test.
-          git tag -f new
-          # Our container image contains a source tree in /sage with a full build of Sage.
-          # But /sage is not a git repository.
-          # We make /sage a worktree whose index is at tag "new".
-          # We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.)
-          # Then we update worktree and index with "git reset --hard new".
-          # (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.)
-          # Finally we reset the index to "old". (This keeps all mtimes unchanged.)
-          # The changed files now show up as uncommitted changes.
-          # The final "git add -N" makes sure that files that were added in "new" do not show
-          # as untracked files, which would be removed by "git clean -fx".
-          git worktree add --detach worktree-image
-          rm -rf /sage/.git && mv worktree-image/.git /sage/
-          rm -rf worktree-image && ln -s /sage worktree-image
-          if [ ! -f worktree-image/.gitignore ]; then cp .gitignore worktree-image/; fi
-          (cd worktree-image && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status)
-
-      - name: Incremental build, test changed files (sage -t --new)
+          .ci/retrofit-worktree.sh worktree-image /sage
+
+      - name: Download upstream artifact
+        uses: actions/download-artifact@v3
+        with:
+          path: upstream
+          name: upstream
+
+      - name: Apply CI fixes from sagemath/sage
+        # After applying the fixes, make sure all changes are marked as uncommitted changes.
+        run: |
+          if [ -r upstream/ci_fixes.patch ]; then
+            (cd worktree-image && git commit -q -m "current changes" --allow-empty -a && git am; git reset --quiet old; git add -N .) < upstream/ci_fixes.patch
+          fi
+
+      - name: Incremental build
         id: incremental
         run: |
           # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps.
-          # We run tests with "sage -t --new"; this only tests the uncommitted changes.
-          ./bootstrap && make build && ./sage -t --new -p2
+          ./bootstrap && make build
         working-directory: ./worktree-image
         env:
-          MAKE: make -j2
+          MAKE: make -j2 --output-sync=recurse
           SAGE_NUM_THREADS: 2
 
-      - name: Build and test modularized distributions
+      - name: Build modularized distributions
         if: always() && steps.worktree.outcome == 'success'
-        run: make V=0 tox && make pypi-wheels
+        run: make V=0 tox && make SAGE_CHECK=no pypi-wheels
         working-directory: ./worktree-image
         env:
-          MAKE: make -j2
+          MAKE: make -j2 --output-sync=recurse
           SAGE_NUM_THREADS: 2
 
       - name: Set up node to install pyright
@@ -123,7 +139,27 @@ jobs:
           make build
         working-directory: ./worktree-image
         env:
-          MAKE: make -j2
+          MAKE: make -j2 --output-sync=recurse
+          SAGE_NUM_THREADS: 2
+
+      # Testing
+
+      - name: Test changed files (sage -t --new)
+        if: always() && steps.build.outcome == 'success'
+        run: |
+          # We run tests with "sage -t --new"; this only tests the uncommitted changes.
+          ./sage -t --new -p2
+        working-directory: ./worktree-image
+        env:
+          MAKE: make -j2 --output-sync=recurse
+          SAGE_NUM_THREADS: 2
+
+      - name: Test modularized distributions
+        if: always() && steps.build.outcome == 'success'
+        run: make V=0 tox && make pypi-wheels-check
+        working-directory: ./worktree-image
+        env:
+          MAKE: make -j2 --output-sync=recurse
           SAGE_NUM_THREADS: 2
 
       - name: Pytest

.github/workflows/ci-conda.yml

@@ -7,13 +7,6 @@ on:
     branches:
       - 'public/build/**-runci'
   pull_request:
-    types:
-      # Defaults
-      - opened
-      - synchronize
-      - reopened
-      # When a CI label is added
-      - labeled
   workflow_dispatch:
     # Allow to run manually
 
@@ -27,20 +20,9 @@ jobs:
     name: Conda
     runs-on: ${{ matrix.os }}
 
-    # Run on push, workflow dispatch and when certain labels are added or are present
-    if: |
-      github.event_name != 'pull_request' ||
-      ((github.event.action != 'labeled' &&
-         (contains(github.event.pull_request.labels.*.name, 'c: packages: standard') ||
-          contains(github.event.pull_request.labels.*.name, 'c: packages: optional') ||
-          contains(github.event.pull_request.labels.*.name, 's: run conda ci'))) ||
-       (github.event.action == 'labeled' &&
-         (github.event.label.name == 'c: packages: optional' ||
-          github.event.label.name == 'c: packages: standard' ||
-          github.event.label.name == 's: run conda ci')))
-
     strategy:
-      fail-fast: false
+      fail-fast: ${{ github.event_name == 'pull_request' }}
+      max-parallel: ${{ github.event_name == 'pull_request' && 2 || 6 }}
       matrix:
         os: [ubuntu-latest, macos-latest]
         python: ['3.9', '3.10', '3.11']
@@ -51,17 +33,11 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Check for Miniconda
-        id: check_conda
-        run: echo ::set-output name=installed::$CONDA
-
-      # Miniconda is installed by default in the ubuntu-latest, however not in the act-image to run it locally
-      - name: Install Miniconda
-        if: steps.check_conda.outputs.installed == ''
+      - name: Merge CI fixes from sagemath/sage
         run: |
-          wget https://repo.anaconda.com/miniconda/Miniconda3-py39_4.10.3-Linux-x86_64.sh -O ~/miniconda.sh
-          bash ~/miniconda.sh -b -p $HOME/miniconda
-          echo "CONDA=$HOME/miniconda" >> $GITHUB_ENV
+          .ci/merge-fixes.sh
+        env:
+          GH_TOKEN: ${{ github.token }}
 
       - name: Create conda environment files
         run: ./bootstrap-conda
@@ -71,17 +47,18 @@ jobs:
         with:
           path: ~/conda_pkgs_dir
           key:
-            ${{ runner.os }}-conda-${{ hashFiles('src/environment.yml') }}
+            ${{ runner.os }}-conda-${{ hashFiles('src/environment-3.11.yml') }}
 
-      - name: Setup Conda
+      - name: Setup Conda environment
         uses: conda-incubator/setup-miniconda@v2
         with:
           python-version: ${{ matrix.python }}
-          mamba-version: "*"
-          channels: conda-forge,defaults
+          miniforge-version: latest
+          use-mamba: true
+          channels: conda-forge
           channel-priority: true
-          activate-environment: sage-build
-          environment-file: src/${{ matrix.conda-env }}.yml
+          activate-environment: sage
+          environment-file: src/${{ matrix.conda-env }}-${{ matrix.python }}.yml
 
       - name: Print Conda environment
         shell: bash -l {0}
@@ -95,19 +72,16 @@ jobs:
         run: |
           ./bootstrap
           echo "::add-matcher::.github/workflows/configure-systempackage-problem-matcher.json"
-          ./configure --enable-build-as-root --with-python=$CONDA_PREFIX/bin/python --prefix=$CONDA_PREFIX $(for pkg in $(./sage -package list :standard: --has-file spkg-configure.m4 --has-file distros/conda.txt --exclude rpy2); do echo --with-system-$pkg=force; done)
+          ./configure --enable-build-as-root --with-python=$CONDA_PREFIX/bin/python --prefix=$CONDA_PREFIX --enable-system-site-packages $(for pkg in $(./sage -package list :standard: --has-file spkg-configure.m4 --has-file distros/conda.txt --exclude rpy2); do echo --with-system-$pkg=force; done)
           echo "::remove-matcher owner=configure-system-package-warning::"
           echo "::remove-matcher owner=configure-system-package-error::"
 
-          # Manually install ptyprocess for now, until https://github.com/sagemath/sage/issues/32147 is fixed
-          pip install --no-build-isolation -v -v ptyprocess==0.5.1
-
       - name: Build
         shell: bash -l {0}
         run: |
           # Use --no-deps and pip check below to verify that all necessary dependencies are installed via conda.
           pip install --no-build-isolation --no-deps -v -v -e ./pkgs/sage-conf ./pkgs/sage-setup
-          pip install --no-build-isolation --no-deps -v -v -e ./src
+          pip install --no-build-isolation --no-deps --config-settings editable_mode=compat -v -v -e ./src
         env:
           SAGE_NUM_THREADS: 2