Merge branch 'dev' into preview-files-of-all-datatypes

.ci/check_test_class_names.sh

@@ -1,7 +1,15 @@
 #!/bin/sh
-n_tests=$(pytest --collect-only --ignore=test/functional lib/galaxy_test/ test/ | grep 'tests collected' | sed -e 's/[^0-9]*\([0-9]*\) tests collected.*/\1/')
-n_tests_extra_classes=$(pytest -o python_classes='Test* *Test *TestCase' --collect-only --ignore=test/functional lib/galaxy_test/ test/ | grep 'tests collected' | sed -e 's/[^0-9]*\([0-9]*\) tests collected.*/\1/')
+TEST=$(mktemp)
+TEST_EXTRA_CLASSES=$(mktemp)
+
+pytest --collect-only --ignore=test/functional lib/galaxy_test/ test/ > "$TEST"
+pytest -o python_classes='Test* *Test *TestCase' --collect-only --ignore=test/functional lib/galaxy_test/ test/ > "$TEST_EXTRA_CLASSES"
+
+n_tests=$(grep 'tests collected' "$TEST" | sed -e 's/[^0-9]*\([0-9]*\) tests collected.*/\1/')
+n_tests_extra_classes=$(grep 'tests collected' "$TEST_EXTRA_CLASSES" | sed -e 's/[^0-9]*\([0-9]*\) tests collected.*/\1/')
+
 if [ "$n_tests_extra_classes" -gt "$n_tests" ]; then
     echo "New test class with name not starting with Test introduced, change it to have tests collected by pytest"
+    diff "$TEST" "$TEST_EXTRA_CLASSES"
     exit 1
 fi

.ci/eslint_wrapper.sh

@@ -5,4 +5,4 @@ set -e
 # Setting NODE_PATH and config appropriately, using dependencies from
 # client/node_modules, run eslint against args passed to this script.
 # Primary use case here is for a pre-commit check.
-NODE_PATH=src/ node client/node_modules/eslint/bin/eslint.js -c client/.eslintrc.json "$@"
+NODE_PATH=src/ node client/node_modules/eslint/bin/eslint.js -c client/.eslintrc.js "$@"

.circleci/config.yml

@@ -24,7 +24,7 @@ jobs:
           command: |-
               # Add github.com to known hosts
               mkdir -p ~/.ssh
-              echo 'github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
+              echo 'github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=
               ' >> ~/.ssh/known_hosts
 
               # Add the user ssh key and set correct perms

.github/workflows/api.yaml

@@ -47,6 +47,11 @@ jobs:
       - uses: actions/checkout@v3
         with:
           path: 'galaxy root'
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.12.1'
+          cache: 'yarn'
+          cache-dependency-path: 'galaxy root/client/yarn.lock'
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}

.github/workflows/build_container_image.yaml

@@ -2,8 +2,10 @@ name: Build Container Image
 on:
   push:
     branches:
-      - 'release*'
       - dev
+      - release_*
+    tags:
+      - v*
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
@@ -20,7 +22,15 @@ jobs:
         run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
       - name: Set branch name
         id: branch
-        run: echo "name=$(BRANCH_NAME=${GITHUB_REF##*/}; echo ${BRANCH_NAME/release_/})" >> $GITHUB_OUTPUT
+        run: |
+          if [[ "$GITHUB_REF" == "refs/tags/"* ]]; then
+            echo "name=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
+          elif [[ "$GITHUB_REF" == "refs/heads/dev" ]]; then
+            echo "name=dev" >> $GITHUB_OUTPUT
+          elif [[ "$GITHUB_REF" == "refs/heads/release_"* ]]; then
+            echo "name=${GITHUB_REF#refs/heads/release_}-auto" >> $GITHUB_OUTPUT
+          fi
+        shell: bash
       - name: Build container image
         run: docker build . --build-arg GIT_COMMIT=$(git rev-parse HEAD) --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') --build-arg IMAGE_TAG=${{ steps.branch.outputs.name }} -t galaxy/galaxy-min:${{ steps.branch.outputs.name }} -t quay.io/galaxyproject/galaxy-min:${{ steps.branch.outputs.name }} -f .k8s_ci.Dockerfile
       - name: Create auto-expiring one for per-commit auto repository

.github/workflows/converter_tests.yaml

@@ -31,6 +31,11 @@ jobs:
       - uses: actions/checkout@v3
         with:
           path: 'galaxy root'
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.12.1'
+          cache: 'yarn'
+          cache-dependency-path: 'galaxy root/client/yarn.lock'
       - name: Clone galaxyproject/galaxy-test-data
         uses: actions/checkout@v3
         with:

.github/workflows/cwl_conformance.yaml

@@ -38,6 +38,11 @@ jobs:
       - uses: actions/checkout@v3
         with:
           path: 'galaxy root'
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.12.1'
+          cache: 'yarn'
+          cache-dependency-path: 'galaxy root/client/yarn.lock'
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}

.github/workflows/first_startup.yaml

@@ -21,14 +21,19 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.10']
+        python-version: ['3.7', '3.11']
     defaults:
       run:
         shell: bash -l {0}
     steps:
       - uses: actions/checkout@v3
         with:
           path: 'galaxy root'
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.12.1'
+          cache: 'yarn'
+          cache-dependency-path: 'galaxy root/client/yarn.lock'
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
@@ -48,9 +53,6 @@ jobs:
         with:
           path: .tox
           key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-first-startup
-      - uses: mvdbeek/gha-yarn-cache@master
-        with:
-          yarn-lock-file: 'galaxy root/client/yarn.lock'
       - name: Install tox
         run: pip install tox
       - name: run tests

.github/workflows/framework.yaml

@@ -43,6 +43,11 @@ jobs:
       - uses: actions/checkout@v3
         with:
           path: 'galaxy root'
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.12.1'
+          cache: 'yarn'
+          cache-dependency-path: 'galaxy root/client/yarn.lock'
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}

.github/workflows/integration.yaml

@@ -71,6 +71,11 @@ jobs:
       - uses: actions/checkout@v3
         with:
           path: 'galaxy root'
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.12.1'
+          cache: 'yarn'
+          cache-dependency-path: 'galaxy root/client/yarn.lock'
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
@@ -89,6 +94,8 @@ jobs:
         with:
           path: 'galaxy root/.venv'
           key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-integration
+      - name: Install Apptainer's singularity
+        uses: eWaterCycle/setup-apptainer@v2
       - name: Run tests
         run: |
           . .ci/minikube-test-setup/start_services.sh

.github/workflows/integration_selenium.yaml

@@ -63,9 +63,11 @@ jobs:
         with:
           path: 'galaxy root/.venv'
           key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-integration-selenium
-      - uses: mvdbeek/gha-yarn-cache@master
+      - uses: actions/setup-node@v3
         with:
-          yarn-lock-file: 'galaxy root/client/yarn.lock'
+          node-version: '18.12.1'
+          cache: 'yarn'
+          cache-dependency-path: 'galaxy root/client/yarn.lock'
       - uses: nanasess/setup-chromedriver@v1
       - name: Run tests
         run: ./run_tests.sh --coverage -integration test/integration_selenium

.github/workflows/jest.yaml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [16]
+        node: [18]
     steps:
       - uses: actions/checkout@v3
       - name: Setup node
@@ -31,7 +31,5 @@ jobs:
         run: yarn run gulp client
         working-directory: client
       - name: Run Unit Tests
-        run: yarn run qunit
-        working-directory: client
-      - run: yarn jest
+        run: yarn jest
         working-directory: client

.github/workflows/js_lint.yaml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node: [16]
+        node: [18]
     steps:
       - uses: actions/checkout@v3
       - name: Setup node

.github/workflows/lint.yaml

@@ -21,7 +21,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.7', '3.10']
+        python-version: ['3.7', '3.11']
     env:
       LINT_PATH: 'lib/galaxy/dependencies/pinned-lint-requirements.txt'
       TYPE_PATH: 'lib/galaxy/dependencies/pinned-typecheck-requirements.txt'

.github/workflows/lint_openapi_schema.yml

@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7']
+        python-version: ['3.7', '3.11']
     steps:
       - uses: actions/checkout@v3
         with:
@@ -43,7 +43,7 @@ jobs:
           path: 'galaxy root/.venv'
           key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-api
       - name: Install dependencies
-        run: ./scripts/common_startup.sh --skip-client-build
+        run: ./scripts/common_startup.sh --dev-wheels --skip-client-build
         working-directory: 'galaxy root'
       - name: Lint schema
         run: make lint-api-schema

.github/workflows/osx_startup.yaml

@@ -15,13 +15,22 @@ jobs:
   test:
     name: Startup test
     runs-on: macos-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.7', '3.11']
     defaults:
       run:
         shell: bash -l {0}
     steps:
       - uses: actions/checkout@v3
         with:
           path: 'galaxy root'
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.12.1'
+          cache: 'yarn'
+          cache-dependency-path: 'galaxy root/client/yarn.lock'
       - name: Get full Python version
         id: full-python-version
         shell: bash
@@ -31,19 +40,25 @@ jobs:
         id: pip-cache
         with:
           path: ~/Library/Caches/pip
-          # scripts/common_startup.sh creates a conda env for Galaxy containing Python 3.7
-          key: pip-cache-3.7-${{ hashFiles('galaxy root/requirements.txt') }}
+          key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
       - name: Cache tox env
         uses: actions/cache@v3
         with:
           path: .tox
           key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-osx
-      - name: Install and activate miniconda  # use this job to test using Python from a conda environment
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.12.1'
+          cache: 'yarn'
+          cache-dependency-path: 'galaxy root/client/yarn.lock'
+      - name: Install miniconda  # use this job to test using Python from a conda environment
         uses: conda-incubator/setup-miniconda@v2
         with:
           activate-environment: ''
       - name: Install tox
         run: pip install tox
-      - name: run tests
+      - name: Run tests
         run: tox -e first_startup
         working-directory: 'galaxy root'
+        env:
+          GALAXY_CONDA_PYTHON_VERSION: "${{ matrix.python-version }}"

.github/workflows/performance.yaml

@@ -42,6 +42,11 @@ jobs:
       - uses: actions/checkout@v3
         with:
           path: 'galaxy root'
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.12.1'
+          cache: 'yarn'
+          cache-dependency-path: 'galaxy root/client/yarn.lock'
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}

.github/workflows/publish_artifacts.yaml

@@ -0,0 +1,26 @@
+name: Publish release artifacts
+on:
+    release:
+      types: [released, prereleased]
+jobs:
+  build-and-publish:
+    if: github.repository_owner == 'galaxyproject'
+    name: build-and-publish
+    runs-on: ubuntu-latest
+    strategy:
+        matrix:
+            python-version: ['3.7']
+    steps:
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: actions/checkout@v3
+      - name: Install script dependencies
+        run: pip install galaxy-release-util
+      - name: Build and publish
+        run: |
+          galaxy-release-util build-and-upload --no-confirm
+        env:
+            TWINE_USERNAME: __token__
+            TWINE_PASSWORD: ${{ github.event.release.prerelease && secrets.PYPI_TEST_TOKEN || secrets.PYPI_MAIN_TOKEN }}
+            TWINE_REPOSITORY_URL: ${{ github.event.release.prerelease && 'https://test.pypi.org/legacy/' || 'https://upload.pypi.org/legacy/' }}

.github/workflows/reports_startup.yaml

@@ -18,14 +18,19 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.10']
+        python-version: ['3.7', '3.11']
     defaults:
       run:
         shell: bash -l {0}
     steps:
       - uses: actions/checkout@v3
         with:
           path: 'galaxy root'
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.12.1'
+          cache: 'yarn'
+          cache-dependency-path: 'galaxy root/client/yarn.lock'
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
@@ -45,9 +50,6 @@ jobs:
         with:
           path: 'galaxy root/.venv'
           key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-reports-startup
-      - uses: mvdbeek/gha-yarn-cache@master
-        with:
-          yarn-lock-file: 'galaxy root/client/yarn.lock'
       - name: Install tox
         run: pip install tox
       - name: Run tests

.github/workflows/selenium.yaml

@@ -46,6 +46,11 @@ jobs:
       - uses: actions/checkout@v3
         with:
           path: 'galaxy root'
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.12.1'
+          cache: 'yarn'
+          cache-dependency-path: 'galaxy root/client/yarn.lock'
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
@@ -63,9 +68,6 @@ jobs:
         with:
           path: 'galaxy root/.venv'
           key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-selenium
-      - uses: mvdbeek/gha-yarn-cache@master
-        with:
-          yarn-lock-file: 'galaxy root/client/yarn.lock'
       - uses: nanasess/setup-chromedriver@v1
       - name: Run tests
         run: ./run_tests.sh --coverage -selenium lib/galaxy_test/selenium -- --num-shards=3 --shard-id=${{ matrix.chunk }}

.github/workflows/test_galaxy_packages.yaml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.10']
+        python-version: ['3.7', '3.11']
     steps:
       - uses: actions/checkout@v3
         with: