Merge branch 'matrix-org:develop' into develop

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+# Automatically request reviews from the synapse-core team when a pull request comes in.
+* @matrix-org/synapse-core

.github/workflows/docker.yml

@@ -0,0 +1,114 @@
+# GitHub actions workflow which builds and publishes the docker images.
+
+name: Build and deploy docker images
+
+on:
+  push:
+    branches: [develop]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build-sytest-images:
+    name: "Build sytest:${{ matrix.tag }}"
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - base_image: ubuntu:focal
+            tag: focal
+          - base_image: debian:buster
+            tag: buster
+          - base_image: debian:testing
+            tag: testing
+          - base_image: debian:bookworm
+            tag: bookworm
+    steps:
+      - name: Set up QEMU
+        id: QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Inspect builder
+        run: docker buildx inspect
+
+      - name: Log in to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          pull: true
+          push: true
+          platforms: linux/amd64,linux/arm64
+          labels: "gitsha1=${{ github.sha }}"
+          file: docker/base.Dockerfile
+          build-args: "BASE_IMAGE=${{ matrix.base_image }}"
+          tags: matrixdotorg/sytest:${{ matrix.tag }}
+
+  build-dependent-images:
+    needs: build-sytest-images
+    name: "Build sytest-${{ matrix.dockerfile }}:${{ matrix.sytest_image_tag }}"
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - sytest_image_tag: focal
+            dockerfile: synapse
+            tags: "matrixdotorg/sytest-synapse:focal"
+            build_args: "SYTEST_IMAGE_TAG=focal"
+          - sytest_image_tag: buster
+            dockerfile: synapse
+            tags: "matrixdotorg/sytest-synapse:buster"
+            build_args: "SYTEST_IMAGE_TAG=buster"
+          - sytest_image_tag: testing
+            dockerfile: synapse
+            tags: "matrixdotorg/sytest-synapse:testing"
+            build_args: "SYTEST_IMAGE_TAG=testing"
+          - sytest_image_tag: bookworm
+            dockerfile: synapse
+            tags: "matrixdotorg/sytest-synapse:bookworm-python3.10"
+            build_args: |
+              SYTEST_IMAGE_TAG=bookworm
+              PYTHON_VERSION=python3.10
+          - sytest_image_tag: buster
+            dockerfile: dendrite
+            tags: "matrixdotorg/sytest-dendrite:latest"
+            build_args: "SYTEST_IMAGE_TAG=buster"
+
+    steps:
+      - name: Set up QEMU
+        id: QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Inspect builder
+        run: docker buildx inspect
+
+      - name: Log in to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          pull: true
+          push: true
+          platforms: linux/amd64,linux/arm64
+          labels: "gitsha1=${{ github.sha }}"
+          file: docker/${{ matrix.dockerfile }}.Dockerfile
+          build-args: ${{ matrix.build_args }}
+          tags: ${{ matrix.tags }}

.github/workflows/pipeline.yml

@@ -0,0 +1,202 @@
+name: Run sytest
+on:
+  push:
+    branches: ["develop", "release-*"]
+  pull_request:
+
+# Only run this action once per pull request/branch; restart if a new commit arrives.
+# C.f. https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency
+# and https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#github-context
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  synapse:
+    name: "Synapse: ${{ matrix.label }}"
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - label: Py 3.8, SQLite 3.31.1, Monolith
+            sytest-tag: focal
+
+          - label: Py 3.8, PG 12, Monolith
+            sytest-tag: focal
+            postgres: postgres
+
+          - label: Py 3.8, PG 12, Workers
+            sytest-tag: focal
+            postgres: postgres
+            workers: workers
+
+          - label: Py 3.10, PG 14, Monolith
+            sytest-tag: bookworm-python3.10
+            postgres: postgres
+
+          - label: Py 3.10, PG 14, Workers
+            sytest-tag: bookworm-python3.10
+            postgres: postgres
+            workers: workers
+
+          - label: debian-testing, Monolith
+            sytest-tag: testing
+            postgres: postgres
+
+          - label: debian-testing, Workers
+            sytest-tag: testing
+            postgres: postgres
+            workers: workers
+
+    container:
+      image: matrixdotorg/sytest-synapse:${{ matrix.sytest-tag }}
+      volumes:
+        # bootstrap.sh expects the sytest source available at /sytest.
+        # TODO Buildkite mounted /sytest as readonly. Can we do this on GHA? Do we need it?
+        - ${{ github.workspace }}/sytest:/sytest
+        # synapse_sytest.sh expects a synapse checkout at /src
+        - ${{ github.workspace }}/synapse:/src
+      env:
+        POSTGRES: ${{ matrix.postgres && 1 }}
+        WORKERS: ${{ matrix.workers && 1 }}
+        BLACKLIST: ${{ (matrix.workers && 'synapse-blacklist-with-workers') || 'sytest-blacklist' }}
+
+    steps:
+      - name: Checkout sytest
+        uses: actions/checkout@v2
+        with:
+          path: sytest
+
+      # TODO the shell script below is nicked from complement. We use this pattern
+      # in a few places. Can we make this an Action so it's easier to reuse?
+      - name: Fetch corresponding synapse branch
+        shell: bash
+        run: |
+          # Attempt to use the version of synapse which best matches the current
+          # build. Depending on whether this is a PR or release, etc. we need to
+          # use different fallbacks.
+          #
+          # 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
+          #    for pull requests, otherwise GITHUB_REF).
+          # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
+          #    (GITHUB_BASE_REF for pull requests).
+          # 3. Use the default synapse branch ("develop").
+          for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "develop"; do
+            # Skip empty branch names and merge commits.
+            if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
+              continue
+            fi
+            (wget -O - "https://github.com/matrix-org/synapse/archive/$BRANCH_NAME.tar.gz" \
+              | tar -xz --strip-components=1 -C /src/) \
+              && echo "Successfully downloaded and extracted $BRANCH_NAME.tar.gz" \
+              && break
+          done
+
+      - name: Prepare blacklist file for running with workers
+        if: ${{ matrix.workers }}
+        run: cat /src/sytest-blacklist /src/.ci/worker-blacklist > /src/synapse-blacklist-with-workers
+
+      - name: Run sytest
+        run: |
+          echo POSTGRES=${POSTGRES:-<NOT SET>}
+          echo WORKERS=${WORKERS:-<NOT SET>}
+          echo BLACKLIST=${BLACKLIST:-<NOT SET>}
+          bash -xe /bootstrap.sh synapse
+
+      - name: Summarise results.tap
+        # Use always() to run this step even if previous ones failed.
+        if: ${{ always() }}
+        run:  /sytest/scripts/tap_to_gha.pl /logs/results.tap
+
+      - name: Upload SyTest logs
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: Sytest Logs - ${{ job.status }} - (Synapse, ${{ matrix.label }})
+          path: |
+            /logs/results.tap
+            /logs/**/*.log*
+
+  dendrite:
+    runs-on: ubuntu-latest
+    name: "Dendrite: ${{ matrix.label }}"
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - label: SQLite
+
+          - label: SQLite, full HTTP APIs
+            api: full-http
+
+          - label: Postgres
+            postgres: postgres
+
+          - label: Postgres, full HTTP APIs
+            postgres: postgres
+            api: full-http
+
+    container:
+      image: matrixdotorg/sytest-dendrite
+      volumes:
+        # bootstrap.sh expects the sytest source available at /sytest.
+        # TODO Buildkite mounted /sytest as readonly. Can we do this on GHA? Do we need it?
+        - ${{ github.workspace }}/sytest:/sytest
+        # synapse_sytest.sh expects a synapse checkout at /src
+        - ${{ github.workspace }}/dendrite:/src
+      env:
+        POSTGRES: ${{ matrix.postgres && 1 }}
+        API: ${{ matrix.api && 1 }}
+
+    steps:
+      - name: Checkout sytest
+        uses: actions/checkout@v2
+        with:
+          path: sytest
+
+      # TODO the shell script below is nicked from complement. We use this pattern
+      # in a few places. Can we make this an Action so it's easier to reuse?
+      - name: Fetch corresponding dendrite branch
+        shell: bash
+        run: |
+          # Attempt to use the version of dendrite which best matches the current
+          # build. Depending on whether this is a PR or release, etc. we need to
+          # use different fallbacks.
+          #
+          # 1. First check if there's a similarly named branch (GITHUB_HEAD_REF
+          #    for pull requests, otherwise GITHUB_REF).
+          # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
+          #    (GITHUB_BASE_REF for pull requests).
+          # 3. Use the default dendrite branch ("master").
+          for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do
+            # Skip empty branch names and merge commits.
+            if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
+              continue
+            fi
+            (wget -O - "https://github.com/matrix-org/dendrite/archive/$BRANCH_NAME.tar.gz" \
+              | tar -xz --strip-components=1 -C /src/) \
+              && echo "Successfully downloaded and extracted $BRANCH_NAME.tar.gz" \
+              && break
+          done
+
+      - name: Run sytest
+        run: |
+          echo POSTGRES=${POSTGRES:-<NOT SET>}
+          echo API=${API:-<NOT SET>}
+          bash -xe /bootstrap.sh dendrite
+
+      - name: Summarise results.tap
+        if: ${{ always() }}
+        run:  /sytest/scripts/tap_to_gha.pl /logs/results.tap
+
+      - name: Upload SyTest logs
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: Sytest Logs - ${{ job.status }} - (Dendrite, ${{ join(matrix.*, ', ') }})
+          path: |
+            /logs/results.tap
+            /logs/**/*.log*

.gitignore

@@ -11,6 +11,7 @@
 /var
 *.db
 \#*
+signingkeyserver.db
 .vscode
 .coverage*
 .idea

README.rst

@@ -157,7 +157,16 @@ suitable for dictConfig_ called ``log.config`` (it can be copied from a running
 synapse) and place it within the homeserver configuration directory
 (``localhost-<port>``).
 
+A blacklist file can be used to mark certain tests as expected to fail. These
+tests will still be run, but their failure will not cause the entire test suite
+to fail. To use a blacklist file pass the ``--test-blacklist-file`` option
+(shortened to ``-B``). For example, if you are testing Synapse, you will
+probably want to use Synapse's sytest-blacklist_::
+
+    ./run-tests.pl -B synapse/sytest-blacklist
+
 .. _dictConfig: https://docs.python.org/2/library/logging.config.html#logging.config.dictConfig
+.. _sytest-blacklist: https://github.com/matrix-org/synapse/blob/develop/sytest-blacklist
 
 Plugins
 ~~~~~~~

cpanfile

@@ -1,20 +1,8 @@
 # vim:ft=perl
 
-# Sodium won't install without this.
-requires 'Alien::Base::ModuleBuild';
-
 requires 'Class::Method::Modifiers';
 
-# workaround for https://github.com/matrix-org/sytest/issues/942:
-# Crypt::NaCl::Sodium won't install with Alien::Sodium 2.0.
-requires 'Alien::Sodium', '<2.0', 'AJGB/Alien-Sodium-1.0.8.0.tar.gz';
-
-# this can be a pain to install.
-#
-# We used to have a libcrypt-nacl-sodium-perl deb, but it was only built for
-# perl 5.14.2.
-#
-requires 'Crypt::NaCl::Sodium';
+requires 'Crypt::Ed25519';
 
 requires 'Data::Dump';
 
@@ -36,6 +24,7 @@ requires 'File::Slurper';
 # Future 0.45 allows you to return immediate values from sequence functions.
 requires 'Future', '>= 0.45';
 requires 'Getopt::Long';
+requires 'HTTP::Request';
 requires 'IO::Async', '>= 0.69';
 requires 'IO::Async::SSL';
 requires 'IO::Socket::IP', '>= 0.04';
@@ -54,7 +43,6 @@ requires 'Module::Pluggable';
 requires 'Net::Async::HTTP', '>= 0.39';
 requires 'Net::Async::HTTP::Server', '>= 0.09';
 requires 'Net::SSLeay', '>= 1.59';
-requires 'Protocol::Matrix', '>= 0.02';
 requires 'Struct::Dumb', '>= 0.04';
 requires 'URI::Escape';
 requires 'YAML::XS';