pipeline.template.yml

# Copyright Materialize, Inc. and contributors. All rights reserved.
#
# Use of this software is governed by the Business Source License
# included in the LICENSE file at the root of this repository.
#
# As of the Change Date specified in that file, in accordance with
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0.

# This file is processed by mkpipeline.py to trim unnecessary steps in PR
# builds. The inputs for steps using the `mzcompose` plugin are computed
# automatically. Inputs for other steps need to be manually listed in the
# `inputs` key.

dag: true

env:
  CI_BUILDER_SCCACHE: 1

steps:
  - id: build-x86_64
    label: Build x86_64
    command: bin/ci-builder run stable bin/pyactivate -m ci.test.build
    inputs:
      - "*"
    timeout_in_minutes: 60
    priority: 1
    agents:
      queue: builder-linux-x86_64

  - id: build-aarch64
    label: Build aarch64
    command: bin/ci-builder run stable bin/pyactivate -m ci.test.build
    inputs:
      - "*"
    timeout_in_minutes: 60
    priority: 1
    agents:
      queue: builder-linux-aarch64
    coverage: skip

  - id: check-merge-with-target
    label: Merge skew cargo check
    command: ci/test/check-merge-with-target.sh
    inputs:
      - Cargo.lock
      - "**/Cargo.toml"
      - "**/*.rs"
    timeout_in_minutes: 30
    agents:
      queue: linux-x86_64
    if: "build.pull_request.id != null"
    coverage: skip

  - id: devel-docker-tags
    label: Tag development docker images
    command: bin/ci-builder run stable bin/pyactivate -m ci.test.dev_tag
    inputs:
      - "*"
    depends_on:
      - build-x86_64
      - build-aarch64
    timeout_in_minutes: 10
    agents:
      queue: linux-x86_64
    coverage: skip

  - id: lint-fast
    label: Lint and rustfmt
    command: bin/ci-builder run stable ci/test/lint-fast.sh
    inputs:
      - "*"
    timeout_in_minutes: 10
    agents:
      queue: linux-x86_64
    coverage: skip

  - id: lint-slow
    label: Clippy and doctests
    command: bin/ci-builder run stable ci/test/lint-slow.sh
    inputs:
      - Cargo.lock
      - "**/Cargo.toml"
      - "**/*.rs"
    timeout_in_minutes: 30
    agents:
      queue: builder-linux-x86_64
    coverage: skip

  - id: lint-macos
    label: macOS Clippy
    command: cargo clippy --all-targets -- -D warnings
    env:
      CARGO_INCREMENTAL: "0"
      RUSTUP_TOOLCHAIN: $RUST_VERSION
    inputs:
      - Cargo.lock
      - "**/Cargo.toml"
      - "**/*.rs"
    timeout_in_minutes: 30
    agents:
      queue: mac
    coverage: skip

  - id: lint-deps
    label: Lint dependencies
    command: bin/ci-builder run stable ci/test/lint-deps.sh
    inputs:
      - Cargo.lock
      - "**/Cargo.toml"
      - "**/*.rs"
    timeout_in_minutes: 30
    agents:
      queue: linux-x86_64
    coverage: skip

  - id: lint-docs
    label: Lint docs
    command: bin/ci-builder run stable ci/test/lint-docs.sh
    inputs: [doc/user]
    timeout_in_minutes: 30
    agents:
      queue: linux-x86_64
    coverage: skip

  - id: preview-docs
    label: Preview docs
    command: bin/ci-builder run stable ci/test/preview-docs.sh
    inputs: [doc/user]
    timeout_in_minutes: 30
    agents:
      queue: linux-x86_64
    coverage: skip

  - id: cargo-test
    label: Cargo test
    timeout_in_minutes: 30
    artifact_paths: [junit_*.xml, target/nextest/ci/junit_cargo-test.xml]
    inputs:
      - Cargo.lock
      - "**/Cargo.toml"
      - "**/*.rs"
      - "**/*.proto"
      - "**/testdata/**"
    env:
      AWS_DEFAULT_REGION: "us-east-2"
    plugins:
      - ./ci/plugins/scratch-aws-access: ~
      - ./ci/plugins/mzcompose:
          composition: cargo-test
    agents:
      queue: builder-linux-x86_64
    coverage: skip

  - id: miri-test
    label: Miri test
    command: bin/ci-builder run nightly ci/test/cargo-test-miri.sh
    inputs: [src]
    timeout_in_minutes: 30
    agents:
      queue: builder-linux-x86_64
    coverage: skip

  - id: testdrive
    label: Testdrive %n
    depends_on: build-x86_64
    timeout_in_minutes: 30
    inputs: [test/testdrive]
    parallelism: 4
    # Don't need the junit_mzcompose_*.xml for this
    artifact_paths: [junit_testdrive_*.xml, test/testdrive/junit_testdrive_*.xml]
    plugins:
      - ./ci/plugins/scratch-aws-access: ~
      - ./ci/plugins/mzcompose:
          composition: testdrive
          args: [--aws-region=us-east-2]
    agents:
      queue: linux-x86_64

  - id: kafka-persistence
    label: Kafka persistence
    depends_on: build-x86_64
    timeout_in_minutes: 30
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: persistence
          run: kafka-sources
    agents:
      queue: linux-x86_64

  - id: table-persistence
    label: Table persistence
    depends_on: build-x86_64
    timeout_in_minutes: 30
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: persistence
          run: user-tables
    agents:
      queue: linux-x86_64

  - id: cluster-tests
    label: Cluster tests
    depends_on: build-x86_64
    timeout_in_minutes: 20
    artifact_paths: junit_*.xml
    inputs: [test/cluster]
    plugins:
      - ./ci/plugins/mzcompose:
          composition: cluster
    agents:
      queue: linux-x86_64

  - id: cluster-isolation
    label: Cluster isolation test
    depends_on: build-x86_64
    timeout_in_minutes: 10
    inputs: [test/cluster-isolation]
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: cluster-isolation
    agents:
      queue: linux-x86_64

  - id: replica-isolation
    label: Replica isolation
    depends_on: build-x86_64
    timeout_in_minutes: 20
    inputs: [test/replica-isolation]
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: replica-isolation
    agents:
      queue: linux-x86_64

  - id: kafka-ssl
    label: Kafka SSL smoke test
    depends_on: build-x86_64
    timeout_in_minutes: 30
    inputs: [test/kafka-ssl/smoketest.td]
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: kafka-ssl
    agents:
      queue: linux-x86_64

  - id: kafka-sasl-plain
    label: Kafka SASL PLAIN smoke test
    depends_on: build-x86_64
    timeout_in_minutes: 30
    inputs: [test/kafka-sasl-plain/smoketest.td]
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: kafka-sasl-plain
    agents:
      queue: linux-x86_64

  - id: sqllogictest-fast
    label: Fast SQL logic tests
    depends_on: build-x86_64
    timeout_in_minutes: 30
    inputs: [test/sqllogictest]
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: sqllogictest
    agents:
      queue: linux-x86_64

  - id: chbench-demo
    label: chbench smoke test
    depends_on: build-x86_64
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: chbench
          args: [--run-seconds=10, --wait]
    timeout_in_minutes: 30
    agents:
      queue: linux-x86_64

  - id: restarts
    label: Restart test
    depends_on: build-x86_64
    timeout_in_minutes: 30
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: restart
    agents:
      queue: linux-x86_64

  - id: legacy-upgrade
    label: Legacy upgrade tests
    depends_on: build-x86_64
    timeout_in_minutes: 60
    artifact_paths: junit_*.xml
    inputs: [doc/user/content/releases]
    plugins:
      - ./ci/plugins/mzcompose:
          composition: upgrade
    agents:
      queue: linux-x86_64

  - id: metabase-demo
    label: Metabase smoke test
    depends_on: build-x86_64
    timeout_in_minutes: 10
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: metabase
    agents:
      queue: linux-x86_64

  - id: dbt-materialize
    label: dbt-materialize tests
    depends_on: build-x86_64
    timeout_in_minutes: 10
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: dbt-materialize
    agents:
      queue: linux-x86_64

  - id: debezium-postgres
    label: Debezium Postgres tests
    depends_on: build-x86_64
    timeout_in_minutes: 30
    inputs: [test/debezium]
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: debezium
          run: postgres
    agents:
      queue: linux-x86_64

  - id: debezium-sql-server
    label: Debezium SQL Server tests
    depends_on: build-x86_64
    timeout_in_minutes: 30
    inputs: [test/debezium]
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: debezium
          run: sql-server
    agents:
      queue: linux-x86_64

  - id: debezium-mysql
    label: Debezium MySQL tests
    depends_on: build-x86_64
    timeout_in_minutes: 30
    inputs: [test/debezium]
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: debezium
          run: mysql
    agents:
      queue: linux-x86_64

  - id: pg-cdc
    label: Postgres CDC tests
    depends_on: build-x86_64
    timeout_in_minutes: 30
    inputs: [test/pg-cdc]
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: pg-cdc
    agents:
      queue: linux-x86_64

  - id: pg-cdc-resumption
    label: Postgres CDC resumption tests
    depends_on: build-x86_64
    timeout_in_minutes: 30
    inputs: [test/pg-cdc-resumption]
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: pg-cdc-resumption
    agents:
      queue: linux-x86_64

  - id: ssh-connection
    label: SSH connection tests
    depends_on: build-x86_64
    timeout_in_minutes: 10
    inputs: [test/ssh-connection]
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: ssh-connection
    agents:
      queue: linux-x86_64

  - id: kafka-resumption
    label: Kafka resumption tests
    depends_on: build-x86_64
    timeout_in_minutes: 30
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: kafka-resumption
    agents:
      queue: linux-x86_64

  - id: zippy-kafka-sources-short
    label: "Short Zippy"
    depends_on: build-x86_64
    inputs: [misc/python/materialize/zippy]
    timeout_in_minutes: 30
    artifact_paths: junit_*.xml
    agents:
      queue: linux-x86_64
    plugins:
      - ./ci/plugins/mzcompose:
          composition: zippy
          args: [--scenario=KafkaSources, --actions=200]

  - id: kafka-exactly-once
    label: Kafka exactly-once tests
    depends_on: build-x86_64
    timeout_in_minutes: 30
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: kafka-exactly-once
    agents:
      queue: linux-x86_64

  - id: persist-maelstrom
    label: Maelstrom coverage of persist
    depends_on: build-x86_64
    timeout_in_minutes: 10
    inputs: [test/persist]
    artifact_paths: [test/persist/maelstrom/**/*.log, junit_*.xml]
    plugins:
      - ./ci/plugins/mzcompose:
          composition: persist
          args: [--consensus=mem, --blob=mem]
    agents:
      queue: linux-x86_64
    coverage: skip # TODO(def-): Should we support coverage for persist-maelstrom?

  - id: checks-drop-create-default-replica
    label: "Checks + DROP/CREATE replica"
    depends_on: build-x86_64
    inputs: [misc/python/materialize/checks]
    timeout_in_minutes: 30
    artifact_paths: junit_*.xml
    agents:
      queue: linux-x86_64
    plugins:
      - ./ci/plugins/mzcompose:
          composition: platform-checks
          args: [--scenario=DropCreateDefaultReplica, "--seed=$BUILDKITE_JOB_ID"]

  - id: checks-restart-clusterd-compute
    label: "Checks + restart clusterd compute"
    depends_on: build-x86_64
    inputs: [misc/python/materialize/checks]
    timeout_in_minutes: 30
    artifact_paths: junit_*.xml
    agents:
      queue: linux-x86_64
    plugins:
      - ./ci/plugins/mzcompose:
          composition: platform-checks
          args: [--scenario=RestartClusterdCompute, "--seed=$BUILDKITE_JOB_ID"]

  - id: checks-restart-entire-mz
    label: "Checks + restart of the entire Mz"
    depends_on: build-x86_64
    inputs: [misc/python/materialize/checks]
    timeout_in_minutes: 30
    artifact_paths: junit_*.xml
    agents:
      queue: linux-x86_64
    plugins:
      - ./ci/plugins/mzcompose:
          composition: platform-checks
          args: [--scenario=RestartEntireMz, "--seed=$BUILDKITE_JOB_ID"]

  - id: checks-restart-environmentd-clusterd-storage
    label: "Checks + restart of environmentd & storage clusterd"
    depends_on: build-x86_64
    inputs: [misc/python/materialize/checks]
    timeout_in_minutes: 30
    artifact_paths: junit_*.xml
    agents:
      queue: linux-x86_64
    plugins:
      - ./ci/plugins/mzcompose:
          composition: platform-checks
          args: [--scenario=RestartEnvironmentdClusterdStorage, "--seed=$BUILDKITE_JOB_ID"]

  - id: checks-kill-clusterd-storage
    label: "Checks + kill storage clusterd"
    depends_on: build-x86_64
    inputs: [misc/python/materialize/checks]
    timeout_in_minutes: 30
    artifact_paths: junit_*.xml
    agents:
      queue: linux-x86_64
    plugins:
      - ./ci/plugins/mzcompose:
          composition: platform-checks
          args: [--scenario=KillClusterdStorage, "--seed=$BUILDKITE_JOB_ID"]

  - id: checks-restart-source-postgres
    label: "Checks + restart source Postgres"
    depends_on: build-x86_64
    inputs: [misc/python/materialize/checks]
    timeout_in_minutes: 30
    artifact_paths: junit_*.xml
    agents:
      queue: linux-x86_64
    plugins:
      - ./ci/plugins/mzcompose:
          composition: platform-checks
          args: [--scenario=RestartSourcePostgres, --check=DebeziumPostgres, --check=PgCdc]

  - id: cloudtest
    label: Cloudtest
    depends_on: build-x86_64
    parallelism: 3
    timeout_in_minutes: 30
    artifact_paths: junit_*.xml
    inputs: [test/cloudtest, misc/python/materialize/cloudtest]
    plugins:
      - ./ci/plugins/scratch-aws-access: ~
      - ./ci/plugins/cloudtest:
          args: [--exitfirst, -m, "not long", --aws-region=us-east-2, test/cloudtest/]
    coverage: skip # TODO(def-): Should we support coverage for cloudtest?

  - id: source-sink-errors
    label: "Source/Sink Error Reporting"
    artifact_paths: junit_*.xml
    depends_on: build-x86_64
    timeout_in_minutes: 30
    agents:
      queue: linux-x86_64
    plugins:
      - ./ci/plugins/mzcompose:
          composition: source-sink-errors

  - id: storage-usage
    label: "Storage Usage Table Test"
    artifact_paths: junit_*.xml
    depends_on: build-x86_64
    timeout_in_minutes: 30
    agents:
      queue: linux-x86_64
    plugins:
      - ./ci/plugins/mzcompose:
          composition: storage-usage

  - id: lang-csharp
    label: ":csharp: tests"
    depends_on: build-x86_64
    timeout_in_minutes: 10
    inputs: [test/lang/csharp]
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: csharp
    agents:
      queue: linux-x86_64

  - id: lang-js
    label: ":js: tests"
    depends_on: build-x86_64
    timeout_in_minutes: 10
    inputs: [test/lang/js]
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: js
    agents:
      queue: linux-x86_64

  - id: lang-java
    label: ":java: tests"
    depends_on: build-x86_64
    timeout_in_minutes: 10
    inputs: [test/lang/java]
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: java
    agents:
      queue: linux-x86_64

  - id: lang-python
    label: ":python: tests"
    depends_on: build-x86_64
    timeout_in_minutes: 10
    inputs: [test/lang/python]
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: python
    agents:
      queue: linux-x86_64

  - id: feature-benchmark-kafka-only
    label: "Feature benchmark (Kafka only)"
    depends_on: build-x86_64
    timeout_in_minutes: 30
    artifact_paths: junit_*.xml
    plugins:
      - ./ci/plugins/mzcompose:
          composition: feature-benchmark
          args: [--scenario=KafkaUpsertUnique]
    coverage: skip

  - wait: ~
    continue_on_failure: true

  - id: coverage-pr-analyze
    label: Analyze code coverage for PR
    command: bin/ci-builder run stable ci/test/coverage_report.sh
    inputs: ["*"]
    priority: 1
    agents:
      queue: linux-x86_64
    coverage: only

  - wait: ~
    continue_on_failure: true

  - id: analyze
    label: Analyze tests
    inputs: ["*"]
    plugins:
      - junit-annotate#v2.0.2:
          artifacts: "*junit_*.xml"
          job-uuid-file-pattern: _([^_]*).xml
    priority: 1
    agents:
      queue: linux-x86_64

  - wait: ~

  - id: deploy
    label: Deploy
    trigger: deploy
    async: true
    branches: "main v*.*"
    build:
      commit: "$BUILDKITE_COMMIT"
      branch: "$BUILDKITE_BRANCH"
      env:
        BUILDKITE_TAG: "$BUILDKITE_TAG"
    coverage: skip