pipeline-master.yml

# `yq 'explode(.)' .buildkite/pipeline-master.yml` to see expanded anchor/aliases
container:
  kubernetes: &kubernetes
    sidecars:
    - image: us-west1-docker.pkg.dev/ci-compute/buildkite-images/buildkite-dind:v1
      volumeMounts:
        - mountPath: /var/run/
          name: docker-sock
      securityContext:
        privileged: true
        allowPrivilegeEscalation: true
    mirrorVolumeMounts: true # CRITICAL: this must be at the same indentation level as sidecars
    podSpec: &podSpec
      containers:
        - &commandContainer
          image: us-west1-docker.pkg.dev/ci-compute/buildkite-images/buildkite-command-container:v2
          command:
          - |-
            echo "Command step was not overridden."
            exit 1
          volumeMounts:
            - mountPath: /var/run/
              name: docker-sock
          resources:
            requests:
              cpu: 7500m
              memory: 30G
      volumes:
      - name: docker-sock
        emptyDir: {}

agents:
  queue: buildkite-gcp

steps:
  - label: "fossa analyze"
    command: "./scripts/buildkite/fossa.sh"

  - label: ":golang: unit test"
    artifact_paths:
      - ".build/coverage/*.out"
      - ".build/coverage/metadata.txt"
    retry:
      automatic:
        limit: 1
    plugins:
      - kubernetes:
          <<: *kubernetes
          podSpec:
            <<: *podSpec
            containers:
              - <<: *commandContainer
                command:
                - |-
                  # ensure that we are not rebuilding binaries and not regenerating code
                  make .just-build
                  # make install-schema is needed for a server startup test. See main_test.go
                  CASSANDRA_HOST=cassandra make install-schema && make cover_profile
                  ./scripts/buildkite/gen_coverage_metadata.sh .build/coverage/metadata.txt
      - docker-compose#v3.0.0:
          run: unit-test
          config: docker/buildkite/docker-compose.yml

  - label: ":golangci-lint: validate code is clean"
    artifact_paths: []
    retry:
      automatic:
        limit: 1
    plugins:
      - kubernetes:
          <<: *kubernetes
          podSpec:
            <<: *podSpec
            containers:
              - <<: *commandContainer
                command:
                - |-
                  ./scripts/buildkite/golint.sh
      - docker-compose#v3.0.0:
          run: coverage-report
          config: docker/buildkite/docker-compose.yml

  - label: ":golang: integration test with cassandra"
    artifact_paths:
      - ".build/coverage/*.out"
    retry:
      automatic:
        limit: 1
    plugins:
      - kubernetes:
          <<: *kubernetes
          podSpec:
            <<: *podSpec
            containers:
              - <<: *commandContainer
                command:
                - |-
                  make cover_integration_profile
      - docker-compose#v3.0.0:
          run: integration-test-cassandra
          config: docker/buildkite/docker-compose.yml

  - label: ":golang: integration test with cassandra with ElasticSearch V7"
    artifact_paths:
      - ".build/coverage/*.out"
    retry:
      automatic:
        limit: 1
    plugins:
      - kubernetes:
          <<: *kubernetes
          podSpec:
            <<: *podSpec
            containers:
              - <<: *commandContainer
                command:
                - |-
                  make cover_integration_profile
      - docker-compose#v3.0.0:
          run: integration-test-cassandra
          config: docker/buildkite/docker-compose-es7.yml

  - label: ":golang: integration test with cassandra with pinot"
    artifact_paths:
      - ".build/coverage/*.out"
    retry:
      automatic:
        limit: 1
    plugins:
      - kubernetes:
          <<: *kubernetes
          podSpec:
            <<: *podSpec
            containers:
              - <<: *commandContainer
                command:
                  - |-
                    # ensure that we are not rebuilding binaries and not regenerating code
                    make .just-build
                    go test -timeout 600s -run ^TestPinotIntegrationSuite -tags pinotintegration -count 1 -v github.com/uber/cadence/host
      - docker-compose#v3.0.0:
          run: integration-test-cassandra-pinot
          config: docker/buildkite/docker-compose-pinot.yml


  - label: ":golang: integration test with cassandra with OpenSearch v2"
    artifact_paths:
      - ".build/coverage/*.out"
    retry:
      automatic:
        limit: 1
    plugins:
      - kubernetes:
          <<: *kubernetes
          podSpec:
            <<: *podSpec
            containers:
              - <<: *commandContainer
                command:
                - |-
                  make cover_integration_profile
      - docker-compose#v3.0.0:
          run: integration-test-cassandra
          config: docker/buildkite/docker-compose-opensearch2.yml

  - label: ":golang: integration ndc test with cassandra"
    artifact_paths:
      - ".build/coverage/*.out"
    retry:
      automatic:
        limit: 1
    plugins:
      - kubernetes:
          <<: *kubernetes
          podSpec:
            <<: *podSpec
            containers:
              - <<: *commandContainer
                command:
                - |-
                  make cover_ndc_profile
      - docker-compose#v3.0.0:
          run: integration-test-ndc-cassandra
          config: docker/buildkite/docker-compose.yml

  - label: ":golang: integration test with mysql"
    artifact_paths:
      - ".build/coverage/*.out"
    retry:
      automatic:
        limit: 1
    plugins:
      - kubernetes:
          <<: *kubernetes
          podSpec:
            <<: *podSpec
            containers:
              - <<: *commandContainer
                command:
                - |-
                  make cover_integration_profile
      - docker-compose#v3.0.0:
          run: integration-test-mysql
          config: docker/buildkite/docker-compose.yml

  - label: ":golang: integration ndc test with mysql"
    artifact_paths:
      - ".build/coverage/*.out"
    retry:
      automatic:
        limit: 1
    plugins:
      - kubernetes:
          <<: *kubernetes
          podSpec:
            <<: *podSpec
            containers:
              - <<: *commandContainer
                command:
                - |-
                  make cover_ndc_profile
      - docker-compose#v3.0.0:
          run: integration-test-ndc-mysql
          config: docker/buildkite/docker-compose.yml

  - label: ":golang: integration test with postgres"
    artifact_paths:
      - ".build/coverage/*.out"
    retry:
      automatic:
        limit: 1
    plugins:
      - kubernetes:
          <<: *kubernetes
          podSpec:
            <<: *podSpec
            containers:
              - <<: *commandContainer
                command:
                - |-
                  make cover_integration_profile
      - docker-compose#v3.0.0:
          run: integration-test-postgres
          config: docker/buildkite/docker-compose.yml

  - label: ":golang: integration ndc test with postgres"
    artifact_paths:
      - ".build/coverage/*.out"
    retry:
      automatic:
        limit: 1
    plugins:
      - kubernetes:
          <<: *kubernetes
          podSpec:
            <<: *podSpec
            containers:
              - <<: *commandContainer
                command:
                - |-
                  make cover_ndc_profile
      - docker-compose#v3.0.0:
          run: integration-test-ndc-postgres
          config: docker/buildkite/docker-compose.yml

  - label: ":golang: async wf integration test with kafka"
    retry:
      automatic:
        limit: 1
    plugins:
      - kubernetes:
          <<: *kubernetes
          podSpec:
            <<: *podSpec
            containers:
              - <<: *commandContainer
                command:
                - |-
                  # ensure that we are not rebuilding binaries and not regenerating code
                  make .just-build
                  go test -timeout 60s -run ^TestAsyncWFIntegrationSuite -tags asyncwfintegration -count 1 -v github.com/uber/cadence/host
      - docker-compose#v3.0.0:
          run: integration-test-async-wf
          config: docker/buildkite/docker-compose.yml

  - wait

  - label: ":docker: build and push master"
    retry:
      automatic:
        limit: 1
    plugins:
      - kubernetes:
          <<: *kubernetes
          podSpec:
            <<: *podSpec
            containers:
              - <<: *commandContainer
                command:
                - |-
                  scripts/buildkite/docker-push.sh
                env:
                  - name: DOCKER_LOGIN_PASSWORD
                    valueFrom:
                      secretKeyRef:
                        name: ubercadence-dockerhub
                        key: password
      - docker-login#v2.0.1:
          username: ubercadence
          password-env: DOCKER_LOGIN_PASSWORD