e2e.yml

name: End-to-End Tests

on:
  push:
    branches: [main]
  pull_request:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}

jobs:
  diff_check:
    uses: ./.github/workflows/skip-ci.yml

  cancel-previous-workflow:
    runs-on: ubuntu-latest
    needs: [diff_check]
    if: ${{ needs.diff_check.outputs.skip_ci != 'true' }}
    steps:
      - name: Cancel Previous Runs
        uses: styfle/cancel-workflow-action@85880fa0301c86cca9da44039ee3bb12d3bedbfa # pin@0.12.1
        with:
          access_token: ${{ github.token }}

  metrics:
    runs-on: ${{ matrix.runs-on }}
    needs: [diff_check]
    if: ${{ needs.diff_check.outputs.skip_ci != 'true' }}
    env:
      SENTRY_DISABLE_AUTO_UPLOAD: 'true'
    strategy:
      # we want that the matrix keeps running, default is to cancel them if it fails.
      fail-fast: false
      matrix:
        rn-architecture: ['legacy', 'new']
        platform: ["ios", "android"]
        include:
          - platform: ios
            runs-on: macos-12
            name: iOS
            appPlain: test/perf/test-app-plain.ipa
          - platform: android
            runs-on: ubuntu-latest
            name: Android
            appPlain: test/perf/TestAppPlain/android/app/build/outputs/apk/release/app-release.apk
    steps:
      - uses: actions/checkout@v4

      - uses: ./.github/actions/disk-cleanup
        if: ${{ matrix.platform == 'android' }}

      - run: sudo xcode-select -s /Applications/Xcode_14.2.app/Contents/Developer
        if: ${{ matrix.platform == 'ios' }}

      - uses: actions/setup-node@v4
        with:
          node-version: 18
      - uses: actions/setup-java@v4
        with:
          java-version: '17'
          distribution: "adopt"
      - name: Gradle cache
        uses: gradle/gradle-build-action@v3
      - name: Install Global Dependencies
        run: yarn global add react-native-cli @sentry/cli yalc
      - uses: actions/cache@v4
        id: deps-cache
        with:
          path: |
            node_modules
            test/perf/TestAppSentry/node_modules
          key: ${{ github.workflow }}-${{ github.job }}-${{ hashFiles('yarn.lock', 'test/perf/TestAppSentry/yarn.lock') }}
      - name: Install Dependencies
        if: steps.deps-cache.outputs['cache-hit'] != 'true'
        run: yarn install
      - name: Build SDK
        run: yarn build
      - name: Package SDK
        run: yalc publish
      - uses: actions/setup-node@v4
        with:
          node-version: 18
      - uses: actions/cache@v4
        id: app-plain-cache
        with:
          path: ${{ matrix.appPlain }}
          # if the whole plain app folder is hashed the cache is never hit as there are files generated in the folder
          # the cache key is calculated both at cache retrieval and save time
          # hashFiles fails when there are rn android new arch generated files in the folder (exact reason unknown)
          key: ${{ github.workflow }}-${{ github.job }}-appplain-${{ matrix.platform }}-${{ matrix.rn-architecture }}-${{ hashFiles('test/perf/TestAppPlain/yarn.lock') }}
      - name: Build app plain
        if: steps.app-plain-cache.outputs['cache-hit'] != 'true'
        working-directory: ./test/perf/TestAppPlain
        run: |
          yarn install
          cd ${{ matrix.platform }}
          if [[ "${{ matrix.platform }}" == "android" ]]; then
            if [[ ${{ matrix.rn-architecture }} == 'new' ]]; then
              perl -i -pe's/newArchEnabled=false/newArchEnabled=true/g' gradle.properties
            fi
            ./gradlew assembleRelease
          else
            export PRODUCTION=1
            if [[ ${{ matrix.rn-architecture }} == 'new' ]]; then
              export RCT_NEW_ARCH_ENABLED=1
            fi
            pod install
            cd ../..
            fastlane build_perf_test_app_plain
          fi
        env:
          APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }}
          APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }}
          APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }}
          FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }}
          MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }}
          MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
          MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }}
      - name: Build app with Sentry
        working-directory: ./test/perf/TestAppSentry
        run: |
          yalc add @sentry/react-native
          yarn install
          cd ${{ matrix.platform }}
          if [[ "${{ matrix.platform }}" == "android" ]]; then
            if [[ ${{ matrix.rn-architecture }} == 'new' ]]; then
              perl -i -pe's/newArchEnabled=false/newArchEnabled=true/g' gradle.properties
            fi
            ./gradlew assembleRelease
          else
            export PRODUCTION=1
            if [[ ${{ matrix.rn-architecture }} == 'new' ]]; then
              export RCT_NEW_ARCH_ENABLED=1
            fi
            pod install
            cd ../..
            fastlane build_perf_test_app_sentry
            cd TestAppSentry
          fi
          # Remove after build so that the cache isn't invaldiated due to the changes in yarn.lock.
          yarn remove @sentry/react-native
        env:
          APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }}
          APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }}
          APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }}
          FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }}
          MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }}
          MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
          MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }}
      - name: Collect apps metrics
        uses: getsentry/action-app-sdk-overhead-metrics@v1
        with:
          name: ${{ matrix.name }} (${{ matrix.rn-architecture }})
          config: ./test/perf/metrics-${{ matrix.platform }}.yml
          sauce-user: ${{ secrets.SAUCE_USERNAME }}
          sauce-key: ${{ secrets.SAUCE_ACCESS_KEY }}

  react-native-build:
    name: Build RN ${{ matrix.rn-version }} ${{ matrix.rn-architecture }} ${{ matrix.engine }} ${{ matrix.platform }} ${{ matrix.build-type }} ${{ matrix.ios-use-frameworks }}
    runs-on: ${{ matrix.runs-on }}
    needs: [diff_check]
    if: ${{ needs.diff_check.outputs.skip_ci != 'true' }}
    env:
      RN_VERSION: ${{ matrix.rn-version }}
      RN_ENGINE: ${{ matrix.engine }}
      USE_FRAMEWORKS: ${{ matrix.ios-use-frameworks }}
      PRODUCTION: ${{ matrix.build-type == 'production' && '1' || '0' }}
      RCT_NEW_ARCH_ENABLED: ${{ matrix.rn-architecture == 'new' && '1' || '0' }}
      IOS_RUNTIME: ${{ matrix.runtime }}
      IOS_DEVICE: ${{ matrix.device }}
    strategy:
      fail-fast: false # keeps matrix running if one fails
      matrix:
        rn-version: ['0.65.3', '0.73.9']
        rn-architecture: ['legacy', 'new']
        platform: ['android', 'ios']
        build-type: ['production']
        ios-use-frameworks: ['no', 'static', 'dynamic']
        engine: ['hermes', 'jsc']
        include:
          - platform: ios
            rn-version: '0.73.9'
            runs-on: macos-14 # uses m1 https://github.blog/changelog/2024-01-30-github-actions-macos-14-sonoma-is-now-available/
            runtime: 'latest'
            device: 'iPhone 14'
          - platform: ios
            rn-version: '0.65.3'
            runs-on: macos-12
            runtime: 'latest'
            device: 'iPhone 14'
          - platform: android
            runs-on: ubuntu-latest
        exclude:
          # exclude JSC for new RN versions (keeping the matrix manageable)
          - rn-version: '0.73.9'
            engine: 'jsc'
          # exclude all rn versions lower than 0.70.0 for new architecture
          - rn-version: '0.65.3'
            rn-architecture: 'new'
          # exlude old rn version for use frameworks builds (to minimalize the matrix)
          - rn-version: '0.65.3'
            platform: 'ios'
            ios-use-frameworks: 'static'
          - rn-version: '0.65.3'
            platform: 'ios'
            ios-use-frameworks: 'dynamic'
          # use frameworks is ios only feature
          - platform: 'android'
            ios-use-frameworks: 'static'
          - platform: 'android'
            ios-use-frameworks: 'dynamic'
          # exclude new rn architecture and dynamic frameworks
          - rn-architecture: 'new'
            ios-use-frameworks: 'dynamic'

    steps:
      - uses: actions/checkout@v4

      - uses: ./.github/actions/disk-cleanup
        if: ${{ matrix.platform == 'android' }}

      - name: Sentry Release
        run: |
          SENTRY_RELEASE_CANDIDATE=$(echo 'e2e/${{ github.ref }}' | perl -pe 's/\//-/g')
          echo "SENTRY_RELEASE=$SENTRY_RELEASE_CANDIDATE" >> $GITHUB_ENV

      - name: Sentry Dist
        run: |
          SENTRY_DIST_CANDIDATE=${{ matrix.rn-version }}-${{ matrix.rn-architecture }}-${{ matrix.engine }}-${{ matrix.platform }}-${{ matrix.build-type }}-${{ matrix.ios-use-frameworks }}-${GITHUB_SHA:0:8}
          echo "SENTRY_DIST=$SENTRY_DIST_CANDIDATE" >> $GITHUB_ENV

      - name: Sentry Envs
        run: |
          echo "SENTRY_RELEASE=$SENTRY_RELEASE"
          echo "SENTRY_DIST=$SENTRY_DIST"

      - run: sudo xcode-select -s /Applications/Xcode_14.2.app/Contents/Developer
        if: ${{ matrix.platform == 'ios' && matrix.rn-version == '0.65.3' }}

      - name: Enable Corepack # RN 0.73.9 and newer are using yarn 3 which requires corepack
        if: ${{ matrix.rn-version == '0.73.9' }}
        run: corepack enable

      - uses: actions/setup-node@v4
        with:
          node-version: 18

      - uses: actions/setup-java@v4
        with:
          java-version: ${{ matrix.rn-version == '0.65.3' && '11' || '17' }}
          distribution: 'adopt'

      - name: Gradle cache
        uses: gradle/gradle-build-action@v3

      - name: Setup Global Tools
        run: |
          yarn global add yalc semver

      - name: Setup Global Xcode Tools
        if: ${{ matrix.platform == 'ios' }}
        run: which xcbeautify || brew install xcbeautify

      - name: NPM cache SDK
        uses: actions/cache@v4
        id: deps-cache
        with:
          path: node_modules
          key: ${{ github.workflow }}-${{ github.job }}-npm-${{ hashFiles('yarn.lock') }}

      - name: NPM cache E2E Tests Library
        uses: actions/cache@v4
        id: deps-cache-e2e-library
        with:
          path: test/e2e/node_modules
          key: ${{ github.workflow }}-${{ github.job }}-npm-${{ hashFiles('test/e2e/yarn.lock') }}

      - name: Install SDK JS Dependencies
        if: steps.deps-cache.outputs['cache-hit'] != 'true'
        run: yarn install

      - name: Install E2E Tests Library JS Dependencies
        if: steps.deps-cache-e2e-library.outputs['cache-hit'] != 'true'
        working-directory: test/e2e
        run: yarn install

      - uses: actions/setup-node@v4
        if: ${{ matrix.rn-version == '0.65.3' }}
        with:
          node-version: 16

      - name: Setup Plain RN ${{ matrix.rn-version }} App
        run: ./scripts/e2e.mjs ${{ matrix.platform }} --create

      - uses: ruby/setup-ruby@v1
        if: ${{ matrix.platform == 'ios' }}
        with:
          working-directory: test/react-native/versions/${{ matrix.rn-version }}/RnDiffApp
          ruby-version: '3.3.0' # based on what is used in the sample
          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
          cache-version: 1 # cache the installed gems

      - name: Build Plain RN ${{ matrix.rn-version }} App
        run: ./scripts/e2e.mjs ${{ matrix.platform }} --build

      - name: Upload App
        if: matrix.build-type == 'production'
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.rn-version }}-${{ matrix.rn-architecture }}-${{ matrix.engine }}-${{ matrix.platform }}-${{ matrix.build-type }}-${{ matrix.ios-use-frameworks }}-app-package
          path: test/e2e/RnDiffApp.ap*
          retention-days: 1

      - name: Upload logs
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: rn-build-logs-${{ matrix.rn-version }}-${{ matrix.rn-architecture }}-${{ matrix.engine }}-${{ matrix.platform }}-${{ matrix.build-type }}-${{ matrix.ios-use-frameworks }}
          path: test/react-native/versions/${{ matrix.rn-version }}/RnDiffApp/ios/*.log

  react-native-test:
    name:
      Test RN ${{ matrix.rn-version }} ${{ matrix.rn-architecture }} ${{ matrix.engine }} ${{ matrix.platform }} ${{
      matrix.build-type }} ${{ matrix.ios-use-frameworks }}
    needs: [react-native-build, diff_check]
    if: ${{ needs.diff_check.outputs.skip_ci != 'true' }}

    runs-on: ${{ matrix.runs-on }}
    strategy:
      fail-fast: false # keeps matrix running if one fails
      matrix:
        rn-version: ['0.65.3', '0.73.9']
        rn-architecture: ['legacy', 'new']
        platform: ['android', 'ios']
        build-type: ['production']
        ios-use-frameworks: ['no'] # test only no framworks
        engine: ['hermes', 'jsc']
        include:
          - platform: ios
            rn-version: '0.73.9'
            runs-on: macos-14 # uses m1 https://github.blog/changelog/2024-01-30-github-actions-macos-14-sonoma-is-now-available/
            runtime: 'latest'
            device: 'iPhone 14'
          - platform: ios
            rn-version: '0.65.3'
            runs-on: macos-latest
            runtime: 'latest'
            device: 'iPhone 14'
          - platform: android
            runs-on: ubuntu-latest
        exclude:
          # exclude all rn versions lower than 0.70.0 for new architecture
          - rn-version: '0.65.3'
            rn-architecture: 'new'
          # e2e test only the default combinations
          - rn-version: '0.65.3'
            engine: 'hermes'
          - rn-version: '0.73.9'
            engine: 'jsc'
          # E2E timeout due to a race condition https://github.com/facebook/react-native/issues/42123#issuecomment-1881203719
          - rn-version: '0.73.9'
            platform: 'ios'
            rn-architecture: 'new'
    env:
      PLATFORM: ${{ matrix.platform }}
      DEVICE: ${{ matrix.device }}
    steps:
      - uses: actions/checkout@v4

      - uses: ./.github/actions/disk-cleanup
        if: ${{ matrix.platform == 'android' }}

      - name: Setup Global Xcode Tools
        if: ${{ matrix.platform == 'ios' }}
        run: which xcbeautify || brew install xcbeautify

      - name: Download App Package
        if: matrix.build-type == 'production'
        uses: actions/download-artifact@v4
        with:
          name: ${{ matrix.rn-version }}-${{ matrix.rn-architecture }}-${{ matrix.engine }}-${{ matrix.platform }}-${{ matrix.build-type }}-${{ matrix.ios-use-frameworks }}-app-package
          path: test/e2e

      - uses: actions/setup-node@v4
        with:
          node-version: 20

      - uses: actions/setup-java@v4
        with:
          java-version: '17'
          distribution: 'adopt'

      - name: Gradle cache
        uses: gradle/gradle-build-action@v3

      - name: Setup KVM
        if: ${{ matrix.platform == 'android' }}
        shell: bash
        run: |
          # check if virtualization is supported...
          sudo apt install -y --no-install-recommends cpu-checker coreutils && echo "CPUs=$(nproc --all)" && kvm-ok
          # allow access to KVM to run the emulator
          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
            | sudo tee /etc/udev/rules.d/99-kvm4all.rules
          sudo udevadm control --reload-rules
          sudo udevadm trigger --name-match=kvm

      - name: NPM cache E2E Tests Library
        uses: actions/cache@v4
        id: deps-cache-e2e-library
        with:
          path: test/e2e/node_modules
          key: ${{ github.workflow }}-${{ github.job }}-npm-${{ hashFiles('test/e2e/yarn.lock') }}

      - name: Install E2E Tests Library JS Dependencies
        if: steps.deps-cache-e2e-library.outputs['cache-hit'] != 'true'
        working-directory: test/e2e
        run: yarn install

      - name: Run tests on Android
        if: ${{ matrix.platform == 'android' }}
        uses: reactivecircus/android-emulator-runner@f0d1ed2dcad93c7479e8b2f2226c83af54494915 # pin@v2.32.0
        with:
          api-level: 30
          force-avd-creation: false
          disable-animations: true
          disable-spellchecker: true
          target: 'aosp_atd'
          channel: canary # Necessary for ATDs
          emulator-options: >
            -no-window
            -no-snapshot-save
            -gpu swiftshader_indirect
            -noaudio
            -no-boot-anim
            -camera-back none
            -camera-front none
            -timezone US/Pacific
          script: ./scripts/e2e.mjs ${{ matrix.platform }} --test

      - uses: actions/cache@v4
        if: ${{ matrix.platform == 'ios' }}
        with:
          path: test/e2e/DerivedData/Build/Products/Debug-iphonesimulator/WebDriverAgentRunner-Runner.app
          key: appium-webdriveragent-${{ hashFiles('test/e2e/yarn.lock') }}

      - name: Run tests on iOS
        if: ${{ matrix.platform == 'ios' }}
        run: ./scripts/e2e.mjs ${{ matrix.platform }} --test

      - name: Upload logs
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.rn-version }}-${{ matrix.rn-architecture }}-${{ matrix.engine }}-${{ matrix.platform }}-${{ matrix.build-type }}-${{ matrix.ios-use-frameworks }}-logs
          path: |
            test/e2e/*.log
            test/e2e/*.png