Firefox UI Tests #159
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Firefox UI Tests" | |
| on: | |
| workflow_dispatch: {} | |
| schedule: | |
| - cron: "0 2 * * *" | |
| env: | |
| browser: firefox-ios | |
| xcode_version: 15.4 | |
| ios_version: 17.5 | |
| ios_simulator_default: iPhone 15 | |
| xcodebuild_scheme: Fennec | |
| xcodebuild_target: Client | |
| test_results_directory: /Users/runner/tmp | |
| jobs: | |
| compile: | |
| name: Compile | |
| runs-on: macos-14 | |
| steps: | |
| - name: Check out source code | |
| uses: actions/checkout@v4.1.7 | |
| - name: Setup Xcode | |
| id: xcode | |
| run: | | |
| sudo rm -rf /Applications/Xcode.app | |
| sudo xcode-select -s /Applications/Xcode_${{ env.xcode_version }}.app/Contents/Developer | |
| xcodebuild -version | |
| ./checkout.sh | |
| ./bootstrap.sh --force | |
| - name: Compile source code | |
| id: compile | |
| run: | | |
| xcodebuild \ | |
| -resolvePackageDependencies \ | |
| -onlyUsePackageVersionsFromResolvedFile | |
| xcodebuild \ | |
| build-for-testing \ | |
| -scheme ${{ env.xcodebuild_scheme }} \ | |
| -target ${{ env.xcodebuild_target }} \ | |
| -derivedDataPath ~/DerivedData \ | |
| -destination 'platform=iOS Simulator,name=${{ env.ios_simulator_default }},OS=${{ env.ios_version }}' \ | |
| COMPILER_INDEX_STORE_ENABLE=NO CODE_SIGN_IDENTITY= CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO ARCHS="arm64" | |
| working-directory: ${{ env.browser }} | |
| - name: Compress Derived Data | |
| id: compress-dd | |
| run: | | |
| tar czf derived-data.tar.gz ~/DerivedData/Build/Products | |
| - name: Save Derived Data | |
| id: upload-derived-data | |
| uses: actions/upload-artifact@v4.3.4 | |
| with: | |
| name: xcode-cache-deriveddata-${{ github.workflow }}-${{ github.sha }} | |
| path: ./derived-data.tar.gz | |
| retention-days: 2 | |
| run-smoketests: | |
| name: Run smoke tests | |
| runs-on: macos-14 | |
| needs: compile | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| ios_simulator: ['iPhone 15 Plus', 'iPad Pro 13-inch (M4)'] | |
| steps: | |
| - name: Check out source code | |
| uses: actions/checkout@v4.1.7 | |
| - name: Install packages | |
| id: packages | |
| run: | | |
| gem install xcpretty -v 0.3.0 | |
| pip install blockkit==1.9.1 | |
| npm install -g junit-report-merger@7.0.0 | |
| - name: Setup Xcode | |
| id: xcode | |
| run: | | |
| sudo rm -rf /Applications/Xcode.app | |
| sudo xcode-select -s /Applications/Xcode_${{ env.xcode_version }}.app/Contents/Developer | |
| xcodebuild -version | |
| ./checkout.sh | |
| ./bootstrap.sh --force | |
| - name: Get derived data | |
| id: download-derived-data | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: xcode-cache-deriveddata-${{ github.workflow }}-${{ github.sha }} | |
| - name: Decompress derived data | |
| id: decompress-dd | |
| run: | | |
| tar xzf derived-data.tar.gz -C / | |
| - name: Run Smoketest1 | |
| id: run-smoketest1 | |
| run: | | |
| xcodebuild \ | |
| test-without-building \ | |
| -scheme ${{ env.xcodebuild_scheme }} \ | |
| -target ${{ env.xcodebuild_target }} \ | |
| -derivedDataPath ~/DerivedData \ | |
| -destination 'platform=iOS Simulator,name=${{ matrix.ios_simulator }},OS=${{ env.ios_version }}' \ | |
| -testPlan Smoketest1 \ | |
| -resultBundlePath ${{ env.test_results_directory }}/results-smoketest1 \ | |
| | tee xcodebuild-smoketest1.log | xcpretty -r junit --output ./junit-smoketest1.xml && exit ${PIPESTATUS[0]} | |
| working-directory: ${{ env.browser }} | |
| continue-on-error: true | |
| - name: Run Smoketest2 | |
| id: run-smoketest2 | |
| run: | | |
| xcodebuild \ | |
| test-without-building \ | |
| -scheme ${{ env.xcodebuild_scheme }} \ | |
| -target ${{ env.xcodebuild_target }} \ | |
| -derivedDataPath ~/DerivedData \ | |
| -destination 'platform=iOS Simulator,name=${{ matrix.ios_simulator }},OS=${{ env.ios_version }}' \ | |
| -testPlan Smoketest2 \ | |
| -resultBundlePath ${{ env.test_results_directory }}/results-smoketest2 \ | |
| | tee xcodebuild-smoketest2.log | xcpretty -r junit --output ./junit-smoketest2.xml && exit ${PIPESTATUS[0]} | |
| working-directory: ${{ env.browser }} | |
| continue-on-error: true | |
| - name: Run Smoketest3 | |
| id: run-smoketest3 | |
| run: | | |
| xcodebuild \ | |
| test-without-building \ | |
| -scheme ${{ env.xcodebuild_scheme }} \ | |
| -target ${{ env.xcodebuild_target }} \ | |
| -derivedDataPath ~/DerivedData \ | |
| -destination 'platform=iOS Simulator,name=${{ matrix.ios_simulator }},OS=${{ env.ios_version }}' \ | |
| -testPlan Smoketest3 \ | |
| -resultBundlePath ${{ env.test_results_directory }}/results-smoketest3 \ | |
| | tee xcodebuild-smoketest3.log | xcpretty -r junit --output ./junit-smoketest3.xml && exit ${PIPESTATUS[0]} | |
| working-directory: ${{ env.browser }} | |
| continue-on-error: true | |
| - name: Run Smoketest4 | |
| id: run-smoketest4 | |
| run: | | |
| xcodebuild \ | |
| test-without-building \ | |
| -scheme ${{ env.xcodebuild_scheme }} \ | |
| -target ${{ env.xcodebuild_target }} \ | |
| -derivedDataPath ~/DerivedData \ | |
| -destination 'platform=iOS Simulator,name=${{ matrix.ios_simulator }},OS=${{ env.ios_version }}' \ | |
| -testPlan Smoketest4 \ | |
| -resultBundlePath ${{ env.test_results_directory }}/results-smoketest4 \ | |
| | tee xcodebuild-smoketest4.log | xcpretty -r junit --output ./junit-smoketest4.xml && exit ${PIPESTATUS[0]} | |
| working-directory: ${{ env.browser }} | |
| continue-on-error: true | |
| - name: Determine pass/fail status | |
| id: passfail | |
| run: | | |
| echo "Smoketest1 status: "${{ steps.run-smoketest1.outcome }} | |
| echo "Smoketest2 status: "${{ steps.run-smoketest2.outcome }} | |
| echo "Smoketest3 status: "${{ steps.run-smoketest3.outcome }} | |
| echo "Smoketest4 status: "${{ steps.run-smoketest4.outcome }} | |
| if [[ ${{ steps.run-smoketest1.outcome }} != 'success' | |
| || ${{ steps.run-smoketest2.outcome }} != 'success' | |
| || ${{ steps.run-smoketest3.outcome }} != 'success' | |
| || ${{ steps.run-smoketest4.outcome }} != 'success' ]]; then | |
| exit 1 | |
| else | |
| exit 0 | |
| fi | |
| continue-on-error: true | |
| - name: Print test report | |
| id: test-report | |
| run: | | |
| jrm combined.xml junit-*.xml | |
| python ../test-fixtures/ci/convert_junit_to_markdown.py --smoke --github --${{ env.browser }} ./combined.xml ./github.md | |
| cat github.md >> $GITHUB_STEP_SUMMARY | |
| python ../test-fixtures/ci/convert_junit_to_markdown.py --smoke --slack --${{ env.browser }} ./combined.xml ./slack.json | |
| mv ./combined.xml "junit-smoketests-${{ matrix.ios_simulator }}-`date +"%Y-%m-%d"`.xml" | |
| working-directory: ${{ env.browser }} | |
| - name: Upload junit files | |
| id: upload-junit | |
| uses: actions/upload-artifact@v4.3.3 | |
| with: | |
| name: ${{ env.browser }}-smoketests-${{ matrix.ios_simulator }}-junit-${{ github.run_number }} | |
| path: ${{ env.browser }}/junit-smoketests-*.xml | |
| retention-days: 90 | |
| - name: Upload log file | |
| id: upload-log | |
| uses: actions/upload-artifact@v4.3.3 | |
| with: | |
| name: ${{ env.browser }}-smoketests-${{ matrix.ios_simulator }}-xcodebuildlog-${{ github.run_number }} | |
| path: ${{ env.browser }}/xcodebuild-smoketest*.log | |
| retention-days: 90 | |
| - name: Report to Slack | |
| id: slack | |
| uses: slackapi/slack-github-action@v1.26.0 | |
| with: | |
| payload-file-path: ${{ env.browser }}/slack.json | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.WEBHOOK_SLACK_TOKEN }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| ios_simulator: ${{ matrix.ios_simulator }} | |
| pass_fail: ${{ steps.passfail.outcome == 'success' && ':white_check_mark:' || ':x:' }} | |
| xcodebuild_test_plan: Smoketests | |
| ref_name: ${{ github.ref_name }} | |
| repository: ${{ github.repository }} | |
| run_id: ${{ github.run_id }} | |
| server_url: ${{ github.server_url }} | |
| sha: ${{ github.sha }} | |
| - name: Return fail status if a test fails | |
| run: | | |
| exit ${{ steps.passfail.outcome == 'success' && '0' || '1' }} | |
| run-fullfunctionaltests: | |
| name: Run full functional tests | |
| runs-on: macos-14 | |
| needs: run-smoketests | |
| if: false | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| ios_simulator: ['iPhone 15 Pro Max', 'iPad Pro (12.9 inch) (6th generation)'] | |
| steps: | |
| - name: Check out source code | |
| uses: actions/checkout@v4.1.7 | |
| - name: Install packages | |
| id: packages | |
| run: | | |
| gem install xcpretty -v 0.3.0 | |
| pip install blockkit==1.9.1 | |
| - name: Get derived data | |
| id: download-derived-data | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: xcode-cache-deriveddata-${{ github.workflow }}-${{ github.sha }} | |
| - name: Decompress derived data | |
| id: decompress-dd | |
| run: | | |
| tar xzf derived-data.tar.gz -C / | |
| - name: Run tests | |
| id: run-tests | |
| run: | | |
| xcrun simctl boot '${{ matrix.ios_simulator }}' | |
| xcodebuild \ | |
| test-without-building \ | |
| -scheme ${{ env.xcodebuild_scheme }} \ | |
| -target ${{ env.xcodebuild_target }} \ | |
| -derivedDataPath ~/DerivedData \ | |
| -destination 'platform=iOS Simulator,name=${{ matrix.ios_simulator }},OS=${{ env.ios_version }}' \ | |
| -testPlan FullFunctionalTestPlan \ | |
| -resultBundlePath ${{ env.test_results_directory }}/results \ | |
| | tee xcodebuild.log | xcpretty -r junit && exit ${PIPESTATUS[0]} | |
| working-directory: ${{ env.browser }} | |
| continue-on-error: true | |
| - name: Print test report | |
| id: test-report | |
| run: | | |
| python ../test-fixtures/ci/convert_junit_to_markdown.py --github --${{ env.browser }} ./build/reports/junit.xml ./github.md | |
| cat github.md >> $GITHUB_STEP_SUMMARY | |
| python ../test-fixtures/ci/convert_junit_to_markdown.py --slack --${{ env.browser }} ./build/reports/junit.xml ./slack.json | |
| mv ./build/reports/junit.xml "junit-fullfunctional-${{ matrix.ios_simulator }}-`date +"%Y-%m-%d"`.xml" | |
| working-directory: ${{ env.browser }} | |
| - name: Upload junit files | |
| id: upload-junit | |
| uses: actions/upload-artifact@v4.3.3 | |
| with: | |
| name: ${{ env.browser }}-fullfunctional-${{ matrix.ios_simulator }}-junit-${{ github.run_number }} | |
| path: ${{ env.browser }}/junit-*.xml | |
| retention-days: 90 | |
| - name: Upload log file | |
| id: upload-log | |
| uses: actions/upload-artifact@v4.3.3 | |
| with: | |
| name: ${{ env.browser }}-fullfunctional-${{ matrix.ios_simulator }}-xcodebuildlog-${{ github.run_number }} | |
| path: ${{ env.browser }}/xcodebuild.log | |
| retention-days: 90 | |
| - name: Report to Slack | |
| id: slack | |
| uses: slackapi/slack-github-action@v1.26.0 | |
| with: | |
| payload-file-path: ${{ env.browser }}/slack.json | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| ios_simulator: ${{ matrix.ios_simulator }} | |
| pass_fail: ${{ steps.run-tests.outcome == 'success' && ':white_check_mark:' || ':x:' }} | |
| xcodebuild_test_plan: FullFunctionalTestPlan | |
| ref_name: ${{ github.ref_name }} | |
| repository: ${{ github.repository }} | |
| run_id: ${{ github.run_id }} | |
| server_url: ${{ github.server_url }} | |
| sha: ${{ github.sha }} | |
| - name: Return fail status if a test fails | |
| run: | | |
| exit ${{ steps.run-tests.outcome == 'success' && '0' || '1' }} |