Skip to content

Commit

Permalink
Upload Android test spec to ossci-android
Browse files Browse the repository at this point in the history
Differential Revision: D61160050

Pull Request resolved: #4676
  • Loading branch information
huydhn authored Aug 13, 2024
1 parent d51ec6d commit ccaaa46
Show file tree
Hide file tree
Showing 3 changed files with 130 additions and 4 deletions.
40 changes: 37 additions & 3 deletions .github/workflows/android-perf.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,47 @@ on:
description: The list of configs used the benchmark
required: false
type: string
test_spec:
description: The test spec to drive the test on AWS devices
required: false
type: string
default: https://ossci-android.s3.amazonaws.com/executorch/android-llm-device-farm-test-spec.yml
workflow_call:
inputs:
models:
description: Models to be benchmarked
required: false
type: string
default: stories110M
devices:
description: Target devices to run benchmark
required: false
type: string
default: samsung_galaxy_s2x
delegates:
description: Backend delegates
required: false
type: string
default: xnnpack
threadpool:
description: Run with threadpool?
required: false
type: boolean
default: false
benchmark_configs:
description: The list of configs used the benchmark
required: false
type: string
test_spec:
description: The test spec to drive the test on AWS devices
required: false
type: string
default: https://ossci-android.s3.amazonaws.com/executorch/android-llm-device-farm-test-spec.yml

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}
cancel-in-progress: true

permissions: read-all

jobs:
set-parameters:
runs-on: linux.2xlarge
Expand Down Expand Up @@ -234,6 +268,6 @@ jobs:
# TODO: Hard code llm_demo_bpe for now in this job.
android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_bpe/app-debug.apk
android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_bpe/app-debug-androidTest.apk
test-spec: https://ossci-assets.s3.amazonaws.com/android-llm-device-farm-test-spec.yml
test-spec: ${{ inputs.test_spec }}
# Uploaded to S3 from the previous job
extra-data: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/${{ matrix.model }}_${{ matrix.delegate }}/model.zip
2 changes: 1 addition & 1 deletion .github/workflows/android.yml
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ jobs:
# Uploaded to S3 from the previous job, the name of the app comes from the project itself
android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_${{ matrix.tokenizer }}/app-debug.apk
android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_${{ matrix.tokenizer }}/app-debug-androidTest.apk
test-spec: https://ossci-assets.s3.amazonaws.com/android-llm-device-farm-test-spec.yml
test-spec: https://ossci-android.s3.amazonaws.com/executorch/android-llm-device-farm-test-spec.yml
# Among the input, this is the biggest file, so it is cached on AWS to make the test faster. Note that the file is deleted by AWS after 30
# days and the job will automatically re-upload the file when that happens.
extra-data: https://ossci-assets.s3.amazonaws.com/executorch-android-llama2-7b-0717.zip
92 changes: 92 additions & 0 deletions .github/workflows/upload-test-specs.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
name: Upload AWS Device Farm test specs

on:
pull_request:
paths:
- .github/workflows/upload-test-specs.yml
- examples/demo-apps/android/LlamaDemo/android-llm-device-farm-test-spec.yml
push:
branches:
- main
paths:
- .github/workflows/upload-test-specs.yml
- examples/demo-apps/android/LlamaDemo/android-llm-device-farm-test-spec.yml

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}
cancel-in-progress: true

jobs:
upload-android-test-spec-for-validation:
runs-on: linux.2xlarge
steps:
- uses: actions/checkout@v3

- name: Upload the spec as a GitHub artifact for validation
uses: seemethere/upload-artifact-s3@v5
with:
s3-bucket: gha-artifacts
s3-prefix: |
${{ github.repository }}/${{ github.run_id }}/artifact
retention-days: 1
if-no-files-found: error
path: examples/demo-apps/android/LlamaDemo/android-llm-device-farm-test-spec.yml

validate-android-test-spec:
needs: upload-android-test-spec-for-validation
uses: ./.github/workflows/android-perf.yml
permissions:
id-token: write
contents: read
with:
# Just use a small model here with a minimal amount of configuration to test the spec
models: stories110M
devices: samsung_galaxy_s2x
delegates: xnnpack
test_spec: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/android-llm-device-farm-test-spec.yml

upload-android-test-spec:
needs: validate-android-test-spec
runs-on: ubuntu-22.04
timeout-minutes: 15
permissions:
id-token: write
contents: read
steps:
- uses: actions/checkout@v3

- uses: actions/setup-python@v4
with:
python-version: '3.11'
cache: pip

- name: configure aws credentials
uses: aws-actions/configure-aws-credentials@v1.7.0
with:
role-to-assume: arn:aws:iam::308535385114:role/gha_executorch_upload-frameworks-android
aws-region: us-east-1

- name: Only push to S3 when running the workflow manually from main branch
if: ${{ github.ref == 'refs/heads/main' }}
shell: bash
run: |
set -eux
echo "UPLOAD_ON_MAIN=1" >> "${GITHUB_ENV}"
- name: Upload the spec to S3 ossci-android bucket
shell: bash
working-directory: examples/demo-apps/android/LlamaDemo/
env:
SPEC_FILE: android-llm-device-farm-test-spec.yml
run: |
set -eux
pip install awscli==1.32.18
AWS_CMD="aws s3 cp --dryrun"
if [[ "${UPLOAD_ON_MAIN:-0}" == "1" ]]; then
AWS_CMD="aws s3 cp"
fi
shasum -a 256 "${SPEC_FILE}"
${AWS_CMD} "${SPEC_FILE}" s3://ossci-android/executorch/ --acl public-read

0 comments on commit ccaaa46

Please sign in to comment.