[CI] Add AWS EC2 dynamic runner support

.github/workflows/sycl_linux_build_and_test.yml

@@ -40,6 +40,10 @@ on:
         type: string
         required: false
         default: ""
+      lts_aws_matrix:
+        type: string
+        required: false
+        default: ""
       lts_cmake_extra_args:
         type: string
         required: false
@@ -155,9 +159,31 @@ jobs:
         name: sycl_lit_${{ inputs.build_artifact_suffix }}
         path: lit.tar.xz
 
-  llvm_test_suite:
+  aws-start:
+    name: Start AWS
     needs: build
-    if: ${{ inputs.lts_matrix != '' }}
+    if: ${{ inputs.lts_aws_matrix != '' }}
+    runs-on: ubuntu-latest
+    environment: aws
+    steps:
+      - name: Setup script
+        run: |
+          mkdir -p ./aws-ec2
+          wget raw.githubusercontent.com/intel/llvm/sycl/devops/actions/aws-ec2/action.yml   -P ./aws-ec2
+          wget raw.githubusercontent.com/intel/llvm/sycl/devops/actions/aws-ec2/aws-ec2.js   -P ./aws-ec2
+          wget raw.githubusercontent.com/intel/llvm/sycl/devops/actions/aws-ec2/package.json -P ./aws-ec2
+          npm install ./aws-ec2
+      - name: Start AWS EC2 runners
+        uses: ./aws-ec2
+        with:
+          runs-on-list: ${{ inputs.lts_aws_matrix }}
+          GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
+          AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}
+
+  llvm_test_suite:
+    needs: [build, aws-start]
+    if: ${{ !failure() && inputs.lts_matrix != '' }}
     strategy:
       fail-fast: false
       max-parallel: ${{ inputs.max_parallel }}
@@ -203,3 +229,26 @@ jobs:
         check_sycl_all: ${{ matrix.check_sycl_all }}
         results_name_suffix: ${{ matrix.config }}_${{ inputs.build_artifact_suffix }}
         cmake_args: '${{ matrix.cmake_args }} ${{ inputs.lts_cmake_extra_args }}'
+
+  aws-stop:
+    name: Stop AWS
+    needs: [ aws-start, llvm_test_suite ]
+    if: ${{ always() && inputs.lts_ats_matrix != '' }}
+    runs-on: ubuntu-latest
+    environment: aws
+    steps:
+      - name: Setup script
+        run: |
+          mkdir -p ./aws-ec2
+          wget raw.githubusercontent.com/intel/llvm/sycl/devops/actions/aws-ec2/action.yml   -P ./aws-ec2
+          wget raw.githubusercontent.com/intel/llvm/sycl/devops/actions/aws-ec2/aws-ec2.js   -P ./aws-ec2
+          wget raw.githubusercontent.com/intel/llvm/sycl/devops/actions/aws-ec2/package.json -P ./aws-ec2
+          npm install ./aws-ec2
+      - name: Stop AWS EC2 runners
+        uses: ./aws-ec2
+        with:
+          runs-on-list: ${{ inputs.lts_aws_matrix }}
+          mode: stop
+          GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }}
+          AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }}

.github/workflows/sycl_nightly.yml

@@ -20,6 +20,7 @@ jobs:
     if: github.repository == 'intel/llvm'
     uses: ./.github/workflows/sycl_linux_build_and_test.yml
     needs: resolve_matrix
+    secrets: inherit
     with:
       build_cache_root: "/__w/"
       build_artifact_suffix: default
@@ -29,6 +30,7 @@ jobs:
     if: github.repository == 'intel/llvm'
     uses: ./.github/workflows/sycl_linux_build_and_test.yml
     needs: resolve_matrix
+    secrets: inherit
     with:
       build_cache_root: "/__w/"
       build_cache_suffix: opaque_pointers

.github/workflows/sycl_post_commit.yml

@@ -21,13 +21,16 @@ jobs:
     name: Linux Default
     needs: resolve_matrix
     uses: ./.github/workflows/sycl_linux_build_and_test.yml
+    secrets: inherit
     with:
       build_cache_root: "/__w/llvm"
       build_artifact_suffix: "post_commit"
       lts_matrix: ${{ needs.resolve_matrix.outputs.lts_matrix }}
+      lts_aws_matrix: ${{ needs.resolve_matrix.outputs.lts_aws_matrix }}
   linux_no_assert:
     name: Linux (no assert)
     uses: ./.github/workflows/sycl_linux_build_and_test.yml
+    secrets: inherit
     with:
       build_cache_root: "/__w/llvm"
       build_cache_suffix: gcc_no_assertions

.github/workflows/sycl_precommit.yml

@@ -1,7 +1,7 @@
 name: SYCL
 
 on:
-  pull_request:
+  pull_request_target:
     branches:
     - sycl
     # Do not run builds if changes are only in the following locations
@@ -25,6 +25,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
       with:
+        persist-credentials: false
         fetch-depth: 2
     - name: Run clang-format
       uses: ./devops/actions/clang-format
@@ -43,9 +44,11 @@ jobs:
     needs: [lint, resolve_matrix]
     if: always() && (success() || contains(github.event.pull_request.labels.*.name, 'ignore-lint'))
     uses: ./.github/workflows/sycl_linux_build_and_test.yml
+    secrets: inherit
     with:
       build_cache_root: "/__w/"
       build_cache_size: "8G"
       build_artifact_suffix: "default"
       build_cache_suffix: "default"
       lts_matrix: ${{ needs.resolve_matrix.outputs.lts_matrix }}
+      lts_aws_matrix: ${{ needs.resolve_matrix.outputs.lts_aws_matrix }}

.github/workflows/sycl_resolve_test_matrix.yml

@@ -19,10 +19,18 @@ on:
         type: string
         required: true
         default: ""
+      uniq:
+        description: Unique string to name dynamic runners in AWS
+        type: string
+        required: false
+        default: ${{ github.run_id }}-${{ github.run_attempt }}
     outputs:
       lts_matrix:
         description: "Generated Matrix"
         value: ${{ jobs.resolve_matrix.outputs.lts_matrix }}
+      lts_aws_matrix:
+        description: "Generated Matrix AWS subset"
+        value: ${{ jobs.resolve_matrix.outputs.lts_aws_matrix }}
 jobs:
   resolve_matrix:
     name: Resolve Test Matrix

devops/actions/aws-ec2/action.yml

@@ -0,0 +1,66 @@
+name: aws-ec2
+description: Start AWS EC2 instances with Github actions runner agent in it
+
+inputs:
+  runs-on-list:
+    description: "JSON string with array of objects with aws-type, runs-on, aws-ami, aws-spot, aws-disk, aws-timebomb, one-job properties"
+    required: true
+    # aws-type:     AWS EC2 instance type. This property must be present if you want to trigger AWS EC2 instance start/stop.
+    # runs-on:      Name of the unique label assigned to the runner used as 'runs-on' property for the following jobs. Mandatory presence required.
+    # aws-ami:      AWS AMI id. Makes sense only for start mode. Default "ami-0966bccbb521ccb24".
+
+    #   ami-0966bccbb521ccb24: Ubuntu 22.04 (ami-02f3416038bdb17fb with /dev/sda1 disk) with docker installed and gh_runner (1001) like this:
+    #     sudo -s
+    #     apt-get update
+    #     curl -fsSL https://get.docker.com -o /tmp/get-docker.sh
+    #     sh /tmp/get-docker.sh
+    #     groupadd -g 1001 gh_runner; useradd gh_runner -u 1001 -g 1001 -m -s /bin/bash; usermod -aG docker gh_runner; usermod -aG video gh_runner
+    #     sync; shutdown -h now
+
+    #   ami-02ec0f344128253f9: Amazon Linux 2 AMI with NVIDIA TESLA GPU Driver (ami-06bf0a3f89fe08f0a with /dev/xvda disk) with docker installed and gh_runner (1001) like this:
+    #     sudo -s
+    #     yum update -y
+    #     amazon-linux-extras install docker
+    #     sudo systemctl --now enable docker
+    #     distribution=$(. /etc/os-release;echo $ID$VERSION_ID) && curl -s -L https://nvidia.github.io/libnvidia-container/$distribution/libnvidia-container.repo | sudo tee /etc/yum.repos.d/nvidia-container-toolkit.repo
+    #     yum-config-manager --disable amzn2-graphics; yum clean expire-cache; yum install -y nvidia-docker2; systemctl restart docker
+    #     groupadd -g 1001 gh_runner; useradd gh_runner -u 1001 -g 1001 -m -s /bin/bash; usermod -aG docker gh_runner; usermod -aG video gh_runner
+    #     sync; shutdown -h now
+
+    #   ami-0ccda708841dde988: Amazon Linux 2 AMI with AMD Radeon Pro Driver (ami-0bb1072e787242eb6 with /dev/xvda disk) with docker installed and gh_runner (1001) like this:
+    #     sudo -s
+    #     amazon-linux-extras install docker
+    #     sudo systemctl --now enable docker
+    #     groupadd -g 1001 gh_runner; useradd gh_runner -u 1001 -g 1001 -m -s /bin/bash; usermod -aG docker gh_runner; usermod -aG video gh_runner
+    #     sync; shutdown -h now
+
+    # aws-spot:     Enable usage of spot instances to save money (less reliable). Makes sense only for start mode. Default true.
+    # aws-disk:     AWS EC2 instance AMI specific disk device path and size in GB (8 by default). Makes sense only for start mode. Default "/dev/sda1:16".
+    # aws-timebomp: AWS EC2 instance maximum live time. Makes sense only for start mode. Default "1h".
+    # one-job:      Will terminate AWS EC2 instance after one job (not waiting for stop job) saving money. Makes sense only for start mode. Default true.
+
+  mode:
+    description: "Mode of operation: start or stop"
+    required: false
+    default: start
+
+  GH_PERSONAL_ACCESS_TOKEN:
+    description: "Github personal access token with repo permission"
+    required: true
+
+  AWS_ACCESS_KEY:
+    description: "AWS access id"
+    required: true
+
+  AWS_SECRET_KEY:
+    description: "AWS access secret key"
+    required: true
+
+  aws-region:
+    description: "AWS EC2 region"
+    required: false
+    default: "us-east-2" # Ohio
+
+runs:
+  using: node12
+  main: ./aws-ec2.js