From f93bee92e873a579f15fd75a9b8f069c8cb3f49f Mon Sep 17 00:00:00 2001 From: Andrei Elovikov Date: Thu, 20 Jul 2023 14:02:09 -0700 Subject: [PATCH] [CI] Don't use matrix for drivers installation in pre-commit Use functionality introduced in https://github.com/intel/llvm/pull/10503 instead. The only piece remaining in the matrix generation is selection of driverless image if new drivers are to be installed. --- .github/workflows/sycl_detect_changes.yml | 4 +- .github/workflows/sycl_gen_test_matrix.yml | 23 +++++---- .../workflows/sycl_linux_build_and_test.yml | 10 ++-- .github/workflows/sycl_linux_run_tests.yml | 8 --- .../workflows/sycl_windows_build_and_test.yml | 1 - devops/scripts/generate_test_matrix.js | 51 +------------------ 6 files changed, 23 insertions(+), 74 deletions(-) diff --git a/.github/workflows/sycl_detect_changes.yml b/.github/workflows/sycl_detect_changes.yml index bc6c46cc2547b..adc6f22ebeec0 100644 --- a/.github/workflows/sycl_detect_changes.yml +++ b/.github/workflows/sycl_detect_changes.yml @@ -52,6 +52,8 @@ jobs: # devops/* contains config files, including drivers versions. # Allow them to be tested in pre-commit. - devops/*/** + drivers: + - devops/dependencies.json - name: Set output id: result @@ -64,6 +66,6 @@ jobs: return '${{ steps.changes.outputs.changes }}'; } // Treat everything as changed for huge PRs. - return ["llvm", "llvm_spirv", "clang", "sycl_fusion", "xptifw", "libclc", "sycl", "ci"]; + return ["llvm", "llvm_spirv", "clang", "sycl_fusion", "xptifw", "libclc", "sycl", "ci", "drivers"]; - run: echo '${{ steps.result.outputs.result }}' diff --git a/.github/workflows/sycl_gen_test_matrix.yml b/.github/workflows/sycl_gen_test_matrix.yml index d452785687088..d46fe494aac81 100644 --- a/.github/workflows/sycl_gen_test_matrix.yml +++ b/.github/workflows/sycl_gen_test_matrix.yml @@ -33,6 +33,11 @@ on: type: string required: false default: ${{ github.sha }} + drivers_update_needed: + description: "True indicates that dependencies.json has changed" + type: string + required: false + default: false outputs: lts_lx_matrix: description: "Linux LTS" @@ -58,14 +63,12 @@ jobs: cts_matrix: ${{ steps.work.outputs.cts_matrix }} lts_aws_matrix: ${{ steps.work.outputs.lts_aws_matrix }} steps: - - name: Download scripts and configs - shell: bash - run: | - wget raw.githubusercontent.com/intel/llvm/${{ inputs.ref }}/devops/scripts/generate_test_matrix.js - wget raw.githubusercontent.com/intel/llvm/${{ inputs.ref }}/devops/test_configs.json - wget raw.githubusercontent.com/intel/llvm/sycl/devops/dependencies.json - mv dependencies.json dependencies.sycl.json - wget raw.githubusercontent.com/intel/llvm/${{ inputs.ref }}/devops/dependencies.json + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.ref }} + sparse-checkout: | + devops/scripts/generate_test_matrix.js + devops/test_configs.json - id: work uses: actions/github-script@v6 name: Generate matrix @@ -73,7 +76,5 @@ jobs: GHA_INPUTS: ${{ toJSON(inputs) }} with: script: | - const script = require('./generate_test_matrix.js'); + const script = require('./devops/generate_test_matrix.js'); script({core, process}); - - name: Cleanup - run: rm -rf generate_test_matrix.js test_configs.json dependencies.json dependencies.sycl.json diff --git a/.github/workflows/sycl_linux_build_and_test.yml b/.github/workflows/sycl_linux_build_and_test.yml index aced54e0537d2..f9456a97e0305 100644 --- a/.github/workflows/sycl_linux_build_and_test.yml +++ b/.github/workflows/sycl_linux_build_and_test.yml @@ -64,6 +64,8 @@ on: type: choice options: - "[]" + # Use this to test drivers installation: + - "[drivers]" - '[llvm, clang, sycl, llvm_spirv, xptifw, libclc, libdevice]' build_image: type: choice @@ -113,6 +115,8 @@ jobs: with: ref: ${{ inputs.build_ref || github.sha }} lts_config: ${{ inputs.lts_config }} + drivers_update_needed: ${{ contains(inputs.changes, 'drivers') }} + build: needs: [test_matrix] @@ -283,13 +287,12 @@ jobs: ref: ${{ inputs.build_ref || github.sha }} merge_ref: ${{ inputs.merge_ref }} reset_gpu: ${{ contains(matrix.runs-on, 'gen12') && contains(matrix.runs-on, 'Linux') }} + install_drivers: ${{ contains(inputs.changes, 'drivers') && (contains(matrix.runs-on, 'gen12') || contains(matrix.runs-on, 'x86-cpu'))}} sycl_toolchain_artifact: sycl_linux_${{ inputs.build_artifact_suffix }} sycl_toolchain_archive: ${{ needs.build.outputs.artifact_archive_name }} sycl_toolchain_decompress_command: ${{ needs.build.outputs.artifact_decompress_command }} - env: ${{ toJSON(matrix.env) }} - khronos_sycl_cts: needs: [test_matrix, build] if: ${{ needs.test_matrix.outputs.cts_matrix != '[]' && inputs.check_sycl == 'true' }} @@ -309,13 +312,12 @@ jobs: ref: ${{ inputs.build_ref || github.sha }} merge_ref: ${{ inputs.merge_ref }} reset_gpu: ${{ contains(matrix.runs-on, 'gen12') && contains(matrix.runs-on, 'Linux') }} + install_drivers: ${{ contains(inputs.changes, 'drivers') && (contains(matrix.runs-on, 'gen12') || contains(matrix.runs-on, 'x86-cpu'))}} sycl_toolchain_artifact: sycl_linux_${{ inputs.build_artifact_suffix }} sycl_toolchain_archive: ${{ inputs.artifact_archive_name }} sycl_toolchain_decompress_command: ${{ inputs.artifact_decompress_command }} - env: ${{ toJSON(matrix.env) }} - aws-stop: name: Stop AWS needs: [ test_matrix, aws-start, e2e-tests ] diff --git a/.github/workflows/sycl_linux_run_tests.yml b/.github/workflows/sycl_linux_run_tests.yml index 2f447ebb6529e..7eae79a78e50b 100644 --- a/.github/workflows/sycl_linux_run_tests.yml +++ b/.github/workflows/sycl_linux_run_tests.yml @@ -169,14 +169,6 @@ jobs: sudo apt install -yq jq sudo cp devops/scripts/get_release.py /opt/ sudo -E devops/scripts/install_drivers.sh llvm/devops/dependencies.json --all - - name: Install drivers (deprecated CI interface) - if: env.compute_runtime_tag != '' && inputs.install_drivers != 'true' - run: | - if [ -e /opt/install_drivers.sh ]; then - # TODO pack this script into container - sudo cp devops/scripts/get_release.py /opt/ - sudo -E /opt/install_drivers.sh --all - fi - name: Source OneAPI TBB vars.sh shell: bash run: | diff --git a/.github/workflows/sycl_windows_build_and_test.yml b/.github/workflows/sycl_windows_build_and_test.yml index a881fe191d097..179e507f1c217 100644 --- a/.github/workflows/sycl_windows_build_and_test.yml +++ b/.github/workflows/sycl_windows_build_and_test.yml @@ -134,7 +134,6 @@ jobs: include: ${{ fromJSON(inputs.lts_matrix) }} name: ${{ matrix.name }} runs-on: ${{ matrix.runs-on }} - env: ${{ matrix.env }} steps: - uses: ilammy/msvc-dev-cmd@7315a94840631165970262a99c72cfb48a65d25d with: diff --git a/devops/scripts/generate_test_matrix.js b/devops/scripts/generate_test_matrix.js index cb0f24ff3c5d4..26cc1b959d96b 100644 --- a/devops/scripts/generate_test_matrix.js +++ b/devops/scripts/generate_test_matrix.js @@ -1,30 +1,12 @@ module.exports = ({core, process}) => { const fs = require('fs'); - fs.readFile('./test_configs.json', 'utf8', (err, data) => { + fs.readFile('./devops/test_configs.json', 'utf8', (err, data) => { if (err) { console.error(`Error reading file from disk: ${err}`); } else { - const driverNew = - JSON.parse(fs.readFileSync('./dependencies.json', 'utf8')); - const driverOld = - JSON.parse(fs.readFileSync('./dependencies.sycl.json', 'utf8')); const testConfigs = JSON.parse(data); const inputs = JSON.parse(process.env.GHA_INPUTS); - const needsDrivers = - driverNew["linux"]["compute_runtime"]["version"] !== - driverOld["linux"]["compute_runtime"]["version"] || - driverNew["linux"]["igc"]["version"] !== - driverOld["linux"]["igc"]["version"] || - driverNew["linux"]["cm"]["version"] !== - driverOld["linux"]["cm"]["version"] || - driverNew["linux"]["level_zero"]["version"] !== - driverOld["linux"]["level_zero"]["version"] || - driverNew["linux"]["tbb"]["version"] !== - driverOld["linux"]["tbb"]["version"] || - driverNew["linux"]["oclcpu"]["version"] !== - driverOld["linux"]["oclcpu"]["version"] || - driverNew["linux"]["fpgaemu"]["version"] !== - driverOld["linux"]["fpgaemu"]["version"]; + const needsDrivers = inputs.drivers_update_needed == 'true'; const ltsConfigs = inputs.lts_config.split(';'); @@ -36,21 +18,6 @@ module.exports = ({core, process}) => { testConfigs.lts.forEach(v => { if (ltsConfigs.includes(v.config)) { - if (needsDrivers) { - v["env"] = { - "compute_runtime_tag" : - driverNew["linux"]["compute_runtime"]["github_tag"], - "igc_tag" : driverNew["linux"]["igc"]["github_tag"], - "cm_tag" : driverNew["linux"]["cm"]["github_tag"], - "level_zero_tag" : driverNew["linux"]["level_zero"]["github_tag"], - "tbb_tag" : driverNew["linux"]["tbb"]["github_tag"], - "cpu_tag" : driverNew["linux"]["oclcpu"]["github_tag"], - "fpgaemu_tag" : driverNew["linux"]["fpgaemu"]["github_tag"], - }; - } else { - v["env"] = {}; - } - // Check for CUDA machines. If available, add them to // enabledLTSLxConfigs. var hasCuda = false; @@ -106,20 +73,6 @@ module.exports = ({core, process}) => { testConfigs.cts.forEach(v => { if (ctsConfigs.includes(v.config)) { - if (needsDrivers) { - v["env"] = { - "compute_runtime_tag" : - driverNew["linux"]["compute_runtime"]["github_tag"], - "igc_tag" : driverNew["linux"]["igc"]["github_tag"], - "cm_tag" : driverNew["linux"]["cm"]["github_tag"], - "level_zero_tag" : driverNew["linux"]["level_zero"]["github_tag"], - "tbb_tag" : driverNew["linux"]["tbb"]["github_tag"], - "cpu_tag" : driverNew["linux"]["oclcpu"]["github_tag"], - "fpgaemu_tag" : driverNew["linux"]["fpgaemu"]["github_tag"], - }; - } else { - v["env"] = {}; - } enabledCTSConfigs.push(v); } });