integration_test_workload_pipeline.yml

name: Integration Test Workload Basic Pipeline

on:
  pull_request:
    branches:
    - main
    paths:
    - '.github/workflows/integration_test_workload_pipeline.yml'
    - 'Dockerfile-ubuntu18.04/**'
    - 'e2eAIOK/SDA/**'
    - 'e2eAIOK/common/**'
    - 'e2eAIOK/dataloader/**'
    - 'e2eAIOK/utils/**'
    - 'tests/cicd/conf/**'
    - 'tests/cicd/src/**'
    - 'tests/cicd/test_data/**'
    - 'tests/cicd/test_*.bats'
    - 'tests/cicd/Jenkinsfile'
    - 'tests/cicd/jenkins_pipeline_test.sh'
    - 'tests/cicd/docker_horovod_test.sh'
    - 'tests/cicd/docker_torchccl_test'

jobs:
  integration_test:
    name: Integration Test Workload Basic Pipeline
    runs-on: self-hosted
    steps:
    - name: Checkout
      uses: actions/checkout@v2
      with:
        submodules: recursive

    - name: Build Docker
      run: |
        cd Dockerfile-ubuntu18.04 && docker build -t e2eaiok-tensorflow . -f DockerfileTensorflow && docker build -t e2eaiok-pytorch110 . -f DockerfilePytorch110 && cd .. && yes | docker container prune && yes | docker image prune
        
    - name: Test Pipeline
      run: |
        if [[ $(docker ps -q --filter "name=test-workload-pipeline" | wc -l) -ne 0 ]]; then docker ps -q --filter "name=test-workload-pipeline" | grep -q . && docker stop $(docker ps -a -q) && docker rm $(docker ps -a -q); fi
        docker run --rm --name test-workload-pipeline --privileged --network host --device=/dev/dri -v /mnt/DP_disk1/dataset:/home/vmagent/app/dataset -v `pwd`:/home/vmagent/app/e2eaiok -w /home/vmagent/app/ e2eaiok-tensorflow /bin/bash -c "USE_SIGOPT=0 . /home/vmagent/app/e2eaiok/tests/cicd/jenkins_pipeline_test.sh"

    - name: Test Horovod
      run: |
        if [[ $(docker ps -q --filter "name=test-workload-horovod" | wc -l) -ne 0 ]]; then docker ps -q --filter "name=test-workload-horovod" | grep -q . && docker stop $(docker ps -a -q) && docker rm $(docker ps -a -q); fi
        docker run --rm --name test-workload-horovod --privileged --network host --device=/dev/dri -v /mnt/DP_disk1/dataset:/home/vmagent/app/dataset -v `pwd`:/home/vmagent/app/e2eaiok -w /home/vmagent/app/ e2eaiok-tensorflow /bin/bash -c "USE_SIGOPT=0 . /home/vmagent/app/e2eaiok/tests/cicd/docker_horovod_test.sh"

    - name: Test TorchCCL
      run: |
        if [[ $(docker ps -q --filter "name=test-workload-torchccl" | wc -l) -ne 0 ]]; then docker ps -q --filter "name=test-workload-torchccl" | grep -q . && docker stop $(docker ps -a -q) && docker rm $(docker ps -a -q); fi
        docker run --rm --name test-workload-torchccl --privileged --network host --device=/dev/dri -v /mnt/DP_disk1/dataset:/home/vmagent/app/dataset -v `pwd`:/home/vmagent/app/e2eaiok -w /home/vmagent/app/ e2eaiok-pytorch110 /bin/bash -c "USE_SIGOPT=0 . /home/vmagent/app/e2eaiok/tests/cicd/docker_torchccl_test.sh"