Implement Pipeline Automation #62
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow will build opal, install its build, then runs the tests with pytest. | |
| name: Test Model Pipeline Inference | |
| on: [ pull_request ] | |
| jobs: | |
| pipeline: | |
| timeout-minutes: 10 | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Checkout repository with submodules | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: true | |
| - name: Execute Pipeline | |
| working-directory: src/ | |
| shell: bash | |
| run: | | |
| chmod +x pipeline.sh | |
| ./pipeline.sh cache | |
| - name: Check Dataset exists and is not empty | |
| if: always() | |
| working-directory: src/datasets/ | |
| run: | | |
| dataset=*.csv | |
| [ -f "$dataset" ] || { echo "Dataset file not found"; exit 1; } | |
| [ -s "$dataset" ] || { echo "Dataset file is empty"; exit 1; } | |
| - name: Check Pipeline Cache exists and is not empty | |
| if: always() | |
| working-directory: src/.pipeline_cache/ | |
| run: | | |
| pipeline_cache=cache.csv | |
| [ -f "$pipeline_cache" ] || { echo "Pipeline Cache file not found"; exit 1; } | |
| [ -s "$pipeline_cache" ] || { echo "Pipeline Cache file is empty"; exit 1; } | |
| - name: Check Dist exists and is not empty | |
| if: always() | |
| working-directory: src/dist/ | |
| run: | | |
| dist=*.tar.gz2 | |
| [ -f "$dist" ] || { echo "Dist file not found"; exit 1; } | |
| [ -s "$dist" ] || { echo "Dist file is empty"; exit 1; } | |
| - name: Preprocess Docker Logs | |
| working-directory: src/ | |
| if: always() | |
| run: | | |
| for SVC in 1.preprocess 2.svness 3.preprocess 4.export; do | |
| docker compose \ | |
| --profile files \ | |
| -f preprocess/docker-compose.yml \ | |
| --env-file preprocess/osu-data-docker/.env \ | |
| logs $SVC > ${SVC}.log | |
| done | |
| - name: Export src as Artifact | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: src | |
| path: src/ | |
| - name: Export built wheel as Artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: dist | |
| path: src/dist/ | |
| model: | |
| needs: [ pipeline ] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: [ "3.9", "3.10", "3.11" ] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Get Dist | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: dist | |
| path: src/dist/ | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install pytest | |
| - name: Install opal build | |
| working-directory: src/dist/ | |
| run: | | |
| python -m pip install *.whl | |
| - name: Remove local opal | |
| run: | | |
| rm -rf src/ | |
| - name: Test with pytest | |
| run: | | |
| pytest tests/ |