ci: port lint, unit test, and e2e tests to Actions (kedro-org#155)

* Add unit test + lint test on GA * trigger GA - will revert Signed-off-by: Ankita Katiyar <ankitakatiyar2401@gmail.com> * Fix lint Signed-off-by: Ankita Katiyar <ankitakatiyar2401@gmail.com> * Add end to end tests * Add cache key Signed-off-by: Ankita Katiyar <ankitakatiyar2401@gmail.com> * Add cache action Signed-off-by: Ankita Katiyar <ankitakatiyar2401@gmail.com> * Rename workflow files Signed-off-by: Ankita Katiyar <ankitakatiyar2401@gmail.com> * Lint + add comment + default bash Signed-off-by: Ankita Katiyar <ankitakatiyar2401@gmail.com> * Add windows test Signed-off-by: Ankita Katiyar <ankitakatiyar2401@gmail.com> * Update workflow name + revert changes to READMEs Signed-off-by: Ankita Katiyar <ankitakatiyar2401@gmail.com> * Add kedro-telemetry/RELEASE.md to trufflehog ignore Signed-off-by: Ankita Katiyar <ankitakatiyar2401@gmail.com> * Add pytables to test_requirements remove from workflow Signed-off-by: Ankita Katiyar <ankitakatiyar2401@gmail.com> * Revert "Add pytables to test_requirements remove from workflow" This reverts commit 8203daa. * Separate pip freeze step Signed-off-by: Ankita Katiyar <ankitakatiyar2401@gmail.com> --------- Signed-off-by: Ankita Katiyar <ankitakatiyar2401@gmail.com> Signed-off-by: Danny Farah <danny_farah@mckinsey.com>
dannyrfar · May 3, 2023 · 45151ec · 45151ec
1 parent 494fa5f
commit 45151ec
Show file tree

Hide file tree

Showing 6 changed files with 200 additions and 0 deletions.
diff --git a/.github/workflows/check-plugin.yml b/.github/workflows/check-plugin.yml
@@ -0,0 +1,134 @@
+name: Running tests and linter
+
+on:
+  workflow_call:
+    inputs:
+      plugin:
+        type: string
+
+jobs:
+  unit-tests:
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, windows-latest ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10" ]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Python ${{matrix.python-version}}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{matrix.python-version}}
+      - name: Cache python packages for Linux
+        if: matrix.os == 'ubuntu-latest'
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}}
+          restore-keys: ${{inputs.plugin}}
+      - name: Cache python packages for Windows
+        if: matrix.os == 'windows-latest'
+        uses: actions/cache@v3
+        with:
+          path: ~\AppData\Local\pip\Cache
+          key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}}
+          restore-keys: ${{inputs.plugin}}
+      - name: Install Kedro
+        run: pip install git+https://github.com/kedro-org/kedro@main
+      - name: Install dependencies
+        run: |
+          cd ${{ inputs.plugin }}
+          pip install -r test_requirements.txt
+      - name: Install pytables (only for kedro-datasets on windows)
+        if: matrix.os == 'windows-latest' && inputs.plugin == 'kedro-datasets'
+        run: pip install tables
+      - name: pip freeze
+        run: pip freeze
+      - name: Run unit tests for Linux / all plugins
+        if: matrix.os != 'windows-latest'
+        run: make plugin=${{ inputs.plugin }} test
+      - name: Run unit tests for Windows / kedro-airflow, kedro-docker, kedro-telemetry
+        if: matrix.os == 'windows-latest' && inputs.plugin != 'kedro-datasets'
+        run: |
+          cd ${{ inputs.plugin }}
+          pytest tests
+      - name: Run unit tests for Windows / kedro-datasets / no spark sequential
+        if: matrix.os == 'windows-latest' && inputs.plugin == 'kedro-datasets' && matrix.python-version == '3.10'
+        run: |
+          make test-no-spark-sequential
+      - name: Run unit tests for Windows / kedro-datasets / no spark parallel
+        if: matrix.os == 'windows-latest' && inputs.plugin == 'kedro-datasets' && matrix.python-version != '3.10'
+        run: |
+          make test-no-spark
+
+  lint:
+    defaults:
+      run:
+        shell: bash
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v3
+        with:
+          python-version: 3.8
+      - name: Cache python packages
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}}
+          restore-keys: ${{inputs.plugin}}
+      - name: Install dependencies
+        run: |
+            cd ${{ inputs.plugin }}
+            pip install git+https://github.com/kedro-org/kedro@main
+            pip install -r test_requirements.txt
+            pip freeze
+      - name: Install pre-commit hooks
+        run: |
+            cd ${{ inputs.plugin }}
+            pre-commit install --install-hooks
+            pre-commit install --hook-type pre-push
+      - name: Run linter
+        run: make plugin=${{ inputs.plugin }} lint
+
+  e2e-tests:
+    if: inputs.plugin != 'kedro-datasets'
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10" ]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Python ${{matrix.python-version}}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{matrix.python-version}}
+      - name: Cache python packages
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}}
+          restore-keys: ${{inputs.plugin}}
+      - name: Install dependencies
+        run: |
+          cd ${{ inputs.plugin }}
+          pip install git+https://github.com/kedro-org/kedro@main
+          pip install -r test_requirements.txt
+      - name: pip freeze
+        run: pip freeze
+      - name: Run end to end tests
+        # Custom shell to run kedro-docker e2e-tests because -it flag for `docker run`
+        # isn't supported on Github Actions. See https://github.com/actions/runner/issues/241
+        shell: 'script -q -e -c "bash {0}"'
+        run: make plugin=${{ inputs.plugin }} e2e-tests
diff --git a/.github/workflows/kedro-airflow.yml b/.github/workflows/kedro-airflow.yml
@@ -0,0 +1,16 @@
+name: Run checks on kedro-airflow
+
+on:
+  push:
+    paths:
+      - "kedro-airflow/**"
+  pull_request:
+    paths:
+      - "kedro-airflow/**"
+    types: [ synchronize ]
+
+jobs:
+  airflow-test:
+    uses: ./.github/workflows/check-plugin.yml
+    with:
+      plugin: kedro-airflow
diff --git a/.github/workflows/kedro-datasets.yml b/.github/workflows/kedro-datasets.yml
@@ -0,0 +1,16 @@
+name: Run checks on kedro-datasets
+
+on:
+  push:
+    paths:
+      - "kedro-datasets/**"
+  pull_request:
+    paths:
+      - "kedro-datasets/**"
+    types: [ synchronize ]
+
+jobs:
+  datasets-test:
+    uses: ./.github/workflows/check-plugin.yml
+    with:
+      plugin: kedro-datasets
diff --git a/.github/workflows/kedro-docker.yml b/.github/workflows/kedro-docker.yml
@@ -0,0 +1,16 @@
+name: Run checks on kedro-docker
+
+on:
+  push:
+    paths:
+      - "kedro-docker/**"
+  pull_request:
+    paths:
+      - "kedro-docker/**"
+    types: [ synchronize ]
+
+jobs:
+  docker-test:
+    uses: ./.github/workflows/check-plugin.yml
+    with:
+      plugin: kedro-docker
diff --git a/.github/workflows/kedro-telemetry.yml b/.github/workflows/kedro-telemetry.yml
@@ -0,0 +1,16 @@
+name: Run checks on kedro-telemetry
+
+on:
+  push:
+    paths:
+      - "kedro-telemetry/**"
+  pull_request:
+    paths:
+      - "kedro-telemetry/**"
+    types: [ synchronize ]
+
+jobs:
+  telemetry-test:
+    uses: ./.github/workflows/check-plugin.yml
+    with:
+      plugin: kedro-telemetry
diff --git a/trufflehog-ignore.txt b/trufflehog-ignore.txt
@@ -1 +1,3 @@
 kedro-telemetry/README.md
+kedro-telemetry/RELEASE.md
+kedro-datasets/tests/tensorflow/test_tensorflow_model_dataset.py