From 45151ec503ccf1a71ae39d2baf351dbd34800347 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar <110245118+ankatiyar@users.noreply.github.com> Date: Wed, 19 Apr 2023 15:21:17 +0100 Subject: [PATCH] ci: port lint, unit test, and e2e tests to Actions (#155) * Add unit test + lint test on GA * trigger GA - will revert Signed-off-by: Ankita Katiyar * Fix lint Signed-off-by: Ankita Katiyar * Add end to end tests * Add cache key Signed-off-by: Ankita Katiyar * Add cache action Signed-off-by: Ankita Katiyar * Rename workflow files Signed-off-by: Ankita Katiyar * Lint + add comment + default bash Signed-off-by: Ankita Katiyar * Add windows test Signed-off-by: Ankita Katiyar * Update workflow name + revert changes to READMEs Signed-off-by: Ankita Katiyar * Add kedro-telemetry/RELEASE.md to trufflehog ignore Signed-off-by: Ankita Katiyar * Add pytables to test_requirements remove from workflow Signed-off-by: Ankita Katiyar * Revert "Add pytables to test_requirements remove from workflow" This reverts commit 8203daa6405d325c74ec2097c9d0c5859bae8257. * Separate pip freeze step Signed-off-by: Ankita Katiyar --------- Signed-off-by: Ankita Katiyar Signed-off-by: Danny Farah --- .github/workflows/check-plugin.yml | 134 ++++++++++++++++++++++++++ .github/workflows/kedro-airflow.yml | 16 +++ .github/workflows/kedro-datasets.yml | 16 +++ .github/workflows/kedro-docker.yml | 16 +++ .github/workflows/kedro-telemetry.yml | 16 +++ trufflehog-ignore.txt | 2 + 6 files changed, 200 insertions(+) create mode 100644 .github/workflows/check-plugin.yml create mode 100644 .github/workflows/kedro-airflow.yml create mode 100644 .github/workflows/kedro-datasets.yml create mode 100644 .github/workflows/kedro-docker.yml create mode 100644 .github/workflows/kedro-telemetry.yml diff --git a/.github/workflows/check-plugin.yml b/.github/workflows/check-plugin.yml new file mode 100644 index 000000000..a32c0f651 --- /dev/null +++ b/.github/workflows/check-plugin.yml @@ -0,0 +1,134 @@ +name: Running tests and linter + +on: + workflow_call: + inputs: + plugin: + type: string + +jobs: + unit-tests: + defaults: + run: + shell: bash + strategy: + matrix: + os: [ ubuntu-latest, windows-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python ${{matrix.python-version}} + uses: actions/setup-python@v3 + with: + python-version: ${{matrix.python-version}} + - name: Cache python packages for Linux + if: matrix.os == 'ubuntu-latest' + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}} + restore-keys: ${{inputs.plugin}} + - name: Cache python packages for Windows + if: matrix.os == 'windows-latest' + uses: actions/cache@v3 + with: + path: ~\AppData\Local\pip\Cache + key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}} + restore-keys: ${{inputs.plugin}} + - name: Install Kedro + run: pip install git+https://github.com/kedro-org/kedro@main + - name: Install dependencies + run: | + cd ${{ inputs.plugin }} + pip install -r test_requirements.txt + - name: Install pytables (only for kedro-datasets on windows) + if: matrix.os == 'windows-latest' && inputs.plugin == 'kedro-datasets' + run: pip install tables + - name: pip freeze + run: pip freeze + - name: Run unit tests for Linux / all plugins + if: matrix.os != 'windows-latest' + run: make plugin=${{ inputs.plugin }} test + - name: Run unit tests for Windows / kedro-airflow, kedro-docker, kedro-telemetry + if: matrix.os == 'windows-latest' && inputs.plugin != 'kedro-datasets' + run: | + cd ${{ inputs.plugin }} + pytest tests + - name: Run unit tests for Windows / kedro-datasets / no spark sequential + if: matrix.os == 'windows-latest' && inputs.plugin == 'kedro-datasets' && matrix.python-version == '3.10' + run: | + make test-no-spark-sequential + - name: Run unit tests for Windows / kedro-datasets / no spark parallel + if: matrix.os == 'windows-latest' && inputs.plugin == 'kedro-datasets' && matrix.python-version != '3.10' + run: | + make test-no-spark + + lint: + defaults: + run: + shell: bash + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python 3.8 + uses: actions/setup-python@v3 + with: + python-version: 3.8 + - name: Cache python packages + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}} + restore-keys: ${{inputs.plugin}} + - name: Install dependencies + run: | + cd ${{ inputs.plugin }} + pip install git+https://github.com/kedro-org/kedro@main + pip install -r test_requirements.txt + pip freeze + - name: Install pre-commit hooks + run: | + cd ${{ inputs.plugin }} + pre-commit install --install-hooks + pre-commit install --hook-type pre-push + - name: Run linter + run: make plugin=${{ inputs.plugin }} lint + + e2e-tests: + if: inputs.plugin != 'kedro-datasets' + defaults: + run: + shell: bash + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python ${{matrix.python-version}} + uses: actions/setup-python@v3 + with: + python-version: ${{matrix.python-version}} + - name: Cache python packages + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}} + restore-keys: ${{inputs.plugin}} + - name: Install dependencies + run: | + cd ${{ inputs.plugin }} + pip install git+https://github.com/kedro-org/kedro@main + pip install -r test_requirements.txt + - name: pip freeze + run: pip freeze + - name: Run end to end tests + # Custom shell to run kedro-docker e2e-tests because -it flag for `docker run` + # isn't supported on Github Actions. See https://github.com/actions/runner/issues/241 + shell: 'script -q -e -c "bash {0}"' + run: make plugin=${{ inputs.plugin }} e2e-tests diff --git a/.github/workflows/kedro-airflow.yml b/.github/workflows/kedro-airflow.yml new file mode 100644 index 000000000..b68fcce30 --- /dev/null +++ b/.github/workflows/kedro-airflow.yml @@ -0,0 +1,16 @@ +name: Run checks on kedro-airflow + +on: + push: + paths: + - "kedro-airflow/**" + pull_request: + paths: + - "kedro-airflow/**" + types: [ synchronize ] + +jobs: + airflow-test: + uses: ./.github/workflows/check-plugin.yml + with: + plugin: kedro-airflow diff --git a/.github/workflows/kedro-datasets.yml b/.github/workflows/kedro-datasets.yml new file mode 100644 index 000000000..9ff4802b6 --- /dev/null +++ b/.github/workflows/kedro-datasets.yml @@ -0,0 +1,16 @@ +name: Run checks on kedro-datasets + +on: + push: + paths: + - "kedro-datasets/**" + pull_request: + paths: + - "kedro-datasets/**" + types: [ synchronize ] + +jobs: + datasets-test: + uses: ./.github/workflows/check-plugin.yml + with: + plugin: kedro-datasets diff --git a/.github/workflows/kedro-docker.yml b/.github/workflows/kedro-docker.yml new file mode 100644 index 000000000..1812a3a93 --- /dev/null +++ b/.github/workflows/kedro-docker.yml @@ -0,0 +1,16 @@ +name: Run checks on kedro-docker + +on: + push: + paths: + - "kedro-docker/**" + pull_request: + paths: + - "kedro-docker/**" + types: [ synchronize ] + +jobs: + docker-test: + uses: ./.github/workflows/check-plugin.yml + with: + plugin: kedro-docker diff --git a/.github/workflows/kedro-telemetry.yml b/.github/workflows/kedro-telemetry.yml new file mode 100644 index 000000000..fd75e8a71 --- /dev/null +++ b/.github/workflows/kedro-telemetry.yml @@ -0,0 +1,16 @@ +name: Run checks on kedro-telemetry + +on: + push: + paths: + - "kedro-telemetry/**" + pull_request: + paths: + - "kedro-telemetry/**" + types: [ synchronize ] + +jobs: + telemetry-test: + uses: ./.github/workflows/check-plugin.yml + with: + plugin: kedro-telemetry diff --git a/trufflehog-ignore.txt b/trufflehog-ignore.txt index 041fc7ffd..1929a2634 100644 --- a/trufflehog-ignore.txt +++ b/trufflehog-ignore.txt @@ -1 +1,3 @@ kedro-telemetry/README.md +kedro-telemetry/RELEASE.md +kedro-datasets/tests/tensorflow/test_tensorflow_model_dataset.py