Pin pip < 23.2 and Port unit tests, lint and e2e tests to Github Acti…

…ons unblock CI (#2843)

* port tests to GA

Signed-off-by: Ankita Katiyar <ankitakatiyar2401@gmail.com>

* Revert changes to CircleCI config

Signed-off-by: Ankita Katiyar <ankitakatiyar2401@gmail.com>

* Revert changes to CircleCI config

Signed-off-by: Ankita Katiyar <ankitakatiyar2401@gmail.com>

* Suggestions from code review

Signed-off-by: Ankita Katiyar <ankitakatiyar2401@gmail.com>

* Suggestions from code review

Signed-off-by: Ankita Katiyar <ankitakatiyar2401@gmail.com>

* Create merge-gatekeeper.yml

* Update release notes

Signed-off-by: Ankita Katiyar <ankitakatiyar2401@gmail.com>

* Update merge-gatekeeper.yml

* Update merge-gatekeeper.yml

* Update merge-gatekeeper.yml

* Update merge-gatekeeper.yml

* Update merge-gatekeeper.yml

---------

Signed-off-by: Ankita Katiyar <ankitakatiyar2401@gmail.com>
Co-authored-by: Nok Lam Chan <nok.lam.chan@quantumblack.com>

.github/workflows/all-checks.yml

@@ -0,0 +1,44 @@
+name: Run all checks on Kedro
+
+on:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - "docs/**"
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - "docs/**"
+
+jobs:
+  unit-tests:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, windows-latest ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10" ]
+    uses: ./.github/workflows/unit-tests.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}
+
+  lint:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+        python-version: [ "3.10" ]
+    uses: ./.github/workflows/lint.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}
+
+  e2e-tests:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, windows-latest ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10" ]
+    uses: ./.github/workflows/e2e-tests.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}

.github/workflows/docs-only-checks.yml

@@ -0,0 +1,24 @@
+name: Run linter on Kedro Docs
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "docs/**"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "docs/**"
+
+jobs:
+  lint-tests:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10" ]
+    uses: ./.github/workflows/lint.yml
+    with:
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}

.github/workflows/e2e-tests.yml

@@ -0,0 +1,46 @@
+name: Run e2e-tests on Kedro
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        type: string
+      python-version:
+        type: string
+
+env:
+  COLUMNS: 120
+  LINES: 25
+
+jobs:
+  e2e-tests:
+    runs-on: ${{ inputs.os }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Python ${{inputs.python-version}}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{inputs.python-version}}
+      - run: make install-pip-setuptools
+      - name: Cache python packages for Linux
+        if: inputs.os == 'ubuntu-latest'
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{inputs.os}}-python-${{inputs.python-version}}
+      - name: Cache python packages for Windows
+        if: inputs.os == 'windows-latest'
+        uses: actions/cache@v3
+        with:
+          path: ~\AppData\Local\pip\Cache
+          key: ${{inputs.os}}-python-${{inputs.python-version}}
+      - name: Install dependencies
+        run: |
+          pip --version
+          make install-test-requirements
+          make install-pre-commit
+      - name: pip freeze
+        run: pip freeze
+      - name: Run e2e tests
+        run: make e2e-tests

.github/workflows/lint.yml

@@ -0,0 +1,32 @@
+name: Run linters on Kedro
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        type: string
+      python-version:
+        type: string
+
+jobs:
+  lint:
+    runs-on: ${{ inputs.os }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Python ${{ inputs.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ inputs.python-version }}
+      - name: Cache python packages
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{inputs.os}}-python-${{inputs.python-version}}
+      - name: Install dependencies
+        run: |
+            make install-test-requirements
+            make install-pre-commit
+            pip freeze
+      - name: Run linter
+        run: make lint

.github/workflows/merge-gatekeeper.yml

@@ -0,0 +1,27 @@
+name: Merge Gatekeeper
+
+on:
+  pull_request:
+    branches:
+      - main
+      - develop
+
+jobs:
+  merge-gatekeeper:
+    runs-on: ubuntu-latest
+    # Restrict permissions of the GITHUB_TOKEN.
+    # Docs: https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs
+    permissions:
+      checks: read
+      statuses: read
+    steps:
+      - name: Run Merge Gatekeeper
+        # NOTE: v1 is updated to reflect the latest v1.x.y. Please use any tag/branch that suits your needs:
+        #       https://github.com/upsidr/merge-gatekeeper/tags
+        #       https://github.com/upsidr/merge-gatekeeper/branches
+        uses: upsidr/merge-gatekeeper@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          timeout: 1800
+          interval: 30
+          ignored: 'ci/circleci: win_e2e_tests-3.7,ci/circleci: win_pip_compile-3.9,ci/circleci: win_e2e_tests-3.9,ci/circleci: win_pip_compile-3.8,ci/circleci: lint-3.7,ci/circleci: win_pip_compile-3.7,ci/circleci: pip_compile-3.7,ci/circleci: e2e_tests-3.7,ci/circleci: win_unit_tests-3.7,ci/circleci: win_unit_tests-3.9,ci/circleci: e2e_tests-3.8,ci/circleci: win_unit_tests-3.10,ci/circleci: win_pip_compile-3.10,ci/circleci: win_unit_tests-3.8,ci/circleci: e2e_tests-3.9,ci/circleci: unit_tests-3.10,ci/circleci: unit_tests-3.8,ci/circleci: e2e_tests-3.10,ci/circleci: lint-3.8,ci/circleci: unit_tests-3.9,ci/circleci: unit_tests-3.7,ci/circleci: win_e2e_tests-3.10,ci/circleci: pip_compile-3.8,ci/circleci: pip_compile-3.10,ci/circleci: win_e2e_tests-3.8,ci/circleci: lint-3.9,ci/circleci: pip_compile-3.9,ci/circleci: lint-3.10,build_code,ci/circlecici: check-updated-files,regular'

.github/workflows/unit-tests.yml

@@ -0,0 +1,53 @@
+name: Run unit-tests on Kedro
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        type: string
+      python-version:
+        type: string
+jobs:
+  unit-tests:
+    runs-on: ${{ inputs.os }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Python ${{inputs.python-version}}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{inputs.python-version}}
+      - run: make install-pip-setuptools
+      - name: Cache python packages for Linux
+        if: inputs.os == 'ubuntu-latest'
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{inputs.os}}-python-${{inputs.python-version}}
+      - name: Cache python packages for Windows
+        if: inputs.os == 'windows-latest'
+        uses: actions/cache@v3
+        with:
+          path: ~\AppData\Local\pip\Cache
+          key: ${{inputs.os}}-python-${{inputs.python-version}}
+      - name: Install dependencies
+        run: |
+            make install-test-requirements
+            make install-pre-commit
+      - name: Install pytables (only for windows)
+        if: inputs.os == 'windows-latest'
+        run: pip install tables
+      - name: pip freeze
+        run: pip freeze
+      - name: Run unit tests
+        if: inputs.os == 'ubuntu-latest' && inputs.python-version == '3.10'
+        run: make test-sequential
+      - name: Run unit tests
+        if: inputs.os == 'ubuntu-latest' && inputs.python-version != '3.10'
+        run: make test
+      - name: Run unit tests (Windows)
+        if: inputs.os == 'windows-latest' && inputs.python-version == '3.10'
+        run: make test-no-spark-sequential
+      - name: Run unit tests (Windows)
+        if: inputs.os == 'windows-latest' && inputs.python-version != '3.10'
+        run: make test-no-spark

Makefile

@@ -8,7 +8,7 @@ clean:
 	pre-commit clean || true
 
 install-pip-setuptools:
-	pip install -U "pip>=21.2" "setuptools>=65.5.1" wheel
+	python -m pip install -U "pip>=21.2, <23.2" "setuptools>=65.5.1" wheel
 
 lint:
 	pre-commit run -a --hook-stage manual $(hook)
@@ -19,6 +19,12 @@ test:
 test-no-spark:
 	pytest --no-cov --ignore tests/extras/datasets/spark --numprocesses 4 --dist loadfile
 
+test-sequential:
+	pytest tests --cov-config pyproject.toml
+
+test-no-spark-sequential:
+	pytest tests --no-cov --ignore tests/extras/datasets/spark
+
 test-no-datasets:
 	pytest --no-cov --ignore tests/extras/datasets/ --numprocesses 4 --dist loadfile
 

RELEASE.md

@@ -16,6 +16,7 @@
 
 ## Bug fixes and other changes
 * Consolidated dependencies and optional dependencies in `pyproject.toml`.
+* Pin `pip<23.2` for CI due to a breaking change. See https://github.com/kedro-org/kedro/pull/2813
 
 ## Documentation changes
 

features/environment.py

@@ -103,7 +103,9 @@ def _setup_minimal_env(context):
             "pip",
             "install",
             "-U",
-            "pip>=21.2",
+            # pip==23.2 breaks pip-tools<7.0, and pip-tools>=7.0 does not support Python 3.7
+            "pip>=21.2,<23.2; python_version < '3.8'",
+            "pip>=21.2; python_version >= '3.8'",
             "setuptools>=65.5.1",
             "wheel",
         ],

pyproject.toml

@@ -28,7 +28,7 @@ dependencies = [
     "more_itertools~=9.0",
     "omegaconf~=2.3",
     "parse~=1.19.0",
-    "pip-tools~=6.5",
+    "pip-tools>=6.5,<8",
     "pluggy~=1.0",
     "PyYAML>=4.2, <7.0",
     "rich>=12.0, <14.0",

trufflehog-ignore.txt

@@ -9,3 +9,5 @@ static/img/kedro_gitflow.svg
 .coverage.*
 .*\.log
 .*\.iml
+tests/extras/datasets/tensorflow/test_tensorflow_model_dataset.py
+docs/source/meta/images/kedro_gitflow.svg