From 22cf09131a1a9c04f886467bc039efd074163ecc Mon Sep 17 00:00:00 2001 From: Dmitrii Deriabin <44967953+DmitriiDeriabinQB@users.noreply.github.com> Date: Wed, 25 Mar 2020 11:01:22 +0000 Subject: [PATCH] [KED-1253] Drop support for Python 3.5 (#506) --- .circleci/config.yml | 39 ++++++--------- .pre-commit-config.yaml | 5 +- CONTRIBUTING.md | 4 +- README.md | 2 +- RELEASE.md | 1 + .../source/01_introduction/01_introduction.md | 2 +- docs/source/05_api_docs/index.rst | 4 +- docs/source/06_resources/01_faq.md | 4 +- .../06_resources/03_commands_reference.md | 2 +- .../{{ cookiecutter.repo_name }}/kedro_cli.py | 9 +--- .../src/requirements.txt | 2 +- setup.py | 6 +-- test_requirements.txt | 7 ++- tests/io/test_incremental_dataset.py | 4 +- tests/pipeline/test_node.py | 5 +- tests/template/test_kedro_cli.py | 6 +-- tests/tools/ipython/test_ipython_loader.py | 4 +- tools/min_version.py | 48 ------------------- 18 files changed, 39 insertions(+), 115 deletions(-) delete mode 100644 tools/min_version.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 1d18b876f1..7dd969d389 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,9 +1,6 @@ version: 2.1 executors: - py35: - docker: - - image: 350138855857.dkr.ecr.eu-west-2.amazonaws.com/kedro-builder:3.5 py36: docker: - image: 350138855857.dkr.ecr.eu-west-2.amazonaws.com/kedro-builder:3.6 @@ -116,18 +113,6 @@ commands: command: make pip-compile jobs: - unit_tests_35: - executor: py35 - steps: [unit_tests] - - linters_35: - executor: py35 - steps: [lint] - - e2e_tests_35: - executor: py35 - steps: [e2e_tests] - unit_tests_36: executor: py36 steps: [unit_tests] @@ -172,10 +157,18 @@ jobs: executor: py38 steps: [e2e_tests] - pip_compile: + pip_compile_36: + executor: py36 + steps: [pip_compile] + + pip_compile_37: executor: py37 steps: [pip_compile] + pip_compile_38: + executor: py38 + steps: [pip_compile] + all_circleci_checks_succeeded: docker: - image: circleci/python # any light-weight image @@ -189,9 +182,6 @@ workflows: version: 2 regular: jobs: - - unit_tests_35 - - linters_35 - - e2e_tests_35 - unit_tests_36 - linters_36 - e2e_tests_36 @@ -203,12 +193,11 @@ workflows: - unit_tests_38 - linters_38 - e2e_tests_38 - - pip_compile + - pip_compile_36 + - pip_compile_37 + - pip_compile_38 - all_circleci_checks_succeeded: requires: - - unit_tests_35 - - linters_35 - - e2e_tests_35 - unit_tests_36 - linters_36 - e2e_tests_36 @@ -220,4 +209,6 @@ workflows: - unit_tests_38 - linters_38 - e2e_tests_38 - - pip_compile + - pip_compile_36 + - pip_compile_37 + - pip_compile_38 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 49dfb6f65d..fffca85691 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -79,12 +79,11 @@ repos: pass_filenames: false stages: [manual] entry: pylint --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments tests - # We need to make some exceptions for 3.5, that's why it's a custom runner. - id: black name: "Black" language: system pass_filenames: false - entry: python -m tools.min_version 3.6 "black kedro features tests" + entry: black kedro features tests - id: legal name: "Licence check" language: system @@ -94,7 +93,7 @@ repos: name: "Import Linter" language: system pass_filenames: false - entry: python -m tools.min_version 3.6 lint-imports + entry: lint-imports - id: secret_scan name: "Secret scan" language: system diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e1b29bee7d..62fccaaf2a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -83,7 +83,7 @@ make build-docs - We use [Anaconda](https://www.anaconda.com/distribution/) as a preferred virtual environment - We use [SemVer](https://semver.org/) for versioning -Our code is designed to be compatible with Python 3.5 onwards and our style guidelines are (in cascading order): +Our code is designed to be compatible with Python 3.6 onwards and our style guidelines are (in cascading order): * [PEP 8 conventions](https://www.python.org/dev/peps/pep-0008/) for all Python code * [Google docstrings](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) for code comments @@ -193,8 +193,6 @@ make build-docs This command will only work on Unix-like systems and requires `pandoc` to be installed. -> ❗ Running `make build-docs` in a Python 3.5 environment may sometimes yield multiple warning messages like the following: `MemoryDataSet.md: WARNING: document isn't included in any toctree`. You can simply ignore them or switch to Python 3.6+ when building documentation. - ## Hints on pre-commit usage The checks will automatically run on all the changed files on each commit. Even more extensive set of checks (including the heavy set of `pylint` checks) diff --git a/README.md b/README.md index 1547b5c235..ccb8b0ef41 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ | Theme | Status | |------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | Latest Release | [![PyPI version](https://badge.fury.io/py/kedro.svg)](https://pypi.org/project/kedro/) | -| Python Version | [![Python Version](https://img.shields.io/badge/python-3.5%20%7C%203.6%20%7C%203.7%20%7C%203.8-blue.svg)](https://pypi.org/project/kedro/) | +| Python Version | [![Python Version](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue.svg)](https://pypi.org/project/kedro/) | | `master` Branch Build | [![CircleCI](https://circleci.com/gh/quantumblacklabs/kedro/tree/master.svg?style=shield)](https://circleci.com/gh/quantumblacklabs/kedro/tree/master) | | `develop` Branch Build | [![CircleCI](https://circleci.com/gh/quantumblacklabs/kedro/tree/develop.svg?style=shield)](https://circleci.com/gh/quantumblacklabs/kedro/tree/develop) | | Documentation Build | [![Documentation](https://readthedocs.org/projects/kedro/badge/?version=latest)](https://kedro.readthedocs.io/) | diff --git a/RELEASE.md b/RELEASE.md index 19cae67600..1c20d3c6ec 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -24,6 +24,7 @@ * Deleted obsoleted datasets from `kedro.io`. * Deleted `kedro.contrib` and `extras` folders. * `Pipeline.name` has been removed in favour of `Pipeline.tag()`. +* Python 3.5 is no longer supported by the current and all future versions of Kedro. ### Migration guide from Kedro 0.15.* to Upcoming Release #### Migration for datasets diff --git a/docs/source/01_introduction/01_introduction.md b/docs/source/01_introduction/01_introduction.md index b1d604b608..a873d7e52a 100644 --- a/docs/source/01_introduction/01_introduction.md +++ b/docs/source/01_introduction/01_introduction.md @@ -19,7 +19,7 @@ You can also check out the resources section for answers to [frequently asked qu We have designed the documentation in general, and the tutorial in particular, for beginners to get started creating their own Kedro projects in Python. If you have elementary knowledge using Python then you might find the Kedro learning curve more challenging. However, we have simplified the tutorial by providing all Python functions required to create your data pipelines. > -> *Note:* There are a number of excellent online resources for learning Python, but be aware that you should choose those that reference Python 3, as Kedro is built for Python 3.5+. There are many curated lists of online resources, such as: +> *Note:* There are a number of excellent online resources for learning Python, but be aware that you should choose those that reference Python 3, as Kedro is built for Python 3.6+. There are many curated lists of online resources, such as: > > - [Official Python programming language website](https://www.python.org/) > - [List of free programming books and tutorials](https://github.com/EbookFoundation/free-programming-books/blob/master/free-programming-books.md#python) diff --git a/docs/source/05_api_docs/index.rst b/docs/source/05_api_docs/index.rst index 0bb7168d35..769651c55f 100644 --- a/docs/source/05_api_docs/index.rst +++ b/docs/source/05_api_docs/index.rst @@ -19,9 +19,9 @@ Welcome to Kedro's API docs and tutorials! :target: https://opensource.org/licenses/Apache-2.0 :alt: License is Apache 2.0 -.. image:: https://img.shields.io/badge/python-3.5%20%7C%203.6%20%7C%203.7-blue.svg +.. image:: https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue.svg :target: https://pypi.org/project/kedro/ - :alt: Python version 3.5, 3.6, 3.7 + :alt: Python version 3.6, 3.7, 3.8 .. image:: https://badge.fury.io/py/kedro.svg :target: https://pypi.org/project/kedro/ diff --git a/docs/source/06_resources/01_faq.md b/docs/source/06_resources/01_faq.md index 6c14d9502a..7bcfa7137e 100644 --- a/docs/source/06_resources/01_faq.md +++ b/docs/source/06_resources/01_faq.md @@ -135,9 +135,9 @@ You can find a list of Kedro projects in the [`kedro-examples`](https://github.c +----------------+---------------------------------------------------------------------------------------------------+ ``` -## What version of Python does Kedro use? +## What version of Python does Kedro support? -Kedro is built for Python 3.5, 3.6, 3.7 and 3.8. +Kedro is built for Python 3.6, 3.7 and 3.8. ## How do I upgrade Kedro? diff --git a/docs/source/06_resources/03_commands_reference.md b/docs/source/06_resources/03_commands_reference.md index d2be110a08..fd3a40a040 100644 --- a/docs/source/06_resources/03_commands_reference.md +++ b/docs/source/06_resources/03_commands_reference.md @@ -83,7 +83,7 @@ This [line magic](https://ipython.readthedocs.io/en/stable/interactive/magics.ht Copy the code from cells [tagged](https://jupyter-notebook.readthedocs.io/en/stable/changelog.html#cell-tags) with `node` tag into Python files under `src//nodes/` in a Kedro project. ### `kedro lint` -Lint your project code using the `kedro lint` command. Your project is linted with [`black`](https://github.com/psf/black) (Python 3.6+), [`flake8`](https://gitlab.com/pycqa/flake8) and [`isort`](https://github.com/timothycrosley/isort). +Lint your project code using the `kedro lint` command. Your project is linted with [`black`](https://github.com/psf/black), [`flake8`](https://gitlab.com/pycqa/flake8) and [`isort`](https://github.com/timothycrosley/isort). ### `kedro activate-nbstripout` Typically output cells of Jupyter Notebook should not be tracked by git, especially if they contain sensitive information. diff --git a/kedro/template/{{ cookiecutter.repo_name }}/kedro_cli.py b/kedro/template/{{ cookiecutter.repo_name }}/kedro_cli.py index de0b966434..059a461ff5 100755 --- a/kedro/template/{{ cookiecutter.repo_name }}/kedro_cli.py +++ b/kedro/template/{{ cookiecutter.repo_name }}/kedro_cli.py @@ -294,16 +294,11 @@ def lint(files): try: import flake8 import isort + import black except ImportError as exc: raise KedroCliError(NO_DEPENDENCY_MESSAGE.format(exc.name)) - if sys.version_info[:2] >= (3, 6): - try: - import black - except ImportError: - raise KedroCliError(NO_DEPENDENCY_MESSAGE.format("black")) - python_call("black", files) - + python_call("black", files) python_call("flake8", ("--max-line-length=88",) + files) python_call("isort", ("-rc", "-tc", "-up", "-fgw=0", "-m=3", "-w=88") + files) diff --git a/kedro/template/{{ cookiecutter.repo_name }}/src/requirements.txt b/kedro/template/{{ cookiecutter.repo_name }}/src/requirements.txt index b3e00eb78c..c74f020154 100644 --- a/kedro/template/{{ cookiecutter.repo_name }}/src/requirements.txt +++ b/kedro/template/{{ cookiecutter.repo_name }}/src/requirements.txt @@ -1,4 +1,4 @@ -black==v19.10b0; python_version >= '3.6' +black==v19.10b0 flake8>=3.7.9, <4.0 ipython>=7.0.0, <8.0 isort>=4.3.21, <5.0 diff --git a/setup.py b/setup.py index 6e087b554a..6246c4a642 100644 --- a/setup.py +++ b/setup.py @@ -102,8 +102,7 @@ "pyarrow>=0.12.0, <1.0.0", "xlrd>=1.0.0, <2.0", "xlsxwriter>=1.0.0, <2.0", - "tables>=3.4.4, <3.6; python_version < '3.6'", - "tables>=3.6; python_version >= '3.6'", + "tables>=3.6", ], "spark": ["pyspark>=2.2.0, <3.0", "hdfs>=2.5.8, <3.0", "s3fs>=0.3.0, <1.0"], } @@ -118,7 +117,7 @@ long_description=readme, long_description_content_type="text/markdown", url="https://github.com/quantumblacklabs/kedro", - python_requires=">=3.5, <3.9", + python_requires=">=3.6, <3.9", packages=find_packages(exclude=["docs*", "tests*", "tools*", "features*"]), include_package_data=True, tests_require=test_requires, @@ -130,7 +129,6 @@ keywords="pipelines, machine learning, data pipelines, data science, data engineering", classifiers=[ "Development Status :: 4 - Beta", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", diff --git a/test_requirements.txt b/test_requirements.txt index ed6bf77cd3..af1dd8b136 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -5,13 +5,13 @@ azure-storage-queue>=1.1.0, <2.0 bandit>=1.6.2, <2.0 behave==1.2.6 biopython>=1.73, <2.0 -black==v19.10.b0; python_version >= '3.6' +black==v19.10.b0 dask[complete]>=2.6.0, <3.0 flake8>=3.5,<4.0 gcsfs>=0.3.0, <1.0 geopandas>=0.6.0, <1.0 hdfs>=2.5.8, <3.0 -import-linter==1.0; python_version >= '3.6' +import-linter==1.0 joblib>=0.14 jupyter_client>=5.1.0, <6.0 matplotlib>=3.0.3, <4.0 @@ -34,8 +34,7 @@ pytest>=3.9, <4.0 requests-mock>=1.6.0, <2.0.0 s3fs>=0.3.0, <1.0 # Needs to be at least 0.3.0 to make use of `cachable` attribute on S3FileSystem SQLAlchemy>=1.2.0, <2.0 -tables>=3.4.4, <3.6; python_version < '3.6' # 3.6.0 breaks on install, HDF5 library not found -tables>=3.6; python_version >= '3.6' +tables>=3.6 trufflehog>=2.1.0, <3.0 wheel==0.32.2 xlrd>=1.0.0, <2.0 diff --git a/tests/io/test_incremental_dataset.py b/tests/io/test_incremental_dataset.py index b04fc6f731..a253d5b69f 100644 --- a/tests/io/test_incremental_dataset.py +++ b/tests/io/test_incremental_dataset.py @@ -55,9 +55,7 @@ def partitioned_data_pandas(): @pytest.fixture def local_csvs(tmp_path, partitioned_data_pandas): - # tmp_path on Python 3.5 is a pathlib2.Path instance, which cannot be - # handled directly by Path instance, hence interim conversion to string - local_dir = Path(str(tmp_path / "csvs")) + local_dir = Path(tmp_path / "csvs") local_dir.mkdir() for k, data in partitioned_data_pandas.items(): diff --git a/tests/pipeline/test_node.py b/tests/pipeline/test_node.py index 71ea3249eb..1e67c56102 100644 --- a/tests/pipeline/test_node.py +++ b/tests/pipeline/test_node.py @@ -25,7 +25,6 @@ # # See the License for the specific language governing permissions and # limitations under the License. -import sys from functools import partial, update_wrapper, wraps from typing import Callable @@ -217,10 +216,8 @@ def test_node_invalid_equals(self): def test_node_invalid_less_than(self): n = node(identity, "input1", "output1", name="a node") - pattern_36_37 = "'<' not supported between instances of 'Node' and 'str'" - pattern_35 = "unorderable types" + pattern = "'<' not supported between instances of 'Node' and 'str'" - pattern = pattern_35 if sys.version_info[:2] == (3, 5) else pattern_36_37 with pytest.raises(TypeError, match=pattern): n < "hello" # pylint: disable=pointless-statement diff --git a/tests/template/test_kedro_cli.py b/tests/template/test_kedro_cli.py index 95d3f0ffe2..8c751c90db 100644 --- a/tests/template/test_kedro_cli.py +++ b/tests/template/test_kedro_cli.py @@ -351,13 +351,12 @@ def test_bare_lint(self, fake_kedro_cli, python_call_mock, mocker): files = ("src/tests", "src/fake_package") expected_calls = [ + mocker.call("black", files), mocker.call("flake8", ("--max-line-length=88",) + files), mocker.call( "isort", ("-rc", "-tc", "-up", "-fgw=0", "-m=3", "-w=88") + files ), ] - if sys.version_info[:2] >= (3, 6): - expected_calls.insert(0, mocker.call("black", files)) # pragma: no cover assert python_call_mock.call_args_list == expected_calls @@ -367,13 +366,12 @@ def test_file_lint(self, fake_kedro_cli, python_call_mock, mocker): files = ("kedro",) expected_calls = [ + mocker.call("black", files), mocker.call("flake8", ("--max-line-length=88",) + files), mocker.call( "isort", ("-rc", "-tc", "-up", "-fgw=0", "-m=3", "-w=88") + files ), ] - if sys.version_info[:2] >= (3, 6): - expected_calls.insert(0, mocker.call("black", files)) # pragma: no cover assert python_call_mock.call_args_list == expected_calls diff --git a/tests/tools/ipython/test_ipython_loader.py b/tests/tools/ipython/test_ipython_loader.py index c037a05ac9..0add18e39a 100644 --- a/tests/tools/ipython/test_ipython_loader.py +++ b/tests/tools/ipython/test_ipython_loader.py @@ -36,9 +36,7 @@ @pytest.fixture def dummy_project_dir(tmp_path): - # wrap in str is needed for Python 3.5 - # since tmp_path is not directly compatible with Path in that version - root = Path(str(tmp_path)).resolve() / "dummy_project" + root = Path(tmp_path / "dummy_project") root.mkdir() startup_path = root / ".ipython" / "profile_default" / "startup" startup_path.mkdir(parents=True) diff --git a/tools/min_version.py b/tools/min_version.py deleted file mode 100644 index f5267bb7a1..0000000000 --- a/tools/min_version.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2020 QuantumBlack Visual Analytics Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND -# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS -# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN -# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN -# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# -# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo -# (either separately or in combination, "QuantumBlack Trademarks") are -# trademarks of QuantumBlack. The License does not grant you any right or -# license to the QuantumBlack Trademarks. You may not use the QuantumBlack -# Trademarks or any confusingly similar mark as a trademark for your product, -# or use the QuantumBlack Trademarks in any other manner that might cause -# confusion in the marketplace, including but not limited to in advertising, -# on websites, or on software. -# -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Black and import-linter need python 3.6+, but Kedro should work on 3.5 too. -That's why we run the relevant commands conditionally on CI/precommit. - -If python version is 3.5 - just exit with 0 status. -""" -import shlex -import subprocess -import sys - -if __name__ == "__main__": - required_version = tuple(int(x) for x in sys.argv[1].strip().split(".")) - current_version = sys.version_info[:2] - - if current_version < required_version: - print("Python version is too low, exiting") - sys.exit(0) - - run_cmd = shlex.split(sys.argv[2]) - subprocess.run(run_cmd, check=True)