[KED-1253] Drop support for Python 3.5 (#506)

.circleci/config.yml

@@ -1,9 +1,6 @@
 version: 2.1
 
 executors:
-  py35:
-    docker:
-      - image: 350138855857.dkr.ecr.eu-west-2.amazonaws.com/kedro-builder:3.5
   py36:
     docker:
       - image: 350138855857.dkr.ecr.eu-west-2.amazonaws.com/kedro-builder:3.6
@@ -116,18 +113,6 @@ commands:
           command: make pip-compile
 
 jobs:
-  unit_tests_35:
-    executor: py35
-    steps: [unit_tests]
-
-  linters_35:
-    executor: py35
-    steps: [lint]
-
-  e2e_tests_35:
-    executor: py35
-    steps: [e2e_tests]
-
   unit_tests_36:
     executor: py36
     steps: [unit_tests]
@@ -172,10 +157,18 @@ jobs:
     executor: py38
     steps: [e2e_tests]
 
-  pip_compile:
+  pip_compile_36:
+    executor: py36
+    steps: [pip_compile]
+
+  pip_compile_37:
     executor: py37
     steps: [pip_compile]
 
+  pip_compile_38:
+    executor: py38
+    steps: [pip_compile]
+
   all_circleci_checks_succeeded:
     docker:
       - image: circleci/python # any light-weight image
@@ -189,9 +182,6 @@ workflows:
   version: 2
   regular:
     jobs:
-      - unit_tests_35
-      - linters_35
-      - e2e_tests_35
       - unit_tests_36
       - linters_36
       - e2e_tests_36
@@ -203,12 +193,11 @@ workflows:
       - unit_tests_38
       - linters_38
       - e2e_tests_38
-      - pip_compile
+      - pip_compile_36
+      - pip_compile_37
+      - pip_compile_38
       - all_circleci_checks_succeeded:
           requires:
-            - unit_tests_35
-            - linters_35
-            - e2e_tests_35
             - unit_tests_36
             - linters_36
             - e2e_tests_36
@@ -220,4 +209,6 @@ workflows:
             - unit_tests_38
             - linters_38
             - e2e_tests_38
-            - pip_compile
+            - pip_compile_36
+            - pip_compile_37
+            - pip_compile_38

.pre-commit-config.yaml

@@ -79,12 +79,11 @@ repos:
             pass_filenames: false
             stages: [manual]
             entry: pylint --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name,protected-access,too-many-arguments tests
-          # We need to make some exceptions for 3.5, that's why it's a custom runner.
           - id: black
             name: "Black"
             language: system
             pass_filenames: false
-            entry: python -m tools.min_version 3.6 "black kedro features tests"
+            entry: black kedro features tests
           - id: legal
             name: "Licence check"
             language: system
@@ -94,7 +93,7 @@ repos:
             name: "Import Linter"
             language: system
             pass_filenames: false
-            entry: python -m tools.min_version 3.6 lint-imports
+            entry: lint-imports
           - id: secret_scan
             name: "Secret scan"
             language: system

CONTRIBUTING.md

@@ -83,7 +83,7 @@ make build-docs
  - We use [Anaconda](https://www.anaconda.com/distribution/) as a preferred virtual environment
  - We use [SemVer](https://semver.org/) for versioning
 
-Our code is designed to be compatible with Python 3.5 onwards and our style guidelines are (in cascading order):
+Our code is designed to be compatible with Python 3.6 onwards and our style guidelines are (in cascading order):
 
 * [PEP 8 conventions](https://www.python.org/dev/peps/pep-0008/) for all Python code
 * [Google docstrings](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) for code comments
@@ -193,8 +193,6 @@ make build-docs
 
 This command will only work on Unix-like systems and requires `pandoc` to be installed.
 
-> ❗ Running `make build-docs` in a Python 3.5 environment may sometimes yield multiple warning messages like the following: `MemoryDataSet.md: WARNING: document isn't included in any toctree`. You can simply ignore them or switch to Python 3.6+ when building documentation.
-
 ## Hints on pre-commit usage
 The checks will automatically run on all the changed files on each commit.
 Even more extensive set of checks (including the heavy set of `pylint` checks)

README.md

@@ -5,7 +5,7 @@
 | Theme | Status |
 |------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | Latest Release | [![PyPI version](https://badge.fury.io/py/kedro.svg)](https://pypi.org/project/kedro/) |
-| Python Version | [![Python Version](https://img.shields.io/badge/python-3.5%20%7C%203.6%20%7C%203.7%20%7C%203.8-blue.svg)](https://pypi.org/project/kedro/) |
+| Python Version | [![Python Version](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue.svg)](https://pypi.org/project/kedro/) |
 | `master` Branch Build | [![CircleCI](https://circleci.com/gh/quantumblacklabs/kedro/tree/master.svg?style=shield)](https://circleci.com/gh/quantumblacklabs/kedro/tree/master) |
 | `develop` Branch Build | [![CircleCI](https://circleci.com/gh/quantumblacklabs/kedro/tree/develop.svg?style=shield)](https://circleci.com/gh/quantumblacklabs/kedro/tree/develop) |
 | Documentation Build | [![Documentation](https://readthedocs.org/projects/kedro/badge/?version=latest)](https://kedro.readthedocs.io/) |

RELEASE.md

@@ -24,6 +24,7 @@
 * Deleted obsoleted datasets from `kedro.io`.
 * Deleted `kedro.contrib` and `extras` folders.
 * `Pipeline.name` has been removed in favour of `Pipeline.tag()`.
+* Python 3.5 is no longer supported by the current and all future versions of Kedro.
 
 ### Migration guide from Kedro 0.15.* to Upcoming Release
 #### Migration for datasets

docs/source/01_introduction/01_introduction.md

@@ -19,7 +19,7 @@ You can also check out the resources section for answers to [frequently asked qu
 We have designed the documentation in general, and the tutorial in particular, for beginners to get started creating their own Kedro projects in Python. If you have elementary knowledge using Python then you might find the Kedro learning curve more challenging.  However, we have simplified the tutorial by providing all Python functions required to create your data pipelines.
 
 >
-> *Note:* There are a number of excellent online resources for learning Python, but be aware that you should choose those that reference Python 3, as Kedro is built for Python 3.5+. There are many curated lists of online resources, such as:
+> *Note:* There are a number of excellent online resources for learning Python, but be aware that you should choose those that reference Python 3, as Kedro is built for Python 3.6+. There are many curated lists of online resources, such as:
 >
 > - [Official Python programming language website](https://www.python.org/)
 > - [List of free programming books and tutorials](https://github.com/EbookFoundation/free-programming-books/blob/master/free-programming-books.md#python)

docs/source/05_api_docs/index.rst

@@ -19,9 +19,9 @@ Welcome to Kedro's API docs and tutorials!
     :target: https://opensource.org/licenses/Apache-2.0
     :alt: License is Apache 2.0
 
-.. image:: https://img.shields.io/badge/python-3.5%20%7C%203.6%20%7C%203.7-blue.svg
+.. image:: https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue.svg
     :target: https://pypi.org/project/kedro/
-    :alt: Python version 3.5, 3.6, 3.7
+    :alt: Python version 3.6, 3.7, 3.8
 
 .. image:: https://badge.fury.io/py/kedro.svg
     :target: https://pypi.org/project/kedro/

docs/source/06_resources/01_faq.md

@@ -135,9 +135,9 @@ You can find a list of Kedro projects in the [`kedro-examples`](https://github.c
 +----------------+---------------------------------------------------------------------------------------------------+
 ```
 
-## What version of Python does Kedro use?
+## What version of Python does Kedro support?
 
-Kedro is built for Python 3.5, 3.6, 3.7 and 3.8.
+Kedro is built for Python 3.6, 3.7 and 3.8.
 
 ## How do I upgrade Kedro?
 

docs/source/06_resources/03_commands_reference.md

@@ -83,7 +83,7 @@ This [line magic](https://ipython.readthedocs.io/en/stable/interactive/magics.ht
 Copy the code from cells [tagged](https://jupyter-notebook.readthedocs.io/en/stable/changelog.html#cell-tags) with `node` tag into Python files under `src/<package_name>/nodes/` in a Kedro project.
 
 ### `kedro lint`
-Lint your project code using the `kedro lint` command. Your project is linted with [`black`](https://github.com/psf/black) (Python 3.6+), [`flake8`](https://gitlab.com/pycqa/flake8) and [`isort`](https://github.com/timothycrosley/isort).
+Lint your project code using the `kedro lint` command. Your project is linted with [`black`](https://github.com/psf/black), [`flake8`](https://gitlab.com/pycqa/flake8) and [`isort`](https://github.com/timothycrosley/isort).
 
 ### `kedro activate-nbstripout`
 Typically output cells of Jupyter Notebook should not be tracked by git, especially if they contain sensitive information.

kedro/template/{{ cookiecutter.repo_name }}/kedro_cli.py

@@ -294,16 +294,11 @@ def lint(files):
     try:
         import flake8
         import isort
+        import black
     except ImportError as exc:
         raise KedroCliError(NO_DEPENDENCY_MESSAGE.format(exc.name))
 
-    if sys.version_info[:2] >= (3, 6):
-        try:
-            import black
-        except ImportError:
-            raise KedroCliError(NO_DEPENDENCY_MESSAGE.format("black"))
-        python_call("black", files)
-
+    python_call("black", files)
     python_call("flake8", ("--max-line-length=88",) + files)
     python_call("isort", ("-rc", "-tc", "-up", "-fgw=0", "-m=3", "-w=88") + files)
 

kedro/template/{{ cookiecutter.repo_name }}/src/requirements.txt

@@ -1,4 +1,4 @@
-black==v19.10b0; python_version >= '3.6'
+black==v19.10b0
 flake8>=3.7.9, <4.0
 ipython>=7.0.0, <8.0
 isort>=4.3.21, <5.0

setup.py

@@ -102,8 +102,7 @@
         "pyarrow>=0.12.0, <1.0.0",
         "xlrd>=1.0.0, <2.0",
         "xlsxwriter>=1.0.0, <2.0",
-        "tables>=3.4.4, <3.6; python_version < '3.6'",
-        "tables>=3.6; python_version >= '3.6'",
+        "tables>=3.6",
     ],
     "spark": ["pyspark>=2.2.0, <3.0", "hdfs>=2.5.8, <3.0", "s3fs>=0.3.0, <1.0"],
 }
@@ -118,7 +117,7 @@
     long_description=readme,
     long_description_content_type="text/markdown",
     url="https://github.com/quantumblacklabs/kedro",
-    python_requires=">=3.5, <3.9",
+    python_requires=">=3.6, <3.9",
     packages=find_packages(exclude=["docs*", "tests*", "tools*", "features*"]),
     include_package_data=True,
     tests_require=test_requires,
@@ -130,7 +129,6 @@
     keywords="pipelines, machine learning, data pipelines, data science, data engineering",
     classifiers=[
         "Development Status :: 4 - Beta",
-        "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",

test_requirements.txt

@@ -5,13 +5,13 @@ azure-storage-queue>=1.1.0, <2.0
 bandit>=1.6.2, <2.0
 behave==1.2.6
 biopython>=1.73, <2.0
-black==v19.10.b0; python_version >= '3.6'
+black==v19.10.b0
 dask[complete]>=2.6.0, <3.0
 flake8>=3.5,<4.0
 gcsfs>=0.3.0, <1.0
 geopandas>=0.6.0, <1.0
 hdfs>=2.5.8, <3.0
-import-linter==1.0; python_version >= '3.6'
+import-linter==1.0
 joblib>=0.14
 jupyter_client>=5.1.0, <6.0
 matplotlib>=3.0.3, <4.0
@@ -34,8 +34,7 @@ pytest>=3.9, <4.0
 requests-mock>=1.6.0, <2.0.0
 s3fs>=0.3.0, <1.0  # Needs to be at least 0.3.0 to make use of `cachable` attribute on S3FileSystem
 SQLAlchemy>=1.2.0, <2.0
-tables>=3.4.4, <3.6; python_version < '3.6'  # 3.6.0 breaks on install, HDF5 library not found
-tables>=3.6; python_version >= '3.6'
+tables>=3.6
 trufflehog>=2.1.0, <3.0
 wheel==0.32.2
 xlrd>=1.0.0, <2.0

tests/io/test_incremental_dataset.py

@@ -55,9 +55,7 @@ def partitioned_data_pandas():
 
 @pytest.fixture
 def local_csvs(tmp_path, partitioned_data_pandas):
-    # tmp_path on Python 3.5 is a pathlib2.Path instance, which cannot be
-    # handled directly by Path instance, hence interim conversion to string
-    local_dir = Path(str(tmp_path / "csvs"))
+    local_dir = Path(tmp_path / "csvs")
     local_dir.mkdir()
 
     for k, data in partitioned_data_pandas.items():

tests/pipeline/test_node.py

@@ -25,7 +25,6 @@
 #
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import sys
 from functools import partial, update_wrapper, wraps
 from typing import Callable
 
@@ -217,10 +216,8 @@ def test_node_invalid_equals(self):
 
     def test_node_invalid_less_than(self):
         n = node(identity, "input1", "output1", name="a node")
-        pattern_36_37 = "'<' not supported between instances of 'Node' and 'str'"
-        pattern_35 = "unorderable types"
+        pattern = "'<' not supported between instances of 'Node' and 'str'"
 
-        pattern = pattern_35 if sys.version_info[:2] == (3, 5) else pattern_36_37
         with pytest.raises(TypeError, match=pattern):
             n < "hello"  # pylint: disable=pointless-statement
 

tests/template/test_kedro_cli.py

@@ -351,13 +351,12 @@ def test_bare_lint(self, fake_kedro_cli, python_call_mock, mocker):
 
         files = ("src/tests", "src/fake_package")
         expected_calls = [
+            mocker.call("black", files),
             mocker.call("flake8", ("--max-line-length=88",) + files),
             mocker.call(
                 "isort", ("-rc", "-tc", "-up", "-fgw=0", "-m=3", "-w=88") + files
             ),
         ]
-        if sys.version_info[:2] >= (3, 6):
-            expected_calls.insert(0, mocker.call("black", files))  # pragma: no cover
 
         assert python_call_mock.call_args_list == expected_calls
 
@@ -367,13 +366,12 @@ def test_file_lint(self, fake_kedro_cli, python_call_mock, mocker):
 
         files = ("kedro",)
         expected_calls = [
+            mocker.call("black", files),
             mocker.call("flake8", ("--max-line-length=88",) + files),
             mocker.call(
                 "isort", ("-rc", "-tc", "-up", "-fgw=0", "-m=3", "-w=88") + files
             ),
         ]
-        if sys.version_info[:2] >= (3, 6):
-            expected_calls.insert(0, mocker.call("black", files))  # pragma: no cover
 
         assert python_call_mock.call_args_list == expected_calls
 

tests/tools/ipython/test_ipython_loader.py

@@ -36,9 +36,7 @@
 
 @pytest.fixture
 def dummy_project_dir(tmp_path):
-    # wrap in str is needed for Python 3.5
-    # since tmp_path is not directly compatible with Path in that version
-    root = Path(str(tmp_path)).resolve() / "dummy_project"
+    root = Path(tmp_path / "dummy_project")
     root.mkdir()
     startup_path = root / ".ipython" / "profile_default" / "startup"
     startup_path.mkdir(parents=True)