From 704cacb5ecabb65a224dda9d847b3a235e85f12c Mon Sep 17 00:00:00 2001 From: Anton Kirilenko Date: Thu, 1 Aug 2019 11:57:00 +0100 Subject: [PATCH] [KED-901] Introduce pre-commit hooks and cheks (#157) --- .circleci/config.yml | 5 +- .isort.cfg | 1 - .pre-commit-config.yaml | 75 +++++++++++++++++++ .readthedocs.yml | 2 - CONTRIBUTING.md | 29 ++++++- Makefile | 19 +++-- README.md | 4 +- docs/README.md | 14 ++-- docs/conf.py | 2 +- docs/kedro_logo.svg | 2 +- docs/source/02_getting_started/02_install.md | 3 - .../03_tutorial/05_package_a_project.md | 2 +- .../04_user_guide/05_nodes_and_pipelines.md | 2 +- docs/source/04_user_guide/07_advanced_io.md | 2 +- docs/source/04_user_guide/10_ipython.md | 1 - .../11_working_with_databricks.md | 2 +- docs/source/05_api_docs/kedro.pipeline.rst | 1 - docs/source/06_resources/01_faq.md | 1 - docs/source/css/qb1-sphinx-rtd.css | 2 +- extras/README.md | 2 +- features/pipeline.feature | 2 - features/run.feature | 1 - kedro/contrib/README.md | 6 +- .../contrib/io/catalog_with_default/README.md | 5 +- kedro/contrib/io/pyspark/README.md | 10 +-- .../{{ cookiecutter.repo_name }}/README.md | 2 +- .../conf/base/credentials.yml | 5 +- .../src/requirements.txt | 4 +- .../src/tests/test_run.py | 1 - requirements.txt | 14 ++-- setup.py | 7 +- test_requirements.txt | 29 +++---- tools/black_runner.py | 52 +++++++++++++ tools/license_and_headers.py | 1 - 34 files changed, 230 insertions(+), 80 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 tools/black_runner.py diff --git a/.circleci/config.yml b/.circleci/config.yml index a4997cd97e..58e657324a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,8 +22,11 @@ jobs: # Install requirements pip install -r requirements.txt -U + # venv is required for some pre-commit hooks + conda install -y virtualenv + # Install test requirements - pip install -r test_requirements.txt -U + make install-pre-commit - run: name: Run pylint and flake8 command: | diff --git a/.isort.cfg b/.isort.cfg index 3453115e06..0fb28acd32 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -6,6 +6,5 @@ include_trailing_comma=True force_grid_wrap=0 use_parentheses=True line_length=88 -skip_glob=**/template/** known_first_party=features,kedro,tests default_section=THIRDPARTY diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..d062c7c5dd --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,75 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks + +default_stages: [commit, push] + +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.2.3 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml # Checks yaml files for parseable syntax. + exclude: "^kedro/template/" + - id: check-json # Checks json files for parseable syntax. + - id: check-added-large-files + - id: check-case-conflict # Check for files that would conflict in case-insensitive filesystems + - id: check-merge-conflict # Check for files that contain merge conflict strings. + - id: debug-statements # Check for debugger imports and py37+ `breakpoint()` calls in python source. + exclude: "^kedro/template/" + - id: detect-private-key # Detects the presence of private keys + - id: requirements-txt-fixer # Sorts entries in requirements.txt + - id: flake8 + exclude: "^kedro/template/" + +- repo: https://github.com/pre-commit/mirrors-isort + rev: v4.3.21 + hooks: + - id: isort + exclude: "^kedro/template/" + +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.720 + hooks: + - id: mypy + args: [--allow-redefinition, --ignore-missing-imports] + exclude: | + (?x)( + ^kedro/template/| + ^docs/ + ) + +- repo: local + hooks: + # It's impossible to specify per-directory configuration, so we just run it many times. + # https://github.com/PyCQA/pylint/issues/618 + - id: pylint-kedro + name: "PyLint on kedro/*" + language: system + pass_filenames: false + stages: [push] + entry: pylint -j0 --disable=unnecessary-pass kedro + - id: pylint-features + name: "PyLint on features/*" + language: system + pass_filenames: false + stages: [push] + entry: pylint -j0 --disable=missing-docstring,no-name-in-module features + - id: pylint-extras + name: "PyLint on extras/*" + language: system + pass_filenames: false + stages: [push] + entry: pylint -j0 extras + - id: pylint-tests + name: "PyLint on tests/*" + language: system + pass_filenames: false + stages: [push] + entry: pylint -j0 --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name tests + # We need to make some exceptions for 3.5, that's why it's a custom runner. + - id: black + name: "Black" + language: system + pass_filenames: false + entry: python -m tools.black_runner kedro extras features tests diff --git a/.readthedocs.yml b/.readthedocs.yml index c456e06f5e..9db8ca825b 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -26,5 +26,3 @@ python: extra_requirements: - docs - requirements: test_requirements.txt - - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c36f457ce9..46bb564605 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -101,10 +101,13 @@ You can add new work to `contrib` if you do not need to create a new Kedro CLI c See the [`plugin` development documentation](https://kedro.readthedocs.io/en/latest/04_user_guide/09_developing_plugins.html) for guidance on how to design and develop a Kedro `plugin`. ## CI / CD and running checks locally -To run E2E tests you need to install the test requirements which includes `behave`, do this using the following command: +To run E2E tests you need to install the test requirements which includes `behave`. +Also we use [pre-commit](https://pre-commit.com) hooks for the repository to run the checks automatically. +It can all be installed using the following command: ```bash -pip install -r test_requirements.txt +make install-test-requirements +make install-pre-commit ``` ### Running checks locally @@ -146,3 +149,25 @@ make build-docs This command will only work on Unix-like systems and requires `pandoc` to be installed. > ❗ Running `make build-docs` in a Python 3.5 environment may sometimes yield multiple warning messages like the following: `MemoryDataSet.md: WARNING: document isn't included in any toctree`. You can simply ignore them or switch to Python 3.6+ when building documentation. + +## Hints on pre-commit usage +The checks will automatically run on all the changed files on each commit. +Even more extensive set of checks (including the heavy set of `pylint` checks) +will run before the push. + +The pre-commit/pre-push checks can be omitted by running with `--no-verify` flag, as per below: + +```bash +git commit --no-verify <...> +git push --no-verify <...> +``` +(`-n` alias works for `git commit`, but not for `git push`) + +All checks will run during CI build, so skipping checks on push will +not allow you to merge your code with failing checks. + +You can uninstall the pre-commit hooks by running +```bash +make uninstall-pre-commit +``` +`pre-commit` will still be used by `make lint`, but will install the git hooks. diff --git a/Makefile b/Makefile index ff9fb1fdc7..9366e595fb 100644 --- a/Makefile +++ b/Makefile @@ -14,13 +14,7 @@ legal: python tools/license_and_headers.py lint: - isort - pylint -j 0 --disable=unnecessary-pass kedro - pylint -j 0 --disable=missing-docstring,redefined-outer-name,no-self-use,invalid-name tests - pylint -j 0 --disable=missing-docstring,no-name-in-module features - pylint -j 0 extras - flake8 kedro tests features extras --exclude kedro/template* - mypy --allow-redefinition --ignore-missing-imports kedro tests features extras + pre-commit run -a --hook-stage push test: pytest tests @@ -38,3 +32,14 @@ devserver: build-docs package: clean install python setup.py sdist bdist_wheel + +install-test-requirements: + pip install -r test_requirements.txt + +install-pre-commit: install-test-requirements + pre-commit install --install-hooks + pre-commit install --hook-type pre-push + +uninstall-pre-commit: + pre-commit uninstall + pre-commit uninstall --hook-type pre-push diff --git a/README.md b/README.md index a8057cf16f..74632ed945 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ![Kedro Logo Banner](https://raw.githubusercontent.com/quantumblacklabs/kedro/master/img/kedro_banner.jpg) -`develop` | `master` -----------|--------- +`develop` | `master` +----------|--------- [![CircleCI](https://circleci.com/gh/quantumblacklabs/kedro/tree/develop.svg?style=shield)](https://circleci.com/gh/quantumblacklabs/kedro/tree/develop) | [![CircleCI](https://circleci.com/gh/quantumblacklabs/kedro/tree/master.svg?style=shield)](https://circleci.com/gh/quantumblacklabs/kedro/tree/master) [![Build status](https://ci.appveyor.com/api/projects/status/2u74p5g8fdc45wwh/branch/develop?svg=true)](https://ci.appveyor.com/project/QuantumBlack/kedro/branch/develop) | [![Build status](https://ci.appveyor.com/api/projects/status/2u74p5g8fdc45wwh/branch/master?svg=true)](https://ci.appveyor.com/project/QuantumBlack/kedro/branch/master) diff --git a/docs/README.md b/docs/README.md index bb7b8855d3..5e4074656e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,7 +2,7 @@ # Kedro documentation style guide -This is the style guide we have used to create [documentation about Kedro](https://kedro.readthedocs.io/en/latest/). +This is the style guide we have used to create [documentation about Kedro](https://kedro.readthedocs.io/en/latest/). When you are writing documentation for your own project, you may find it useful to follow these rules. We will also ask anyone kind enough to contribute to the Kedro documentation to follow our preferred style to maintain consistency and simplicity. However, we are not over-proscriptive and are happy to take contributions regardless, as long as you are happy if we edit your text to follow these rules. @@ -15,7 +15,7 @@ Please follow these simple rules: * Use sentence case in titles. We prefer this, _"Sentence case only has one capital except for names like Kedro"_ and not this, _"Title Case Means Capitalise Every Word"_ * Mark code blocks with the appropriate language to enable [syntax highlighting](https://support.codebasehq.com/articles/tips-tricks/syntax-highlighting-in-markdown) * We use a `bash` lexer for all codeblocks that represent the terminal, and we don't include the prompt -* Bullet points start with capitals and do not end with full-stops +* Bullet points start with capitals and do not end with full-stops * Prefer to use symbols for bullets instead of numbers unless you are specifically giving a sequence of instructions * Keep your sentences short and easy to read * Do not plagiarise other authors. Link to their text and credit them @@ -24,21 +24,21 @@ If you are in doubt, take a look at how we've written the Kedro documentation. I ## How do I build your documentation? -If you have installed Kedro, the documentation can be found by running `kedro docs` from the command line or following [this link](https://kedro.readthedocs.io/en/latest/). +If you have installed Kedro, the documentation can be found by running `kedro docs` from the command line or following [this link](https://kedro.readthedocs.io/en/latest/). If you make changes to our documentation, which is stored in the `docs/` folder of your Kedro installation, you can rebuild them within a Unix-like environment (with `pandoc` installed) with: ```bash make build-docs -``` +``` -We use the [Sphinx](https://www.sphinx-doc.org) framework to build our documentation. The resulting HTML files can be found in `docs/build/html/`. +We use the [Sphinx](https://www.sphinx-doc.org) framework to build our documentation. The resulting HTML files can be found in `docs/build/html/`. -If you are a Windows user, you can still contribute to the documentation, but you cannot rebuild it. This is fine! As long as you have made an effort to verify that your Markdown is rendering correctly, and you have followed our basic guidelines above, we will be happy to take your final draft as a pull request and rebuild it for you. +If you are a Windows user, you can still contribute to the documentation, but you cannot rebuild it. This is fine! As long as you have made an effort to verify that your Markdown is rendering correctly, and you have followed our basic guidelines above, we will be happy to take your final draft as a pull request and rebuild it for you. ## Can I contribute to Kedro documentation? -Yes! If you want to fix or extend our documentation, you'd be welcome to do so. When you are ready to submit, please read the full guide to [contributing to Kedro](../CONTRIBUTING.md). +Yes! If you want to fix or extend our documentation, you'd be welcome to do so. When you are ready to submit, please read the full guide to [contributing to Kedro](../CONTRIBUTING.md). Before you contribute any documentation, please do read the above rules for styling your Markdown. If there's something you think is missing or incorrect, and you'd like to get really meta and contribute to our style guide, please branch this file and submit a PR! diff --git a/docs/conf.py b/docs/conf.py index 6932a5bebc..e4de8e26c6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -256,7 +256,7 @@ def autolink_replacements(what: str) -> List[Tuple[str, str, str]]: These ``LambdaDataSet``s load and save Will convert to: - These :class:`kedro.io.LambdaDataSet`\s load and save + These :class:`kedro.io.LambdaDataSet` load and save Args: what: The objects to create replacement tuples for. Possible values diff --git a/docs/kedro_logo.svg b/docs/kedro_logo.svg index 45d9bcebfa..754f41860d 100644 --- a/docs/kedro_logo.svg +++ b/docs/kedro_logo.svg @@ -1 +1 @@ - \ No newline at end of file + diff --git a/docs/source/02_getting_started/02_install.md b/docs/source/02_getting_started/02_install.md index e8db143d4c..a85c27fb30 100644 --- a/docs/source/02_getting_started/02_install.md +++ b/docs/source/02_getting_started/02_install.md @@ -17,6 +17,3 @@ You should see an ASCII art graphic and the Kedro version number. For example: ![](images/kedro_graphic.png) If you do not see the graphic displayed, or have any issues with your installation, see the [FAQs](../06_resources/01_faq.md) for help. - - - diff --git a/docs/source/03_tutorial/05_package_a_project.md b/docs/source/03_tutorial/05_package_a_project.md index d76e70c313..4d8c2a5182 100644 --- a/docs/source/03_tutorial/05_package_a_project.md +++ b/docs/source/03_tutorial/05_package_a_project.md @@ -19,7 +19,7 @@ You can package your project by running `kedro package` from the command line. T You can also check out [Kedro-Docker](https://github.com/quantumblacklabs/kedro-docker), an officially supported Kedro plugin for packaging and shipping Kedro projects within [Docker](https://www.docker.com/) containers. -We also support converting your Kedro project into an Airflow project with the [Kedro-Airflow](https://github.com/quantumblacklabs/kedro-airflow) plugin. +We also support converting your Kedro project into an Airflow project with the [Kedro-Airflow](https://github.com/quantumblacklabs/kedro-airflow) plugin. ## What is next? diff --git a/docs/source/04_user_guide/05_nodes_and_pipelines.md b/docs/source/04_user_guide/05_nodes_and_pipelines.md index 1d656ec421..ff935c90d2 100644 --- a/docs/source/04_user_guide/05_nodes_and_pipelines.md +++ b/docs/source/04_user_guide/05_nodes_and_pipelines.md @@ -111,7 +111,7 @@ To tag a node, you can simply specify the `tag` argument, as follows: ```python node(func=add, inputs=["a", "b"], outputs="sum", name="adding_a_and_b", tag="node_tag") -``` +``` Moreover, you can [tag all nodes in a ``Pipeline``](./05_nodes_and_pipelines.md#tagging-pipeline-nodes). diff --git a/docs/source/04_user_guide/07_advanced_io.md b/docs/source/04_user_guide/07_advanced_io.md index f68ab43a18..e80c699ab4 100644 --- a/docs/source/04_user_guide/07_advanced_io.md +++ b/docs/source/04_user_guide/07_advanced_io.md @@ -52,7 +52,7 @@ In order to enable versioning, all of the following conditions must be met: 4. modify its `_describe`, `_load` and `_save` methods respectively to support versioning (see [`kedro.io.CSVLocalDataSet`](/kedro.io.CSVLocalDataSet) for an example implementation) AND 2. In the `catalog.yml` config file you must enable versioning by setting `versioned` attribute to `true` for the given dataset. -An example dataset could look similar to the below: +An example dataset could look similar to the below: ```python from pathlib import Path diff --git a/docs/source/04_user_guide/10_ipython.md b/docs/source/04_user_guide/10_ipython.md index 044da8844a..c055b82ec4 100644 --- a/docs/source/04_user_guide/10_ipython.md +++ b/docs/source/04_user_guide/10_ipython.md @@ -174,4 +174,3 @@ Placing your `Notebook.ipynb` file anywhere in `new-kedro-project/notebooks/`, ` #### Error handling In case this script fails to execute any of your Kedro project startup scripts, global variable `load_kedro_errors` will contain a dictionary with the key pointing to the failed script path and the value containing exception object. - diff --git a/docs/source/04_user_guide/11_working_with_databricks.md b/docs/source/04_user_guide/11_working_with_databricks.md index 6d774f05a4..dfc31bcc63 100644 --- a/docs/source/04_user_guide/11_working_with_databricks.md +++ b/docs/source/04_user_guide/11_working_with_databricks.md @@ -9,7 +9,7 @@ Databricks Connect connects your favourite IDE (IntelliJ, Eclipse, [VS Code](01_ You can setup Databricks Connect according to the instructions listed [here](https://docs.databricks.com/user-guide/dev-tools/db-connect.html). -> *Note:* You will need to uninstall PySpark, as Databricks Connect will install it for you. This method only works for 5.x versions of Databricks clusters and disables use of Databricks Notebook. +> *Note:* You will need to uninstall PySpark, as Databricks Connect will install it for you. This method only works for 5.x versions of Databricks clusters and disables use of Databricks Notebook. ## GitHub workflow with Databricks diff --git a/docs/source/05_api_docs/kedro.pipeline.rst b/docs/source/05_api_docs/kedro.pipeline.rst index ab5af7e414..f6fbebbe20 100644 --- a/docs/source/05_api_docs/kedro.pipeline.rst +++ b/docs/source/05_api_docs/kedro.pipeline.rst @@ -30,4 +30,3 @@ kedro.pipeline kedro.pipeline.decorators.log_time kedro.pipeline.decorators.mem_profile - diff --git a/docs/source/06_resources/01_faq.md b/docs/source/06_resources/01_faq.md index 6f0296756a..76682c7ebe 100644 --- a/docs/source/06_resources/01_faq.md +++ b/docs/source/06_resources/01_faq.md @@ -128,4 +128,3 @@ A local copy of documentation about Kedro can be generated by running `kedro doc ## How can I find out more about Kedro? Kedro is on GitHub, and our preferred community channel for feedback is through [GitHub issues](https://github.com/quantumblacklabs/kedro/issues). We will be updating the codebase regularly, and you can find news about updates and features we introduce by heading over to [RELEASE.md](https://github.com/quantumblacklabs/kedro/blob/develop/RELEASE.md). - diff --git a/docs/source/css/qb1-sphinx-rtd.css b/docs/source/css/qb1-sphinx-rtd.css index 79b7d2f1a6..911878abe1 100644 --- a/docs/source/css/qb1-sphinx-rtd.css +++ b/docs/source/css/qb1-sphinx-rtd.css @@ -402,4 +402,4 @@ h1, h2, .rst-content .toctree-wrapper p.caption, h3, h4, h5, h6, legend { .wy-body-for-nav .rst-versions { font-size: 16px; line-height: 1; -} \ No newline at end of file +} diff --git a/extras/README.md b/extras/README.md index 41bf2c1ebd..799619c280 100644 --- a/extras/README.md +++ b/extras/README.md @@ -6,4 +6,4 @@ This script helps to locate Kedro project and run IPython startup scripts in it when working with Jupyter Notebooks and IPython sessions. This script will automatically identify your Kedro project root directory and execute all Python scripts from `/.ipython/profile_default/startup` directory. -The details can be found in [the user guide](](https://kedro.readthedocs.io/en/latest/04_user_guide/10_ipython.html)). \ No newline at end of file +The details can be found in [the user guide](](https://kedro.readthedocs.io/en/latest/04_user_guide/10_ipython.html)). diff --git a/features/pipeline.feature b/features/pipeline.feature index 5add70a791..e5724516f4 100644 --- a/features/pipeline.feature +++ b/features/pipeline.feature @@ -42,5 +42,3 @@ Feature: Pipelines that can be run from a library or a template project Given I have executed the kedro command "install" When the template pipeline is run Then it should successfully produce the results - - diff --git a/features/run.feature b/features/run.feature index c08899862e..6eadfea658 100644 --- a/features/run.feature +++ b/features/run.feature @@ -89,4 +89,3 @@ Feature: Run Project And I have deleted the credentials file When I execute the kedro command "run" Then I should get a successful exit code - diff --git a/kedro/contrib/README.md b/kedro/contrib/README.md index 074469d444..33eceb5660 100644 --- a/kedro/contrib/README.md +++ b/kedro/contrib/README.md @@ -1,6 +1,6 @@ # Kedro contrib -The contrib directory is meant to contain user contributions, these +The contrib directory is meant to contain user contributions, these contributions might get merged into core Kedro at some point in the future. When create a new module in `contrib`, place it exactly where it would be if it @@ -9,7 +9,7 @@ was merged into core Kedro. For example, data sets are under the core package `kedro.io`. If you are contributing a Data Set you should have the following directory: `kedro/contrib/my_project/io/` - i.e., the name of your project before the -`kedro` package path. +`kedro` package path. This is how a module would look like under `kedro/contrib`: ``` @@ -36,4 +36,4 @@ extras_require={ ``` Please notice that a readme with instructions about how to use your module -and 100% test coverage are required to accept a PR. +and 100% test coverage are required to accept a PR. diff --git a/kedro/contrib/io/catalog_with_default/README.md b/kedro/contrib/io/catalog_with_default/README.md index 967ad8e350..5df1a3b125 100644 --- a/kedro/contrib/io/catalog_with_default/README.md +++ b/kedro/contrib/io/catalog_with_default/README.md @@ -14,8 +14,8 @@ from kedro.runner import SequentialRunner def default_io(name): return ParquetLocalDataSet(name) -catalog = DataCatalogWithDefault({}, - default=default_io, +catalog = DataCatalogWithDefault({}, + default=default_io, default_prefix='data/') def my_node(input): @@ -36,4 +36,3 @@ Less safe during production, very handy during development. ### Motivation and Context Very useful during development, saves a lot of time. Usually most datasets in a pipeline come from a single-source. - diff --git a/kedro/contrib/io/pyspark/README.md b/kedro/contrib/io/pyspark/README.md index 524ef6c1a4..8ce01022f5 100644 --- a/kedro/contrib/io/pyspark/README.md +++ b/kedro/contrib/io/pyspark/README.md @@ -60,8 +60,8 @@ Loading and saving spark `DataFrame`s using Kedro can be easily done using the [ ```python from kedro.contrib.io.pyspark import SparkDataSet -csv = SparkDataSet('../data/01_raw/2015_points_old.csv', - file_format='csv', +csv = SparkDataSet('../data/01_raw/2015_points_old.csv', + file_format='csv', load_args={'header': True}) df = csv.load() \ @@ -74,9 +74,9 @@ df.show() ### Save it as parquet to your local disk ```python -parquet = SparkDataSet('../data/01_raw/2015_points_old_parquet', - file_format='parquet', - save_args={'mode': 'overwrite', +parquet = SparkDataSet('../data/01_raw/2015_points_old_parquet', + file_format='parquet', + save_args={'mode': 'overwrite', 'compression': 'none'}) parquet.save(df) ``` diff --git a/kedro/template/{{ cookiecutter.repo_name }}/README.md b/kedro/template/{{ cookiecutter.repo_name }}/README.md index 673ab77909..c3dc8018c1 100644 --- a/kedro/template/{{ cookiecutter.repo_name }}/README.md +++ b/kedro/template/{{ cookiecutter.repo_name }}/README.md @@ -87,7 +87,7 @@ scope: `proj_dir`, `proj_name`, `conf`, `io`, `parameters` and `startup_error`. Once you are happy with a notebook, you may want to move your code over into the Kedro project structure for the next stage in your development. This is done through a mixture of [cell tagging](https://jupyter-notebook.readthedocs.io/en/stable/changelog.html#cell-tags) and Kedro CLI commands. -By adding the `node` tag to a cell and running the command below, the cell's source code will be copied over to a Python file within `src//nodes/`. +By adding the `node` tag to a cell and running the command below, the cell's source code will be copied over to a Python file within `src//nodes/`. ``` kedro jupyter convert ``` diff --git a/kedro/template/{{ cookiecutter.repo_name }}/conf/base/credentials.yml b/kedro/template/{{ cookiecutter.repo_name }}/conf/base/credentials.yml index fe22a1a0ae..82cbc95589 100644 --- a/kedro/template/{{ cookiecutter.repo_name }}/conf/base/credentials.yml +++ b/kedro/template/{{ cookiecutter.repo_name }}/conf/base/credentials.yml @@ -1,8 +1,8 @@ # Here you can define credentials for different data sets and environment. -# +# # THIS FILE MUST BE PLACED IN `conf/local`. DO NOT PUSH THIS FILE TO GitHub. -# +# # Example: # # dev_s3: @@ -16,4 +16,3 @@ # dev_sql: # username: admin # password: admin - diff --git a/kedro/template/{{ cookiecutter.repo_name }}/src/requirements.txt b/kedro/template/{{ cookiecutter.repo_name }}/src/requirements.txt index 374041a02d..088c9a148d 100644 --- a/kedro/template/{{ cookiecutter.repo_name }}/src/requirements.txt +++ b/kedro/template/{{ cookiecutter.repo_name }}/src/requirements.txt @@ -1,9 +1,9 @@ -kedro=={{ cookiecutter.kedro_version }} ipython>=7.0.0, <8.0 jupyter>=1.0.0, <2.0 jupyterlab==0.31.1 +kedro=={{ cookiecutter.kedro_version }} nbstripout==0.3.3 -pytest>=3.4, <4.0 pytest-cov>=2.5, <3.0 pytest-mock>=1.7.1,<2.0 +pytest>=3.4, <4.0 wheel==0.32.2 diff --git a/kedro/template/{{ cookiecutter.repo_name }}/src/tests/test_run.py b/kedro/template/{{ cookiecutter.repo_name }}/src/tests/test_run.py index 92ca544d42..5a8a33cc30 100644 --- a/kedro/template/{{ cookiecutter.repo_name }}/src/tests/test_run.py +++ b/kedro/template/{{ cookiecutter.repo_name }}/src/tests/test_run.py @@ -56,4 +56,3 @@ def test_project_name(self, project_context): def test_project_version(self, project_context): assert project_context.project_version == "{{ cookiecutter.kedro_version }}" - diff --git a/requirements.txt b/requirements.txt index 15587246ad..74e243a34b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,15 @@ +anyconfig==0.9.7 click<8.0 cookiecutter==1.6.0 +memory_profiler==0.54.0 pandas>=0.23.0, <0.25.0 -PyYAML>=4.2, <6.0 -tables==3.5.1 pyarrow>=0.12.0, <1.0.0 +python-json-logger==0.1.9 +PyYAML>=4.2, <6.0 +requests>=2.21.0, <3.0.0 +s3fs>=0.3.0, <1.0 SQLAlchemy>=1.2.0, <2.0 +tables==3.5.1 toposort==1.5 xlrd>=1.0.0, <2.0 xlsxwriter>=1.0.7, <2.0 -anyconfig==0.9.7 -python-json-logger==0.1.9 -memory_profiler==0.54.0 -s3fs>=0.3.0, <1.0 -requests>=2.21.0, <3.0.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 4833644141..63aef69885 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,12 @@ # get package version with open(path.join(here, name, "__init__.py"), encoding="utf-8") as f: - version = re.search(r'__version__ = ["\']([^"\']+)', f.read()).group(1) + result = re.search(r'__version__ = ["\']([^"\']+)', f.read()) + + if not result: + raise ValueError("Can't find the version in kedro/__init__.py") + + version = result.group(1) # get the dependencies and installs with open("requirements.txt", "r", encoding="utf-8") as f: diff --git a/test_requirements.txt b/test_requirements.txt index cadfd8b356..a18268e794 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,21 +1,22 @@ -r requirements.txt -pytest>=3.9, <4.0 -pytest-cov>=2.5, <3.0 -pytest-mock>=1.7.1,<2.0 -pylint>=2.3.1, <3.0 -flake8>=3.5,<4.0 -behave==1.2.6 -moto==1.3.7 -nbconvert>=5.3.1, <6.0 -nbformat>=4.4.0, <5.0 -pyspark>=2.2.0, <3.0 -codacy-coverage azure-storage-blob>=1.1.0, <2.0 azure-storage-file>=1.1.0, <2.0 azure-storage-queue>=1.1.0, <2.0 +behave==1.2.6 +biopython>=1.73, <2.0 +codacy-coverage +flake8>=3.5,<4.0 joblib==0.12.3 +moto==1.3.7 +mypy<=1.0 +nbconvert>=5.3.1, <6.0 +nbformat>=4.4.0, <5.0 +pre-commit>=1.17.0, <2.0.0 psutil==5.4.7 -wheel==0.32.2 -biopython>=1.73, <2.0 +pylint>=2.3.1, <3.0 +pyspark>=2.2.0, <3.0 +pytest-cov>=2.5, <3.0 +pytest-mock>=1.7.1,<2.0 +pytest>=3.9, <4.0 requests-mock>=1.6.0, <2.0.0 -mypy<=1.0 +wheel==0.32.2 diff --git a/tools/black_runner.py b/tools/black_runner.py new file mode 100644 index 0000000000..2fa0b1577c --- /dev/null +++ b/tools/black_runner.py @@ -0,0 +1,52 @@ +# Copyright 2018-2019 QuantumBlack Visual Analytics Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +# NONINFRINGEMENT. IN NO EVENT WILL THE LICENSOR OR OTHER CONTRIBUTORS +# BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF, OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# The QuantumBlack Visual Analytics Limited ("QuantumBlack") name and logo +# (either separately or in combination, "QuantumBlack Trademarks") are +# trademarks of QuantumBlack. The License does not grant you any right or +# license to the QuantumBlack Trademarks. You may not use the QuantumBlack +# Trademarks or any confusingly similar mark as a trademark for your product, +# or use the QuantumBlack Trademarks in any other manner that might cause +# confusion in the marketplace, including but not limited to in advertising, +# on websites, or on software. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Black needs python 3.6+, but Kedro should work on 3.5 too, +that's why we can't put ``black`` into test_requirements.txt and +have to install it manually like that. + +If python version is 3.5 - just exit with 0 status. +""" +import platform +import subprocess +import sys + +VERSION = tuple(map(int, platform.python_version_tuple()[:2])) + +if VERSION < (3, 6): + print("Python version is too low, exiting") + sys.exit(0) + +try: + import black # noqa: F401 pylint: disable=unused-import +except ImportError: + subprocess.run(["pip", "install", "black"], check=True) + + +subprocess.run(["black"] + sys.argv[1:], check=True) diff --git a/tools/license_and_headers.py b/tools/license_and_headers.py index 14f1730711..594137398a 100644 --- a/tools/license_and_headers.py +++ b/tools/license_and_headers.py @@ -27,7 +27,6 @@ # limitations under the License. import glob -import sys PATHS_REQUIRING_HEADER = ["kedro", "tests"] LEGAL_HEADER_FILE = "legal_header.txt"