From d3171f1fc12292adc90a59fbb9ad357cbab0a906 Mon Sep 17 00:00:00 2001 From: Jason Hite <868722+jasonmhite@users.noreply.github.com> Date: Tue, 12 Sep 2023 06:43:31 -0400 Subject: [PATCH] Allow for customization of pipeline templates (#2701) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Remove non-user-facing release notes about pip pin (#2854) Signed-off-by: Jason Hite * Allow for customization of pipeline templates Controlled via `settings.PIPELINE_TEMPLATE_PATH`. Signed-off-by: Jason Hite * Update lookup based on feedback Instead of lookups on `settings.py` just statically look for a `templates` directory in the project root. If this folder is present it will override the default global template, otherwise fall back to the previous global default. Signed-off-by: Jason Hite * Add CLI flag for pipeline creation template -t/--template Precedence for template_path is: command line > project templates/pipeline dir > global default pipeline create now will also print the template folder used. Signed-off-by: Jason Hite * First basic unit test TODO is testing the CLI flag too, and maybe testing that the cleanup works with custom templates. Signed-off-by: Jason Hite * Add test to check overriding template path on the CLI Signed-off-by: Jason Hite * Linter formatting fixes Signed-off-by: Jason Hite * Release 0.18.12 (#2871) * bump version and prepare for new release Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update RELEASE.md Added community contributions * add DeltaTableDataSet to rst Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update setup.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update setup.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update setup.py Co-authored-by: Juan Luis Cano Rodríguez * temporarily use the fixed kedro-plugins branch Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update setup.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * test docsbuild on merels branch Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * use kedro-datasets new release Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> --------- Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Co-authored-by: Jo Stichbury Co-authored-by: Juan Luis Cano Rodríguez Signed-off-by: Jason Hite * Fix broken link to Prefect website in deploy guide (#2885) Signed-off-by: Jason Hite * Clarify the argument to kedro micropkg package (#2835) Co-authored-by: Juan Luis Cano Rodríguez Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> Signed-off-by: Jason Hite * Allow registering of custom resolvers to `OmegaConfigLoader` (#2869) * Allow registering of custom resolvers to OCL Signed-off-by: Ankita Katiyar * Complete doc string Signed-off-by: Ankita Katiyar * Add test for overwritten resolvers Signed-off-by: Ankita Katiyar * Update test for overwritten resolvers Signed-off-by: Ankita Katiyar * Remove replace=True by default Signed-off-by: Ankita Katiyar * Update release notes Signed-off-by: Ankita Katiyar * Update release notes Signed-off-by: Ankita Katiyar * Add debug level log for registering new resolver Signed-off-by: Ankita Katiyar --------- Signed-off-by: Ankita Katiyar Signed-off-by: Jason Hite * Document the use of custom resolvers with `OmegaConfigLoader` (#2896) * Update docs for custom resolvers for OCL Signed-off-by: Ankita Katiyar * Update FAQ with correct link Signed-off-by: Ankita Katiyar * Fix typo Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> * Add note for oc.env Signed-off-by: Ankita Katiyar * Suggestions from code review Signed-off-by: Ankita Katiyar --------- Signed-off-by: Ankita Katiyar Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> Signed-off-by: Jason Hite * Update kedro pipeline create to use new /conf file structure (#2856) * add parameters_pipelineNane.yml template in the conf folder Signed-off-by: Dmitry Sorokin * del yml template from parameters folder Signed-off-by: Dmitry Sorokin * modify delete pipeline command to use correct path to yml files Signed-off-by: Dmitry Sorokin * modify test_pipeline move parameters foldername to the yml filename Signed-off-by: Dmitry Sorokin * collect configs to package not only from parameters folder, but from core conf folder also Signed-off-by: Dmitry Sorokin * move parameters foldername to the yml filename, remove nesting Signed-off-by: Dmitry Sorokin * move parameters foldername to the yml filename, remove nesting Signed-off-by: Dmitry Sorokin * Add changes to RELEASE.md Signed-off-by: Dmitry Sorokin * Update RELEASE.md Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> * Update RELEASE.md, change to release 0.18.13 Signed-off-by: Dmitry Sorokin * save old structure for 'pipeline delete' command for backward compatibility Signed-off-by: Dmitry Sorokin * modify test_pipeline for 'pipeline delete' backward compability Signed-off-by: Dmitry Sorokin --------- Signed-off-by: Dmitry Sorokin Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> Signed-off-by: Jason Hite * Update kedro catalog create to use new /conf structure (#2884) * modify test_catalog, remove nesting in the catalog folder Signed-off-by: Dmitry Sorokin * move catalog foldername to the yml filename, remove nesting Signed-off-by: Dmitry Sorokin * Update RELEASE.md Signed-off-by: Dmitry Sorokin * Modify docs, fix broken link Signed-off-by: Dmitry Sorokin --------- Signed-off-by: Dmitry Sorokin Signed-off-by: Jason Hite * Add migration steps for `ConfigLoader` to `OmegaConfigLoader` (#2887) * Add migration steps for CL to OCL Signed-off-by: Merel Theisen Signed-off-by: Jason Hite * Fix #2498 Adding logging issue 2498 (#2842) * Fixing typo n on line 114 file /docs/source/nodes_and_pipelines/run_a_pipeline.md from issue #2607 Signed-off-by: laizaparizotto * Removed redundant packages at lines 66, 67 and 68 on file kedro/docs/source/nodes_and_pipelines/run_a_pipeline.md, packages: from kedro.io import AbstractDataSet, DataCatalog, MemoryDataSet, from kedro.pipeline import Pipeline and from kedro.runner.runner import AbstractRunner Signed-off-by: laizaparizotto * Add logger object for logging when computing micropackage version Signed-off-by: laizaparizotto * Added new format to the logging Signed-off-by: laizaparizotto --------- Signed-off-by: laizaparizotto Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> Signed-off-by: Jason Hite * Try only trigger docs build when release notes updated (#2907) Signed-off-by: Merel Theisen Signed-off-by: Jason Hite * Add Python 3.11 support to Kedro (#2851) * add 3.11 Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update setup.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * limit number of python version tests Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * fix setup.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * delta-spark version Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * try original setup Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * remove 3.11 from circleci Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update setup.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update setup.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * pin matplotlib to 3.7.2 for python 3.11 Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update test_import.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * add noks test adjustments Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update setup.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update setup.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * replace eq with is for multi_catalog Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * minor fixes Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * remove matplotlib 3.11 pin Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * matplotlib pin for 3.11 Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * run original matplotlib pin with ubuntu only Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * run circleci Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * pin spark < 3.4 as its breaking Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * spark datasets revert earlier changes back to original Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * delta Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * remove all windows tests from GHA Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * skip coverage as not needed Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Checking only WIN tests (will revert) Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update unit-tests.yml Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * undo previous commit Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * add msbuild Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * add ubuntu tests back in and remove 3.11 tests from circleci add in msbuild for the other tests Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * set hdf5 disable check and lint Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * pin tables for 3.8 Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * pin 3.8 tables Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * lint Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * remove 3.11 completely from circleci and tidy up Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * add 3.11 to pyarrow pin for docs Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * undo previous commit Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * update rtd to python 3.11 Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * try 3.10 on rtd Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * fix outdated pylint to ruff Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * changes based on review Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * add mock for pyarrow and deltalake for docs Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * undo previous commit Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update setup.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update .readthedocs.yml Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update conf.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * changes based on review Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * fix comments Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * test running 3.10 parallel Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Add to release notes and update badges on docs Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Add migration tips Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> --------- Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Signed-off-by: Jason Hite * Revise FAQs and README (#2909) * Revise FAQs and README Signed-off-by: Jo Stichbury * Add back the data layers FAQ as I've no idea where else it fits Signed-off-by: Jo Stichbury * minor changes from review Signed-off-by: Jo Stichbury --------- Signed-off-by: Jo Stichbury Signed-off-by: Jason Hite * Update Generator example (#2858) * update release note Signed-off-by: Nok * update to binary mode when opening file Signed-off-by: Nok Chan * Simplify the implementation of Dataset Signed-off-by: Nok Chan * simplify the example - wip Signed-off-by: Nok Chan * Update the nodes Signed-off-by: Nok Chan * update example Signed-off-by: Nok Chan * Update example Signed-off-by: Nok * Apply suggestions from code review Co-authored-by: Jo Stichbury * Apply suggestions from code review Co-authored-by: Jo Stichbury * Apply suggestions from code review Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> Co-authored-by: Jo Stichbury * Apply suggestions from code review Co-authored-by: Jo Stichbury * add more instruction to guide user to create the custom dataset Signed-off-by: Nok * update the reference from `pipeline.py` -> `nodes.py` Signed-off-by: Nok * Collapse the nodes.py function into a expandable section Signed-off-by: Nok * Update nodes.md * Update nodes.md --------- Signed-off-by: Nok Signed-off-by: Nok Chan Co-authored-by: Jo Stichbury Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> Signed-off-by: Jason Hite * update docs to reflect change to /conf file structure (#2913) * update docs to reflect change to /conf file structure Signed-off-by: Dmitry Sorokin * update docs, folder structure diagrams Signed-off-by: Dmitry Sorokin --------- Signed-off-by: Dmitry Sorokin Signed-off-by: Jason Hite * Change CONTRIBUTING.md file based on PR #2894 (#2914) Signed-off-by: lrcouto Signed-off-by: Jason Hite * Move contribution guidelines from CONTRIBUTING.md to the Wiki (#2894) * Add drafts for new contributing.md and wiki page Signed-off-by: lrcouto * Add contribution details to wiki page draft Signed-off-by: lrcouto * Remove contributing-to-Kedro.md draft from the branch Signed-off-by: lrcouto * Fix broken link to Prefect website in deploy guide (#2885) Signed-off-by: lrcouto * Clarify the argument to kedro micropkg package (#2835) Co-authored-by: Juan Luis Cano Rodríguez Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> Signed-off-by: lrcouto * Allow registering of custom resolvers to `OmegaConfigLoader` (#2869) * Allow registering of custom resolvers to OCL Signed-off-by: Ankita Katiyar * Complete doc string Signed-off-by: Ankita Katiyar * Add test for overwritten resolvers Signed-off-by: Ankita Katiyar * Update test for overwritten resolvers Signed-off-by: Ankita Katiyar * Remove replace=True by default Signed-off-by: Ankita Katiyar * Update release notes Signed-off-by: Ankita Katiyar * Update release notes Signed-off-by: Ankita Katiyar * Add debug level log for registering new resolver Signed-off-by: Ankita Katiyar --------- Signed-off-by: Ankita Katiyar Signed-off-by: lrcouto * Some adjustments to ordering of text Signed-off-by: Jo Stichbury Signed-off-by: lrcouto * Document the use of custom resolvers with `OmegaConfigLoader` (#2896) * Update docs for custom resolvers for OCL Signed-off-by: Ankita Katiyar * Update FAQ with correct link Signed-off-by: Ankita Katiyar * Fix typo Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> * Add note for oc.env Signed-off-by: Ankita Katiyar * Suggestions from code review Signed-off-by: Ankita Katiyar --------- Signed-off-by: Ankita Katiyar Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> Signed-off-by: lrcouto * Update kedro pipeline create to use new /conf file structure (#2856) * add parameters_pipelineNane.yml template in the conf folder Signed-off-by: Dmitry Sorokin * del yml template from parameters folder Signed-off-by: Dmitry Sorokin * modify delete pipeline command to use correct path to yml files Signed-off-by: Dmitry Sorokin * modify test_pipeline move parameters foldername to the yml filename Signed-off-by: Dmitry Sorokin * collect configs to package not only from parameters folder, but from core conf folder also Signed-off-by: Dmitry Sorokin * move parameters foldername to the yml filename, remove nesting Signed-off-by: Dmitry Sorokin * move parameters foldername to the yml filename, remove nesting Signed-off-by: Dmitry Sorokin * Add changes to RELEASE.md Signed-off-by: Dmitry Sorokin * Update RELEASE.md Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> * Update RELEASE.md, change to release 0.18.13 Signed-off-by: Dmitry Sorokin * save old structure for 'pipeline delete' command for backward compatibility Signed-off-by: Dmitry Sorokin * modify test_pipeline for 'pipeline delete' backward compability Signed-off-by: Dmitry Sorokin --------- Signed-off-by: Dmitry Sorokin Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> Signed-off-by: lrcouto * Update CONTRIBUTING.md Signed-off-by: lrcouto * Fix broken link Signed-off-by: Jo Stichbury Signed-off-by: lrcouto * Update kedro catalog create to use new /conf structure (#2884) * modify test_catalog, remove nesting in the catalog folder Signed-off-by: Dmitry Sorokin * move catalog foldername to the yml filename, remove nesting Signed-off-by: Dmitry Sorokin * Update RELEASE.md Signed-off-by: Dmitry Sorokin * Modify docs, fix broken link Signed-off-by: Dmitry Sorokin --------- Signed-off-by: Dmitry Sorokin Signed-off-by: lrcouto * Add migration steps for `ConfigLoader` to `OmegaConfigLoader` (#2887) * Add migration steps for CL to OCL Signed-off-by: Merel Theisen Signed-off-by: lrcouto * Fix #2498 Adding logging issue 2498 (#2842) * Fixing typo n on line 114 file /docs/source/nodes_and_pipelines/run_a_pipeline.md from issue #2607 Signed-off-by: laizaparizotto * Removed redundant packages at lines 66, 67 and 68 on file kedro/docs/source/nodes_and_pipelines/run_a_pipeline.md, packages: from kedro.io import AbstractDataSet, DataCatalog, MemoryDataSet, from kedro.pipeline import Pipeline and from kedro.runner.runner import AbstractRunner Signed-off-by: laizaparizotto * Add logger object for logging when computing micropackage version Signed-off-by: laizaparizotto * Added new format to the logging Signed-off-by: laizaparizotto --------- Signed-off-by: laizaparizotto Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> Signed-off-by: lrcouto * Try only trigger docs build when release notes updated (#2907) Signed-off-by: Merel Theisen Signed-off-by: lrcouto * Add Python 3.11 support to Kedro (#2851) * add 3.11 Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update setup.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * limit number of python version tests Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * fix setup.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * delta-spark version Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * try original setup Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * remove 3.11 from circleci Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update setup.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update setup.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * pin matplotlib to 3.7.2 for python 3.11 Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update test_import.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * add noks test adjustments Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update setup.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update setup.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * replace eq with is for multi_catalog Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * minor fixes Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * remove matplotlib 3.11 pin Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * matplotlib pin for 3.11 Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * run original matplotlib pin with ubuntu only Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * run circleci Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * pin spark < 3.4 as its breaking Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * spark datasets revert earlier changes back to original Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * delta Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * remove all windows tests from GHA Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * skip coverage as not needed Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Checking only WIN tests (will revert) Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update unit-tests.yml Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * undo previous commit Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * add msbuild Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * add ubuntu tests back in and remove 3.11 tests from circleci add in msbuild for the other tests Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * set hdf5 disable check and lint Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * pin tables for 3.8 Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * pin 3.8 tables Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * lint Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * remove 3.11 completely from circleci and tidy up Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * add 3.11 to pyarrow pin for docs Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * undo previous commit Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * update rtd to python 3.11 Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * try 3.10 on rtd Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * fix outdated pylint to ruff Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * changes based on review Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * add mock for pyarrow and deltalake for docs Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * undo previous commit Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update setup.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update .readthedocs.yml Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update conf.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * changes based on review Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * fix comments Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * test running 3.10 parallel Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Add to release notes and update badges on docs Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Add migration tips Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> --------- Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Signed-off-by: lrcouto * Revise FAQs and README (#2909) * Revise FAQs and README Signed-off-by: Jo Stichbury * Add back the data layers FAQ as I've no idea where else it fits Signed-off-by: Jo Stichbury * minor changes from review Signed-off-by: Jo Stichbury --------- Signed-off-by: Jo Stichbury * Update Generator example (#2858) * update release note Signed-off-by: Nok * update to binary mode when opening file Signed-off-by: Nok Chan * Simplify the implementation of Dataset Signed-off-by: Nok Chan * simplify the example - wip Signed-off-by: Nok Chan * Update the nodes Signed-off-by: Nok Chan * update example Signed-off-by: Nok Chan * Update example Signed-off-by: Nok * Apply suggestions from code review Co-authored-by: Jo Stichbury * Apply suggestions from code review Co-authored-by: Jo Stichbury * Apply suggestions from code review Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> Co-authored-by: Jo Stichbury * Apply suggestions from code review Co-authored-by: Jo Stichbury * add more instruction to guide user to create the custom dataset Signed-off-by: Nok * update the reference from `pipeline.py` -> `nodes.py` Signed-off-by: Nok * Collapse the nodes.py function into a expandable section Signed-off-by: Nok * Update nodes.md * Update nodes.md --------- Signed-off-by: Nok Signed-off-by: Nok Chan Co-authored-by: Jo Stichbury Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> Signed-off-by: lrcouto * Update CONTRIBUTING.md --------- Signed-off-by: lrcouto Signed-off-by: Ankita Katiyar Signed-off-by: Jo Stichbury Signed-off-by: Dmitry Sorokin Signed-off-by: Merel Theisen Signed-off-by: laizaparizotto Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Signed-off-by: Nok Signed-off-by: Nok Chan Co-authored-by: Deepyaman Datta Co-authored-by: Jonathan Cohen Co-authored-by: Juan Luis Cano Rodríguez Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> Co-authored-by: Ankita Katiyar <110245118+ankatiyar@users.noreply.github.com> Co-authored-by: Jo Stichbury Co-authored-by: Dmitry Sorokin <40151847+DimedS@users.noreply.github.com> Co-authored-by: Laíza Milena Scheid Parizotto <41296740+laizaparizotto@users.noreply.github.com> Co-authored-by: Sajid Alam <90610031+SajidAlamQB@users.noreply.github.com> Co-authored-by: Nok Lam Chan Signed-off-by: Jason Hite * Remove redundant pages and direct users to wiki (#2917) * Remove redundant pages and direct users to wiki Signed-off-by: lrcouto * Lint docs changes Signed-off-by: Jo Stichbury * Few tweaks to index page Signed-off-by: Jo Stichbury --------- Signed-off-by: lrcouto Signed-off-by: Jo Stichbury Co-authored-by: Jo Stichbury Signed-off-by: Jason Hite * Deprecate abstract "DataSet" in favor of "Dataset" (#2746) * Deprecate abstract "DataSet" in favor of "Dataset" Signed-off-by: Deepyaman Datta * Update docs references to abstract "DataSet" class Signed-off-by: Deepyaman Datta * Fix `AbstractDataSet` reference to `AbstractDataset` Signed-off-by: Deepyaman Datta * Update docs/source/conf.py Signed-off-by: Deepyaman Datta * Change remaining `_DEPRECATED_ERROR_CLASSES` to `_DEPRECATED_CLASSES`, update type hints Signed-off-by: Deepyaman Datta * Update RELEASE.md * Reformat kedro/io/__init__.py with Black and isort Signed-off-by: Deepyaman Datta * Remove duplicate imports Signed-off-by: Deepyaman Datta * Move imports Signed-off-by: Deepyaman Datta --------- Signed-off-by: Deepyaman Datta Signed-off-by: Jason Hite * Reorganise and improve the data catalog documentation (#2888) * First drop of newly organised data catalog docs Signed-off-by: Jo Stichbury * linter Signed-off-by: Jo Stichbury * Added to-do notes Signed-off-by: Jo Stichbury * Afternoon's work in rewriting/reorganising content Signed-off-by: Jo Stichbury * More changes Signed-off-by: Jo Stichbury * Further changes Signed-off-by: Jo Stichbury * Another chunk of changes Signed-off-by: Jo Stichbury * Final changes Signed-off-by: Jo Stichbury * Revise ordering of pages Signed-off-by: Jo Stichbury * Add new CLI commands to dataset factory docs (#2935) * Add changes from #2930 Signed-off-by: Ahdra Merali * Lint Signed-off-by: Ahdra Merali * Apply suggestions from code review Co-authored-by: Jo Stichbury * Make code snippets collapsable Signed-off-by: Ahdra Merali --------- Signed-off-by: Ahdra Merali Co-authored-by: Jo Stichbury * Bunch of changes from feedback Signed-off-by: Jo Stichbury * A few more tweaks Signed-off-by: Jo Stichbury * Update h1,h2,h3 font sizes Signed-off-by: Tynan DeBold * Add code snippet for using DataCatalog with Kedro config Signed-off-by: Ankita Katiyar * Few more tweaks Signed-off-by: Jo Stichbury * Update docs/source/data/data_catalog.md * Upgrade kedro-datasets for docs Signed-off-by: Juan Luis Cano Rodríguez * Improve prose Signed-off-by: Juan Luis Cano Rodríguez Co-authored-by: Jo Stichbury --------- Signed-off-by: Jo Stichbury Signed-off-by: Ahdra Merali Signed-off-by: Tynan DeBold Signed-off-by: Ankita Katiyar Signed-off-by: Juan Luis Cano Rodríguez Co-authored-by: Ahdra Merali <90615669+AhdraMeraliQB@users.noreply.github.com> Co-authored-by: Tynan DeBold Co-authored-by: Ankita Katiyar Co-authored-by: Juan Luis Cano Rodríguez Signed-off-by: Jason Hite * Add line about Viz to PR template (#2929) Signed-off-by: Jason Hite * Add architecture graphic back to docs with revisions (#2916) * Revise FAQs and README Signed-off-by: Jo Stichbury * Add back the data layers FAQ as I've no idea where else it fits Signed-off-by: Jo Stichbury * minor changes from review Signed-off-by: Jo Stichbury * Re-add kedro arch diagram, with revised graphic Signed-off-by: Jo Stichbury * fix broken anchor Signed-off-by: Jo Stichbury * fix broken anchor Signed-off-by: Jo Stichbury * Update docs/source/extend_kedro/architecture_overview.md Co-authored-by: Nok Lam Chan * Update docs/source/extend_kedro/architecture_overview.md Co-authored-by: Nok Lam Chan * Update docs/source/extend_kedro/architecture_overview.md Co-authored-by: Nok Lam Chan * Update docs/source/extend_kedro/architecture_overview.md Co-authored-by: Nok Lam Chan * Changes to architecture page following review Signed-off-by: Jo Stichbury * Change diagram following reivew * Add links to API docs Signed-off-by: Jo Stichbury * Added in type of users * Fix linting error Signed-off-by: Jo Stichbury --------- Signed-off-by: Jo Stichbury Co-authored-by: Nok Lam Chan Co-authored-by: Yetunde Dada <43755008+yetudada@users.noreply.github.com> Signed-off-by: Jason Hite * Add kedro catalog resolve command (#2891) * Implement working draft Signed-off-by: Ahdra Merali * Remove comments Signed-off-by: Ahdra Merali * Remove raw_config Signed-off-by: Ahdra Merali * Use config loader to access catalog config Signed-off-by: Ahdra Merali * Use dict comprehension Signed-off-by: Ahdra Merali * Remove pipeline filtering Signed-off-by: Ahdra Merali * Prevent overwrite or param inclusion Signed-off-by: Ahdra Merali * Trim filepath Signed-off-by: Ahdra Merali * Appease linter Signed-off-by: Ahdra Merali * Add test for resolve Signed-off-by: Ahdra Merali * Add test to exclude params Signed-off-by: Ahdra Merali * Add test for overwrite (not working) Signed-off-by: Ahdra Merali * Fix test Signed-off-by: Ahdra Merali * Remove print Signed-off-by: Ahdra Merali * Add changes to RELEASE.md Signed-off-by: Ahdra Merali * Trim unreachable code Signed-off-by: Ahdra Merali * Make helper function private Signed-off-by: Ahdra Merali * Refactor Signed-off-by: Ahdra Merali * Add changes to the docs Signed-off-by: Ahdra Merali * Refactor 2 Signed-off-by: Ahdra Merali * Fix docs build Signed-off-by: Ahdra Merali * Add suggestions from code review (docs) Signed-off-by: Ahdra Merali * Add suggestions from code review (code) Signed-off-by: Ahdra Merali * Rename catalog_config variable to explicit_datasets Signed-off-by: Ahdra Merali * Apply suggestions from code review Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> * Fix mocking in test Signed-off-by: Ahdra Merali * Change test fixture name Signed-off-by: Ahdra Merali * Lint Signed-off-by: Ahdra Merali --------- Signed-off-by: Ahdra Merali Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> Signed-off-by: Jason Hite * Replace "DataSet" with "Dataset" in Markdown files (#2735) * LambdaDataSet->LambdaDataset in .md files Signed-off-by: Deepyaman Datta * MemoryDataSet->MemoryDataset in .md files Signed-off-by: Deepyaman Datta * PartitionedDataSet->PartitionedDataset in .md files Signed-off-by: Deepyaman Datta * IncrementalDataSet->IncrementalDataset in .md files Signed-off-by: Deepyaman Datta * CachedDataSet->CachedDataset in .md files Signed-off-by: Deepyaman Datta * DataSetError->DatasetError in .md files Signed-off-by: Deepyaman Datta * DataSetNotFoundError->DatasetNotFoundError in .md files Signed-off-by: Deepyaman Datta * Replace "DataSet" with "Dataset" in Markdown files * Update RELEASE.md * Fix remaining instance of "*DataSet*"->"*Dataset*" Signed-off-by: Deepyaman Datta * `find . -name '*.md' -print0 | xargs -0 sed -i "" "s/\([^A-Za-z]\)DataSet/\1Dataset/g"` Signed-off-by: Deepyaman Datta * Change non-class instances of Dataset to dataset * Replace any remaining instances of DataSet in docs * Fix a broken link to docs for `PartitionedDataset` --------- Signed-off-by: Deepyaman Datta Co-authored-by: Jo Stichbury Co-authored-by: Juan Luis Cano Rodríguez Signed-off-by: Jason Hite * Update h1,h2,h3 font sizes in the docs pages (#2938) * Update h1,h2,h3 font sizes Signed-off-by: Tynan DeBold * Update h2 font size Signed-off-by: Tynan DeBold --------- Signed-off-by: Tynan DeBold Co-authored-by: Jo Stichbury Co-authored-by: Juan Luis Cano Rodríguez Signed-off-by: Jason Hite * Automatically trigger `kedro-starters` release on the release of `kedro` (#2889) * Add GA for triggering starters release Signed-off-by: Ankita Katiyar * Update token name Signed-off-by: Ankita Katiyar --------- Signed-off-by: Ankita Katiyar Signed-off-by: Jason Hite * Create issues_metrics.yml (#2814) * Create issues_metrics.yml * Update issues_metrics.yml Signed-off-by: Jason Hite * Clean up setuptools and wheel requirements to align with PEP-518 (#2927) Signed-off-by: Jason Hite * Clean up `kedro pipeline create` outdated docs (#2945) * remove the README template generate by kedro pipeline create Signed-off-by: Nok * fix docstring for kedro catalog create Signed-off-by: Nok * Fix broken link Signed-off-by: Nok * fix broken link Signed-off-by: Nok --------- Signed-off-by: Nok Co-authored-by: Juan Luis Cano Rodríguez Signed-off-by: Jason Hite * Add globals feature for `OmegaConfigLoader` using a globals resolver (#2921) * Refactor load_and_merge_dir() Signed-off-by: Ankita Katiyar * Try adding globals resolver Signed-off-by: Ankita Katiyar * Minor change Signed-off-by: Ankita Katiyar * Add globals resolver Signed-off-by: Ankita Katiyar * Revert refactoring Signed-off-by: Ankita Katiyar * Add test + remove self.globals Signed-off-by: Ankita Katiyar * Allow for nested variables in globals Signed-off-by: Ankita Katiyar * Add documentation Signed-off-by: Ankita Katiyar * Typo Signed-off-by: Ankita Katiyar * Add error message + test Signed-off-by: Ankita Katiyar * Apply suggestions from code review Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> * Split test into multiple tests Signed-off-by: Ankita Katiyar * Restrict the globals config_patterns Signed-off-by: Ankita Katiyar * Release notes Signed-off-by: Ankita Katiyar * Update docs/source/configuration/advanced_configuration.md Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> * Add helpful error message for keys starting with _ Signed-off-by: Ankita Katiyar * Enable setting default value for globals resolver Signed-off-by: Ankita Katiyar * Typo Signed-off-by: Ankita Katiyar * Move test for keys starting with _ to the top Signed-off-by: Ankita Katiyar * Fix cross ref link in docs Signed-off-by: Ankita Katiyar --------- Signed-off-by: Ankita Katiyar Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> Co-authored-by: Juan Luis Cano Rodríguez Co-authored-by: Nok Lam Chan Signed-off-by: Jason Hite * Consolidate two `ruff-pre-commit` entries into one (#2881) * Clean up Ruff configuration in base pyproject.toml Signed-off-by: Deepyaman Datta * Consolidate two `ruff-pre-commit` entries into one Signed-off-by: Deepyaman Datta * Move Ruff hook to the top, because it's aggressive Signed-off-by: Deepyaman Datta * Add `kedro` as `known-first-party` for `isort` run Signed-off-by: Deepyaman Datta * Add noqa in docs/source/conf.py Signed-off-by: Deepyaman Datta * Fix ruff linter precommit config Signed-off-by: Nok --------- Signed-off-by: Deepyaman Datta Signed-off-by: Nok Co-authored-by: Nok Signed-off-by: Jason Hite * Fix typos across the documentation (#2956) * Fix typos across docs Signed-off-by: Ankita Katiyar * Capitalisation stuff Signed-off-by: Ankita Katiyar --------- Signed-off-by: Ankita Katiyar Signed-off-by: Jason Hite * Setup Vale linter as a GHA workflow (#2953) * Add vale GHA Signed-off-by: Ankita Katiyar * Update condition Signed-off-by: Ankita Katiyar * Update .github/styles/Kedro/ignore.txt Co-authored-by: Jo Stichbury * add some styles Signed-off-by: Jo Stichbury * Remove styles for now Signed-off-by: Ankita Katiyar --------- Signed-off-by: Ankita Katiyar Signed-off-by: Jo Stichbury Co-authored-by: Jo Stichbury Signed-off-by: Jason Hite * Fix README to show graphics on PyPI (#2961) * Fix for PyPI Signed-off-by: Jo Stichbury * Few more tweaks Signed-off-by: Jo Stichbury * Few more tweaks Signed-off-by: Jo Stichbury * Few more tweaks Signed-off-by: Jo Stichbury * Few more tweaks Signed-off-by: Jo Stichbury * Update README.md * Update README.md --------- Signed-off-by: Jo Stichbury Signed-off-by: Jason Hite * Add some Vale styles (#2963) * add some styles Signed-off-by: Jo Stichbury * Tweaks to styles Signed-off-by: Jo Stichbury * more tweaks Signed-off-by: Jo Stichbury * Revise some styles Signed-off-by: Jo Stichbury * Revise severity of some styles Signed-off-by: Jo Stichbury * Update .github/styles/Kedro/words.yml Co-authored-by: Deepyaman Datta --------- Signed-off-by: Jo Stichbury Co-authored-by: Deepyaman Datta Signed-off-by: Jason Hite * Minor changes to test + release notes + docs Signed-off-by: Ankita Katiyar Signed-off-by: Jason Hite * Update anyconfig requirement from ~=0.10.0 to >=0.10,<0.14 (#2876) Updates the requirements on [anyconfig](https://github.com/ssato/python-anyconfig) to permit the latest version. - [Changelog](https://github.com/ssato/python-anyconfig/blob/next/NEWS) - [Commits](https://github.com/ssato/python-anyconfig/compare/RELEASE_0.10.0...RELEASE_0.13.0) Signed-off-by: Jason Hite --- updated-dependencies: - dependency-name: anyconfig dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Juan Luis Cano Rodríguez Signed-off-by: Jason Hite * Move default template to static `pyproject.toml`, take 2 (#2853) * Remove leftover reference to micropkg setup.py Signed-off-by: Juan Luis Cano Rodríguez * Replace outdated extras requirement with kedro-datasets Signed-off-by: Juan Luis Cano Rodríguez * Remove outdated mention to environment.yml Signed-off-by: Juan Luis Cano Rodríguez * Move flake8 configuration to dedicated file Signed-off-by: Juan Luis Cano Rodríguez * Replace setup.py with pyproject.toml in default project template Signed-off-by: Juan Luis Cano Rodríguez * Adapt link to moving file in docs Signed-off-by: Juan Luis Cano Rodríguez * Revert "Replace outdated extras requirement with kedro-datasets" This reverts commit dfa8a9e8377ca60dcc3026e93087c5538791a0b5. Signed-off-by: Juan Luis Cano Rodríguez * Add release notes for pyproject.toml change Signed-off-by: Juan Luis Cano Rodríguez --------- Signed-off-by: Juan Luis Cano Rodríguez Signed-off-by: Jason Hite * Add deprecation warnings to top-level use of layer in catalog definition (#2964) * add deprecation Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * lint Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Use FutureWarning Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * changes based on review Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * add link to docs Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update RELEASE.md Co-authored-by: Jo Stichbury --------- Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Co-authored-by: Jo Stichbury Signed-off-by: Jason Hite * Update on credentials.md (#2787) * Update on credentials.md Updating example code in credentials.md to make it usable when copy/pasting. Since as of now it will generate a TypeError Signed-off-by: Jose * adding a more explicit code example, now we directly show the user that project_path is a pathlib.Path object Signed-off-by: Jose * Update docs/source/configuration/credentials.md Co-authored-by: Jo Stichbury --------- Signed-off-by: Jose Co-authored-by: Jo Stichbury Co-authored-by: Nok Lam Chan Signed-off-by: Jason Hite * Cap pluggy 1.3 release (#2981) Signed-off-by: Dmitry Sorokin Signed-off-by: Jason Hite * Make vale linter only run when PR opened or reopened (#2982) Signed-off-by: Ankita Katiyar Signed-off-by: Jason Hite * Update merge-gatekeeper.yml (#2960) Signed-off-by: Jason Hite * Configure starters to use OmegaConfigLoader (#2974) * Change starters to use OmegaConfigLoader Signed-off-by: lrcouto * Fix linter error Signed-off-by: lrcouto * Suppress import outside toplevel linting for starters template Signed-off-by: L. R. Couto * Fix linter error Signed-off-by: lrcouto * Add changes to release notes Signed-off-by: lrcouto --------- Signed-off-by: lrcouto Signed-off-by: L. R. Couto Signed-off-by: Jason Hite * Stop OmegaConfigLoader from reading config from hidden directory like `ipynb_checkpoints` (#2977) * Check plugins implement valid hooks Signed-off-by: Nok Chan * Add release note Signed-off-by: Nok Chan * Staging work - add custom functions to check hidden folder and files. Tests still failing Signed-off-by: Nok Chan * Fix test - checkpoints should use the same environment Signed-off-by: Nok Chan * Revert "Check plugins implement valid hooks" This reverts commit f10bede0401b80e3385e93458995835a2db721c9. * Update RELEASE.md Co-authored-by: Ankita Katiyar <110245118+ankatiyar@users.noreply.github.com> * fix lint Signed-off-by: Nok --------- Signed-off-by: Nok Chan Signed-off-by: Nok Co-authored-by: Ankita Katiyar <110245118+ankatiyar@users.noreply.github.com> Signed-off-by: Jason Hite * Add migration steps for `TemplatedConfigLoader` to `OmegaConfigLoader` (#2904) Signed-off-by: Merel Theisen Signed-off-by: Jason Hite * Introduce a sentinel value _NO_VALUE to improve Global resolvers to support defaults `0` or `None` (#2976) * Add None support to globals Signed-off-by: Ankita Katiyar * Add warning when default value is used Signed-off-by: Ankita Katiyar * Check keys Signed-off-by: Ankita Katiyar * Nok's suggestions Signed-off-by: Ankita Katiyar * Create the test to check the non-existing keys Signed-off-by: Nok * add more tests to catch case when global key is not a dict Signed-off-by: Nok * Fix the null test Signed-off-by: Nok * Introduce sentinel value _NO_VALUE Signed-off-by: Nok * rename test Signed-off-by: Nok * Improve error mesasge and raise InterpolationResolutionError when key does not exist and no default Signed-off-by: Nok * Fix non exist default test Signed-off-by: Nok * Fix test Signed-off-by: Nok * Use omegaconf to replace the custom resolving logic Signed-off-by: Nok * uncommented some tests Signed-off-by: Nok * Remove dead code Signed-off-by: Ankita Katiyar * Update error message Signed-off-by: Ankita Katiyar --------- Signed-off-by: Ankita Katiyar Signed-off-by: Nok Co-authored-by: Nok Signed-off-by: Jason Hite * Release 0.18.13 (#2988) Signed-off-by: Jason Hite * Fix docstrings on kedro/extras/datasets (#2995) * Fix docstrings on kedro/extras/datasets Signed-off-by: lrcouto * Fix formatting error Signed-off-by: lrcouto * Explicitly define code block Signed-off-by: lrcouto * Fix empty line under code block Signed-off-by: lrcouto * Fix formatting error Signed-off-by: lrcouto * Fix broken link Signed-off-by: lrcouto * Bump kedro-datasets version Signed-off-by: Ankita Katiyar * Update links in partitioned and incremental datasets Signed-off-by: Ankita Katiyar * Update links in partitioned and incremental datasets Signed-off-by: Ankita Katiyar * Update links in partitioned dataset Signed-off-by: Ankita Katiyar * Update links in partitioned dataset Signed-off-by: Ankita Katiyar * Add polars.GenericDataSet to .rst Signed-off-by: Ankita Katiyar --------- Signed-off-by: lrcouto Signed-off-by: Ankita Katiyar Co-authored-by: Ankita Katiyar Signed-off-by: Jason Hite * Minor docs changes on data section to create a PR and test Vale styles (#2966) * Minor changes to create a PR and test Vale styles Signed-off-by: Jo Stichbury * fix some vale warnings Signed-off-by: Jo Stichbury --------- Signed-off-by: Jo Stichbury Co-authored-by: Juan Luis Cano Rodríguez Signed-off-by: Jason Hite * PiP/pyproject.toml and Conda/Meta.yaml Sync (#2922) * Resynced Signed-off-by: rxm7706 <95496360+rxm7706@users.noreply.github.com> * restore lower bound for pluggy Signed-off-by: rxm7706 <95496360+rxm7706@users.noreply.github.com> --------- Signed-off-by: rxm7706 <95496360+rxm7706@users.noreply.github.com> Co-authored-by: Juan Luis Cano Rodríguez Signed-off-by: Jason Hite * fix typo for build (#3001) Signed-off-by: rxm7706 <95496360+rxm7706@users.noreply.github.com> Signed-off-by: Jason Hite * Add hook example to access `metadata` (#2998) * Check plugins implement valid hooks Signed-off-by: Nok Chan * add Metadtahook Signed-off-by: Nok Chan * Fix docs according to comments Signed-off-by: Nok --------- Signed-off-by: Nok Chan Signed-off-by: Nok Co-authored-by: Jo Stichbury Signed-off-by: Jason Hite * Bump release notes version Signed-off-by: Jason Hite * Expand docs for customized pipeline templates. Signed-off-by: Jason Hite * Update release notes and docs Signed-off-by: Ankita Katiyar Signed-off-by: Jason Hite * Update language linter to also run when PR converted ready for review Signed-off-by: Ankita Katiyar Signed-off-by: Jason Hite * Fix ci - Use `--resolver=backtracking` with `kedro build-reqs` in e2e tests (#3010) * Bump build version Signed-off-by: Ankita Katiyar * Remove telemetry from test default starter Signed-off-by: Ankita Katiyar * Add package_name back Signed-off-by: Ankita Katiyar * Pin build only for 3.7 Signed-off-by: Ankita Katiyar * Try upgrade pip Signed-off-by: Ankita Katiyar * Add constraint Signed-off-by: Ankita Katiyar * Update for windows Signed-off-by: Ankita Katiyar * Run build-reqs with backtracking resolver Signed-off-by: Ankita Katiyar * Run build-reqs with backtracking resolver Signed-off-by: Ankita Katiyar --------- Signed-off-by: Ankita Katiyar Signed-off-by: Jason Hite * Update docs Signed-off-by: Ankita Katiyar Signed-off-by: Jason Hite * Update style to catch some more US spellings Signed-off-by: Jo Stichbury Signed-off-by: Jason Hite * Make dataset factory resolve nested dict properly (#2993) Signed-off-by: Jason Hite * Apply suggestions from code review Co-authored-by: Jo Stichbury Signed-off-by: Jason Hite * Resolve suggestions from code review Signed-off-by: Jason Hite * Revert changes ukspelling.yml Signed-off-by: Ankita Katiyar --------- Signed-off-by: Jason Hite Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Signed-off-by: Ankita Katiyar Signed-off-by: Dmitry Sorokin Signed-off-by: Merel Theisen Signed-off-by: laizaparizotto Signed-off-by: Jo Stichbury Signed-off-by: Nok Signed-off-by: Nok Chan Signed-off-by: lrcouto Signed-off-by: Deepyaman Datta Signed-off-by: Ahdra Merali Signed-off-by: Tynan DeBold Signed-off-by: Juan Luis Cano Rodríguez Signed-off-by: dependabot[bot] Signed-off-by: Jose Signed-off-by: L. R. Couto Signed-off-by: rxm7706 <95496360+rxm7706@users.noreply.github.com> Co-authored-by: Deepyaman Datta Co-authored-by: Sajid Alam <90610031+SajidAlamQB@users.noreply.github.com> Co-authored-by: Jo Stichbury Co-authored-by: Juan Luis Cano Rodríguez Co-authored-by: Jonathan Cohen Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> Co-authored-by: Ankita Katiyar <110245118+ankatiyar@users.noreply.github.com> Co-authored-by: Dmitry Sorokin <40151847+DimedS@users.noreply.github.com> Co-authored-by: Laíza Milena Scheid Parizotto <41296740+laizaparizotto@users.noreply.github.com> Co-authored-by: Nok Lam Chan Co-authored-by: L. R. Couto <57910428+lrcouto@users.noreply.github.com> Co-authored-by: Ahdra Merali <90615669+AhdraMeraliQB@users.noreply.github.com> Co-authored-by: Tynan DeBold Co-authored-by: Ankita Katiyar Co-authored-by: Yetunde Dada <43755008+yetudada@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jose Nuñez <37588143+jmnunezd@users.noreply.github.com> Co-authored-by: rxm7706 <95496360+rxm7706@users.noreply.github.com> --- RELEASE.md | 7 ++ .../nodes_and_pipelines/modular_pipelines.md | 31 +++++++- kedro/framework/cli/pipeline.py | 26 +++++-- tests/framework/cli/pipeline/conftest.py | 50 +++++++++++++ tests/framework/cli/pipeline/test_pipeline.py | 72 +++++++++++++++++++ 5 files changed, 181 insertions(+), 5 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 6f18db1fe1..cc6032a7d1 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -10,12 +10,19 @@ # Upcoming Release 0.18.14 ## Major features and improvements +* Allowed using of custom cookiecutter templates for creating pipelines with `--template` flag for `kedro pipeline create` or via `template/pipeline` folder. + ## Bug fixes and other changes * Updated dataset factories to resolve nested catalog config properly. ## Documentation changes ## Breaking changes to the API ## Upcoming deprecations for Kedro 0.19.0 +## Community contributions +Many thanks to the following Kedroids for contributing PRs to this release: + +* [Jason Hite](https://github.com/jasonmhite) + # Release 0.18.13 diff --git a/docs/source/nodes_and_pipelines/modular_pipelines.md b/docs/source/nodes_and_pipelines/modular_pipelines.md index 5064ae7b87..09c0d5b54e 100644 --- a/docs/source/nodes_and_pipelines/modular_pipelines.md +++ b/docs/source/nodes_and_pipelines/modular_pipelines.md @@ -59,7 +59,6 @@ Running the `kedro pipeline create` command adds boilerplate folders and files f │ └── pipelines │ ├── __init__.py │ └── {{pipeline_name}} <-- This folder defines the modular pipeline - │ ├── README.md <-- Pipeline-specific documentation │ ├── __init__.py <-- So that Python treats this pipeline as a module │ ├── nodes.py <-- To declare your nodes │ └── pipeline.py <-- To structure the pipeline itself @@ -77,6 +76,36 @@ Running the `kedro pipeline create` command adds boilerplate folders and files f If you want to do the reverse and remove a modular pipeline, you can use ``kedro pipeline delete `` to do so. +### Custom templates + +If you want to generate a pipeline with a custom Cookiecutter template, you can save it in `/templates/pipeline`. +The `kedro pipeline create` command will pick up the custom template in your project as the default. You can also specify the path to your custom +Cookiecutter pipeline template with the `--template` flag like this: +```bash +kedro pipeline create --template +``` +A template folder passed to `kedro pipeline create` using the `--template` argument will take precedence over any local templates. +Kedro supports having a single pipeline template in your project. If you need to have multiple pipeline templates, consider saving them in a +separate folder and pointing to them with the `--template` flag. + +#### Creating custom templates + +It is your responsibility to create functional Cookiecutter templates for custom modular pipelines. Please ensure you understand the +basic structure of a modular pipeline. Your template should render to a valid, importable Python module containing a +`create_pipeline` function at the top level that returns a `Pipeline` object. You will also need appropriate +`config` and `tests` subdirectories that will be copied to the project `config` and `tests` directories when the pipeline is created. +The `config` and `tests` directories need to follow the same layout as in the default template and cannot +be customised, although the contents of the parameters and actual test file can be changed. File and folder names or structure +do not matter beyond that and can be customised according to your needs. You can use [the +default template that Kedro](https://github.com/kedro-org/kedro/tree/main/kedro/templates/pipeline) uses as a starting point. + +Pipeline templates are rendered using [Cookiecutter](https://cookiecutter.readthedocs.io/), and must also contain a `cookiecutter.json` +See the [`cookiecutter.json` file in the Kedro default template](https://github.com/kedro-org/kedro/tree/main/kedro/templates/pipeline/cookiecutter.json) for an example. +It is important to note that if you are embedding your custom pipeline template within a +Kedro starter template, you must tell Cookiecutter not to render this template when creating a new project from the starter. To do this, +you must add [`_copy_without_render: ["templates"]`](https://cookiecutter.readthedocs.io/en/latest/advanced/copy_without_render.html) to the `cookiecutter.json` file for the starter +and not the `cookiecutter.json` for the pipeline template. + ### Ensuring portability Modular pipelines are shareable between Kedro codebases via [micro-packaging](micro_packaging.md), but you must follow a couple of rules to ensure portability: diff --git a/kedro/framework/cli/pipeline.py b/kedro/framework/cli/pipeline.py index d3d9b2d2fd..b3ddb46b6d 100644 --- a/kedro/framework/cli/pipeline.py +++ b/kedro/framework/cli/pipeline.py @@ -90,10 +90,17 @@ def pipeline(): is_flag=True, help="Skip creation of config files for the new pipeline(s).", ) +@click.option( + "template_path", + "-t", + "--template", + type=click.Path(file_okay=False, dir_okay=True, exists=True, path_type=Path), + help="Path to cookiecutter template to use for pipeline(s). Will override any local templates.", +) @env_option(help="Environment to create pipeline configuration in. Defaults to `base`.") @click.pass_obj # this will pass the metadata as first argument def create_pipeline( - metadata: ProjectMetadata, name, skip_config, env, **kwargs + metadata: ProjectMetadata, name, template_path, skip_config, env, **kwargs ): # noqa: unused-argument """Create a new modular pipeline by providing a name.""" package_dir = metadata.source_dir / metadata.package_name @@ -107,7 +114,19 @@ def create_pipeline( f"Make sure it exists in the project configuration." ) - result_path = _create_pipeline(name, package_dir / "pipelines") + # Precedence for template_path is: command line > project templates/pipeline dir > global default + # If passed on the CLI, click will verify that the path exists so no need to check again + if template_path is None: + # No path provided on the CLI, try `PROJECT_PATH/templates/pipeline` + template_path = Path(metadata.project_path / "templates" / "pipeline") + + if not template_path.exists(): + # and if that folder doesn't exist fall back to the global default + template_path = Path(kedro.__file__).parent / "templates" / "pipeline" + + click.secho(f"Using pipeline template at: '{template_path}'") + + result_path = _create_pipeline(name, template_path, package_dir / "pipelines") _copy_pipeline_tests(name, result_path, package_dir) _copy_pipeline_configs(result_path, project_conf_path, skip_config, env=env) click.secho(f"\nPipeline '{name}' was successfully created.\n", fg="green") @@ -191,12 +210,11 @@ def _echo_deletion_warning(message: str, **paths: list[Path]): click.echo(indent(paths_str, " " * 2)) -def _create_pipeline(name: str, output_dir: Path) -> Path: +def _create_pipeline(name: str, template_path: Path, output_dir: Path) -> Path: with _filter_deprecation_warnings(): # noqa: import-outside-toplevel from cookiecutter.main import cookiecutter - template_path = Path(kedro.__file__).parent / "templates" / "pipeline" cookie_context = {"pipeline_name": name, "kedro_version": kedro.__version__} click.echo(f"Creating the pipeline '{name}': ", nl=False) diff --git a/tests/framework/cli/pipeline/conftest.py b/tests/framework/cli/pipeline/conftest.py index f934ab6939..672fee3eb6 100644 --- a/tests/framework/cli/pipeline/conftest.py +++ b/tests/framework/cli/pipeline/conftest.py @@ -1,10 +1,24 @@ +import json import shutil +from pathlib import Path import pytest from kedro.framework.project import settings +def _write_json(filepath: Path, content: dict): + filepath.parent.mkdir(parents=True, exist_ok=True) + json_str = json.dumps(content, indent=4) + filepath.write_text(json_str) + + +def _write_dummy_file(filepath: Path, content: str = ""): + filepath.parent.mkdir(parents=True, exist_ok=True) + with filepath.open("w") as f: + f.write(content) + + @pytest.fixture(autouse=True) def cleanup_micropackages(fake_repo_path, fake_package_path): packages = {p.name for p in fake_package_path.iterdir() if p.is_dir()} @@ -82,3 +96,39 @@ def cleanup_pyproject_toml(fake_repo_path): yield pyproject_toml.write_text(existing_toml) + + +@pytest.fixture() +def fake_local_template_dir(fake_repo_path): + """Set up a local template directory. This won't be functional we're just testing the actual layout works. + + Note that this is not scoped to module because we don't want to have this folder present in most of the tests, + so we will tear it down every time. + """ + template_path = fake_repo_path / Path("templates") + pipeline_template_path = template_path / Path("pipeline") + cookiecutter_template_path = ( + pipeline_template_path / "{{ cookiecutter.pipeline_name }}" + ) + + cookiecutter_template_path.mkdir(parents=True) + + # Create the absolute bare minimum files + cookiecutter_json = { + "pipeline_name": "default", + } + _write_json(pipeline_template_path / "cookiecutter.json", cookiecutter_json) + _write_dummy_file( + cookiecutter_template_path / "pipeline_{{ cookiecutter.pipeline_name }}.py", + ) + _write_dummy_file(cookiecutter_template_path / "__init__.py", "") + _write_dummy_file( + cookiecutter_template_path + / r"config/parameters/{{ cookiecutter.pipeline_name }}.yml", + ) + _write_dummy_file( + cookiecutter_template_path / r"tests/test_{{ cookiecutter.pipeline_name }}.py", + ) + yield template_path.resolve() + + shutil.rmtree(template_path) diff --git a/tests/framework/cli/pipeline/test_pipeline.py b/tests/framework/cli/pipeline/test_pipeline.py index 0414e79656..2426a352af 100644 --- a/tests/framework/cli/pipeline/test_pipeline.py +++ b/tests/framework/cli/pipeline/test_pipeline.py @@ -79,6 +79,78 @@ def test_create_pipeline( # pylint: disable=too-many-locals actual_files = {f.name for f in test_dir.iterdir()} assert actual_files == expected_files + @pytest.mark.parametrize("env", [None, "local"]) + def test_create_pipeline_template( # pylint: disable=too-many-locals + self, + fake_repo_path, + fake_project_cli, + fake_metadata, + env, + fake_package_path, + fake_local_template_dir, + ): + pipelines_dir = fake_package_path / "pipelines" + assert pipelines_dir.is_dir() + + assert not (pipelines_dir / PIPELINE_NAME).exists() + + cmd = ["pipeline", "create", PIPELINE_NAME] + cmd += ["-e", env] if env else [] + result = CliRunner().invoke(fake_project_cli, cmd, obj=fake_metadata) + + assert ( + f"Using pipeline template at: '{fake_repo_path / 'templates'}" + in result.output + ) + assert f"Creating the pipeline '{PIPELINE_NAME}': OK" in result.output + assert f"Location: '{pipelines_dir / PIPELINE_NAME}'" in result.output + assert f"Pipeline '{PIPELINE_NAME}' was successfully created." in result.output + + # Dummy pipeline rendered correctly + assert (pipelines_dir / PIPELINE_NAME / f"pipeline_{PIPELINE_NAME}.py").exists() + + assert result.exit_code == 0 + + @pytest.mark.parametrize("env", [None, "local"]) + def test_create_pipeline_template_command_line_override( # pylint: disable=too-many-locals + self, + fake_repo_path, + fake_project_cli, + fake_metadata, + env, + fake_package_path, + fake_local_template_dir, + ): + pipelines_dir = fake_package_path / "pipelines" + assert pipelines_dir.is_dir() + + assert not (pipelines_dir / PIPELINE_NAME).exists() + + # Rename the local template dir to something else so we know the command line flag is taking precedence + try: + # Can skip if already there but copytree has a dirs_exist_ok flag in >python 3.8 only + shutil.copytree(fake_local_template_dir, fake_repo_path / "local_templates") + except FileExistsError: + pass + + cmd = ["pipeline", "create", PIPELINE_NAME] + cmd += ["-t", str(fake_repo_path / "local_templates/pipeline")] + cmd += ["-e", env] if env else [] + result = CliRunner().invoke(fake_project_cli, cmd, obj=fake_metadata) + + assert ( + f"Using pipeline template at: '{fake_repo_path / 'local_templates'}" + in result.output + ) + assert f"Creating the pipeline '{PIPELINE_NAME}': OK" in result.output + assert f"Location: '{pipelines_dir / PIPELINE_NAME}'" in result.output + assert f"Pipeline '{PIPELINE_NAME}' was successfully created." in result.output + + # Dummy pipeline rendered correctly + assert (pipelines_dir / PIPELINE_NAME / f"pipeline_{PIPELINE_NAME}.py").exists() + + assert result.exit_code == 0 + @pytest.mark.parametrize("env", [None, "local"]) def test_create_pipeline_skip_config( self, fake_repo_path, fake_project_cli, fake_metadata, env