From ab576bc6524c01c2fff9e98044c0a090b30bb6e2 Mon Sep 17 00:00:00 2001 From: Sajid Alam <90610031+SajidAlamQB@users.noreply.github.com> Date: Tue, 10 Oct 2023 16:41:35 +0100 Subject: [PATCH] MVP for "add-ons" flow within kedro new CLI command (#2987) * Update prompts.yml Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update starters.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * add post_gen_project in cookiecutter hooks Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * add confirmation message for the options selected Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Update post_gen_project.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * changes based on review Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> * Lint Signed-off-by: Ahdra Merali * Remove documentation requirements Signed-off-by: Ahdra Merali * Remove testing requirements Signed-off-by: Ahdra Merali * Remove leftover linting requirements Signed-off-by: Ahdra Merali * Lint Signed-off-by: Ahdra Merali * Add add-on requirements when an addon is selected Signed-off-by: lrcouto * Correct file path Signed-off-by: lrcouto * Update tests with new default template files number Signed-off-by: Ahdra Merali * Update tests with add-ons argument Signed-off-by: Ahdra Merali * Make lint Signed-off-by: Ahdra Merali * Lint Signed-off-by: Ahdra Merali * Make tests use all add-ons by default Signed-off-by: Ahdra Merali * Make unit tests use no add-ons by default Signed-off-by: Ahdra Merali * Add installing project dependencies to e2e tests Signed-off-by: Ahdra Merali * Add linting requirements, organize code Signed-off-by: lrcouto * Refactor test for all add on options Signed-off-by: Ahdra Merali * Add test to check parsing add-ons Signed-off-by: Ahdra Merali * Add scaffolding for add-ons tests Signed-off-by: Ahdra Merali * Change name of test class Signed-off-by: Ahdra Merali * Correct test names Signed-off-by: Ahdra Merali * Correct tests directory Signed-off-by: Ahdra Merali * Clean up success message Signed-off-by: Ahdra Merali * Fix logging option Signed-off-by: Ahdra Merali * Update lint add-on logic Signed-off-by: Ahdra Merali * Ensure add-ons message only shows when add-ons are configurable Signed-off-by: Ahdra Merali * Add requirement checks to tests Signed-off-by: lrcouto * Refactor unit tests Signed-off-by: Ahdra Merali * Add validation to add ons in config file Signed-off-by: Ahdra Merali * Refactor add-ons flow script Signed-off-by: lrcouto * Pass through correct repo name in test Signed-off-by: Ahdra Merali * Clean up and clarify text Signed-off-by: Ahdra Merali * Wrap hook script inside main function Signed-off-by: Ahdra Merali * Revert displayed default Signed-off-by: Ahdra Merali * Add range validation Signed-off-by: Ahdra Merali * Add tests for add-on range validation Signed-off-by: Ahdra Merali * Apply suggestions from code review Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> * Update kedro/templates/project/hooks/utils.py Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> * Apply suggestions from code review Signed-off-by: Ahdra Merali * Remove traceback from add-on validation - review suggestion Signed-off-by: Ahdra Merali * Output add-on names when selected (via CLI) Signed-off-by: Ahdra Merali * Revert 47d935e and fix tests Signed-off-by: Ahdra Merali * Fix test errors Signed-off-by: Ahdra Merali * Try remove validation Signed-off-by: Ahdra Merali * Add validation back Signed-off-by: Ahdra Merali * Add config file input validation Signed-off-by: Ahdra Merali * Add /site-packages/ to coverage report omit Signed-off-by: Ahdra Merali * Lint Signed-off-by: Ahdra Merali * Remove duplicate error message Signed-off-by: Ahdra Merali * Lint Signed-off-by: Ahdra Merali * Remove suppression Signed-off-by: Ahdra Merali * Merge develop into fead/add-ons-flow Signed-off-by: Ahdra Merali * Prep for merge Signed-off-by: Ahdra Merali * Remove suppression Signed-off-by: Ahdra Merali * Apply changes from code review Signed-off-by: Ahdra Merali * Add clarification to error messages Signed-off-by: Ahdra Merali * Remove suppression Signed-off-by: Ahdra Merali * Correct files in starter template Signed-off-by: Ahdra Merali * Fix broken link Signed-off-by: Ahdra Merali * Add project teardown Signed-off-by: Ahdra Merali * Try be more direct Signed-off-by: Ahdra Merali * Try be more direct pt 2 Signed-off-by: Ahdra Merali * Try be more direct pt 3 Signed-off-by: Ahdra Merali * Try be more direct pt 4 Signed-off-by: Ahdra Merali * Fix clean-up Signed-off-by: Ahdra Merali * Fix type Signed-off-by: Ahdra Merali * Apply suggestions from code review Co-authored-by: Sajid Alam <90610031+SajidAlamQB@users.noreply.github.com> * Lint Signed-off-by: Ahdra Merali * Add docstring Signed-off-by: Ahdra Merali * Add changes to RELEASE.md Signed-off-by: Ahdra Merali --------- Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Signed-off-by: Ahdra Merali Signed-off-by: lrcouto Co-authored-by: L. R. Couto <57910428+lrcouto@users.noreply.github.com> Co-authored-by: Ahdra Merali Co-authored-by: lrcouto Co-authored-by: Ahdra Merali <90615669+AhdraMeraliQB@users.noreply.github.com> Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> --- RELEASE.md | 1 + features/steps/cli_steps.py | 1 + kedro/framework/cli/starters.py | 66 +++- kedro/templates/project/cookiecutter.json | 3 +- .../project/hooks/post_gen_project.py | 27 ++ kedro/templates/project/hooks/utils.py | 157 +++++++++ kedro/templates/project/prompts.yml | 18 + .../conf/logging.yml | 41 +++ .../pyproject.toml | 10 - .../requirements.txt | 4 - pyproject.toml | 3 +- tests/framework/cli/test_starters.py | 310 +++++++++++++++++- 12 files changed, 615 insertions(+), 26 deletions(-) create mode 100644 kedro/templates/project/hooks/post_gen_project.py create mode 100644 kedro/templates/project/hooks/utils.py create mode 100644 kedro/templates/project/{{ cookiecutter.repo_name }}/conf/logging.yml diff --git a/RELEASE.md b/RELEASE.md index 485ebd2612..8aa565658d 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -2,6 +2,7 @@ ## Major features and improvements * Dropped Python 3.7 support. +* Introduced add-ons to the `kedro new` CLI flow. ## Bug fixes and other changes diff --git a/features/steps/cli_steps.py b/features/steps/cli_steps.py index 87b796b8d5..3db8716bd5 100644 --- a/features/steps/cli_steps.py +++ b/features/steps/cli_steps.py @@ -151,6 +151,7 @@ def create_config_file(context): context.root_project_dir = context.temp_dir / context.project_name context.package_name = context.project_name.replace("-", "_") config = { + "add_ons": "all", "project_name": context.project_name, "repo_name": context.project_name, "output_dir": str(context.temp_dir), diff --git a/kedro/framework/cli/starters.py b/kedro/framework/cli/starters.py index a3a2a80541..d91e16a026 100644 --- a/kedro/framework/cli/starters.py +++ b/kedro/framework/cli/starters.py @@ -9,6 +9,7 @@ import re import shutil import stat +import sys import tempfile from collections import OrderedDict from itertools import groupby @@ -30,6 +31,7 @@ _safe_load_entry_point, command_with_verbosity, ) +from kedro.templates.project.hooks.utils import parse_add_ons_input KEDRO_PATH = Path(kedro.__file__).parent TEMPLATE_PATH = KEDRO_PATH / "templates" / "project" @@ -226,9 +228,12 @@ def new(config_path, starter_alias, checkout, directory, **kwargs): config = {} if config_path: config = _fetch_config_from_file(config_path) + _validate_config_file_inputs(config) + elif config_path: config = _fetch_config_from_file(config_path) - _validate_config_file(config, prompts_required) + _validate_config_file_against_prompts(config, prompts_required) + _validate_config_file_inputs(config) else: config = _fetch_config_from_user_prompts(prompts_required, cookiecutter_context) @@ -333,6 +338,22 @@ def _make_cookiecutter_args( return cookiecutter_args +def _get_add_ons_text(add_ons): + add_ons_dict = { + "1": "Linting", + "2": "Testing", + "3": "Custom Logging", + "4": "Documentation", + "5": "Data structure", + } + add_ons_list = parse_add_ons_input(add_ons) + add_ons_text = [add_ons_dict[add_on] for add_on in add_ons_list] + return ( + " ".join(str(add_on) + "," for add_on in add_ons_text[:-1]) + + f" and {add_ons_text[-1]}" + ) + + def _create_project(template_path: str, cookiecutter_args: dict[str, Any]): """Creates a new kedro project using cookiecutter. @@ -363,6 +384,17 @@ def _create_project(template_path: str, cookiecutter_args: dict[str, Any]): python_package = extra_context.get( "python_package", project_name.lower().replace(" ", "_").replace("-", "_") ) + add_ons = extra_context.get("add_ons") + + # Only non-starter projects have configurable add-ons + if template_path == str(TEMPLATE_PATH): + if add_ons == "none": + click.secho("\nYou have selected no add-ons") + else: + click.secho( + f"\nYou have selected the following add-ons: {_get_add_ons_text(add_ons)}" + ) + click.secho( f"\nThe project name '{project_name}' has been applied to: " f"\n- The project title in {result_path}/README.md " @@ -504,10 +536,10 @@ def __str__(self) -> str: def validate(self, user_input: str) -> None: """Validate a given prompt value against the regex validator""" if self.regexp and not re.match(self.regexp, user_input): - message = f"'{user_input}' is an invalid value for {self.title}." + message = f"'{user_input}' is an invalid value for {(self.title).lower()}." click.secho(message, fg="red", err=True) click.secho(self.error_message, fg="red", err=True) - raise ValueError(message, self.error_message) + sys.exit(1) def _get_available_tags(template_path: str) -> list: @@ -530,7 +562,9 @@ def _get_available_tags(template_path: str) -> list: return sorted(unique_tags) -def _validate_config_file(config: dict[str, str], prompts: dict[str, Any]): +def _validate_config_file_against_prompts( + config: dict[str, str], prompts: dict[str, Any] +): """Checks that the configuration file contains all needed variables. Args: @@ -553,3 +587,27 @@ def _validate_config_file(config: dict[str, str], prompts: dict[str, Any]): f"'{config['output_dir']}' is not a valid output directory. " "It must be a relative or absolute path to an existing directory." ) + + +def _validate_config_file_inputs(config: dict[str, str]): + """Checks that variables provided through the config file are of the expected format. + + Args: + config: The config as a dictionary. + + Raises: + SystemExit: If the provided variables are not properly formatted. + """ + project_name_reg_ex = r"^[\w -]{2,}$" + input_project_name = config.get("project_name", "New Kedro Project") + if not re.match(project_name_reg_ex, input_project_name): + message = f"'{input_project_name}' is an invalid value for project name. It must contain only alphanumeric symbols, spaces, underscores and hyphens and be at least 2 characters long" + click.secho(message, fg="red", err=True) + sys.exit(1) + + add_on_reg_ex = r"^(all|none|(\d(,\d)*|(\d-\d)))$" + input_add_ons = config.get("add_ons", "none") + if not re.match(add_on_reg_ex, input_add_ons): + message = f"'{input_add_ons}' is an invalid value for project add-ons. Please select valid options for add-ons using comma-separated values, ranges, or 'all/none'." + click.secho(message, fg="red", err=True) + sys.exit(1) diff --git a/kedro/templates/project/cookiecutter.json b/kedro/templates/project/cookiecutter.json index 6c697e9c6b..ea3c62ea2e 100644 --- a/kedro/templates/project/cookiecutter.json +++ b/kedro/templates/project/cookiecutter.json @@ -2,5 +2,6 @@ "project_name": "New Kedro Project", "repo_name": "{{ cookiecutter.project_name.strip().replace(' ', '-').replace('_', '-').lower() }}", "python_package": "{{ cookiecutter.project_name.strip().replace(' ', '_').replace('-', '_').lower() }}", - "kedro_version": "{{ cookiecutter.kedro_version }}" + "kedro_version": "{{ cookiecutter.kedro_version }}", + "add_ons": "none" } diff --git a/kedro/templates/project/hooks/post_gen_project.py b/kedro/templates/project/hooks/post_gen_project.py new file mode 100644 index 0000000000..30b49d0b92 --- /dev/null +++ b/kedro/templates/project/hooks/post_gen_project.py @@ -0,0 +1,27 @@ +from pathlib import Path + +from kedro.templates.project.hooks.utils import ( + parse_add_ons_input, + setup_template_add_ons, + sort_requirements, +) + +def main(): + current_dir = Path.cwd() + requirements_file_path = current_dir / "requirements.txt" + pyproject_file_path = current_dir / "pyproject.toml" + + # Get the selected add-ons from cookiecutter + selected_add_ons = "{{ cookiecutter.add_ons }}" + + # Parse the add-ons to get a list + selected_add_ons_list = parse_add_ons_input(selected_add_ons) + + # Handle template directories and requirements according to selected add-ons + setup_template_add_ons(selected_add_ons_list, requirements_file_path, pyproject_file_path) + + # Sort requirements.txt file in alphabetical order + sort_requirements(requirements_file_path) + +if __name__ == "__main__": + main() diff --git a/kedro/templates/project/hooks/utils.py b/kedro/templates/project/hooks/utils.py new file mode 100644 index 0000000000..d79e879d97 --- /dev/null +++ b/kedro/templates/project/hooks/utils.py @@ -0,0 +1,157 @@ +from pathlib import Path +import shutil +import sys +import click + +current_dir = Path.cwd() + +lint_requirements = "black~=22.12.0\nruff~=0.0.290\n" +lint_pyproject_requirements = """ +[tool.ruff] +select = [ + "F", # Pyflakes + "E", # Pycodestyle + "W", # Pycodestyle + "UP", # pyupgrade + "I", # isort + "PL", # Pylint +] +ignore = ["E501"] # Black takes care of line-too-long +""" + +test_requirements = "pytest-cov~=3.0\npytest-mock>=1.7.1, <2.0\npytest~=7.2" +test_pyproject_requirements = """ +[tool.pytest.ini_options] +addopts = \"\"\" +--cov-report term-missing \\ +--cov src/{{ cookiecutter.python_package }} -ra +\"\"\" + +[tool.coverage.report] +fail_under = 0 +show_missing = true +exclude_lines = ["pragma: no cover", "raise NotImplementedError"] +""" + +docs_pyproject_requirements = """ +docs = [ + "docutils<0.18.0", + "sphinx~=3.4.3", + "sphinx_rtd_theme==0.5.1", + "nbsphinx==0.8.1", + "sphinx-autodoc-typehints==1.11.1", + "sphinx_copybutton==0.3.1", + "ipykernel>=5.3, <7.0", + "Jinja2<3.1.0", + "myst-parser~=0.17.2", +] +""" + +def _validate_range(start, end): + if int(start) > int(end): + message = f"'{start}-{end}' is an invalid range for project add-ons.\nPlease ensure range values go from smaller to larger." + click.secho(message, fg="red", err=True) + sys.exit(1) + +def _validate_selection(add_ons): + for add_on in add_ons: + if int(add_on) < 1 or int(add_on) > 5: + message = f"'{add_on}' is not a valid selection.\nPlease select from the available add-ons: 1, 2, 3, 4, 5." + click.secho(message, fg="red", err=True) + sys.exit(1) + + +def parse_add_ons_input(add_ons_str): + """Parse the add-ons input string. + + Args: + add_ons_str: Input string from prompts.yml. + + Returns: + list: List of selected add-ons as strings. + """ + if add_ons_str == "all": + return ["1", "2", "3", "4", "5"] + if add_ons_str == "none": + return [] + + # Split by comma + add_ons_choices = add_ons_str.split(",") + selected = [] + + for choice in add_ons_choices: + if "-" in choice: + start, end = choice.split("-") + _validate_range(start, end) + selected.extend(str(i) for i in range(int(start), int(end) + 1)) + else: + selected.append(choice.strip()) + + _validate_selection(selected) + return selected + + +def setup_template_add_ons(selected_add_ons_list, requirements_file_path, pyproject_file_path): + """Removes directories and files related to unwanted addons from + a Kedro project template. Adds the necessary requirements for + the addons that were selected. + + Args: + selected_add_ons_list: a list containing numbers from 1 to 5, + representing specific add-ons. + requirements_file_path: the path to the requirements.txt file. + pyproject_file_path: the path to the pyproject.toml file + located on the the root of the template. + """ + if "1" not in selected_add_ons_list: # If Linting not selected + pass + else: + with open(requirements_file_path, 'a') as file: + file.write(lint_requirements) + with open(pyproject_file_path, 'a') as file: + file.write(lint_pyproject_requirements) + + if "2" not in selected_add_ons_list: # If Testing not selected + tests_path = current_dir / "tests" + if tests_path.exists(): + shutil.rmtree(str(tests_path)) + else: + with open(requirements_file_path, 'a') as file: + file.write(test_requirements) + with open(pyproject_file_path, 'a') as file: + file.write(test_pyproject_requirements) + + if "3" not in selected_add_ons_list: # If Logging not selected + logging_yml_path = current_dir / "conf/logging.yml" + if logging_yml_path.exists(): + logging_yml_path.unlink() + + if "4" not in selected_add_ons_list: # If Documentation not selected + docs_path = current_dir / "docs" + if docs_path.exists(): + shutil.rmtree(str(docs_path)) + else: + with open(pyproject_file_path, 'a') as file: + file.write(docs_pyproject_requirements) + + if "5" not in selected_add_ons_list: # If Data Structure not selected + data_path = current_dir / "data" + if data_path.exists(): + shutil.rmtree(str(data_path)) + + +def sort_requirements(requirements_file_path): + """Sort the requirements.txt file in alphabetical order. + + Args: + requirements_file_path: the path to the requirements.txt file. + """ + with open(requirements_file_path, 'r') as requirements: + lines = requirements.readlines() + + lines = [line.strip() for line in lines] + lines.sort() + sorted_content = '\n'.join(lines) + + with open(requirements_file_path, 'w') as requirements: + requirements.write(sorted_content) diff --git a/kedro/templates/project/prompts.yml b/kedro/templates/project/prompts.yml index 7e4bf62f66..2ef3c6c7d2 100644 --- a/kedro/templates/project/prompts.yml +++ b/kedro/templates/project/prompts.yml @@ -1,3 +1,21 @@ +add_ons: + title: "Project Add-Ons" + text: | + Here you can select which add-ons you'd like to include. By default, none are included. + To read more about these add-ons and what they do visit: kedro.org/{insert-documentation} + + Add-Ons + 1) Linting : Provides a basic linting set up with Black and ruff + 2) Testing : Provides basic testing set up with pytest + 3) Custom Logging : Provides more logging options + 4) Documentation: Provides basic documentations setup with Sphinx + 5) Data Structure: Provides a directory structure for storing data + + Which add-ons would you like to include in your project? [1-5/1,3/all/none]: + regex_validator: "^(all|none|(\\d(,\\d)*|(\\d-\\d)))$" + error_message: | + Invalid input. Please select valid options for add-ons using comma-separated values, ranges, or 'all/none'. + project_name: title: "Project Name" text: | diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/conf/logging.yml b/kedro/templates/project/{{ cookiecutter.repo_name }}/conf/logging.yml new file mode 100644 index 0000000000..c6a6fc7057 --- /dev/null +++ b/kedro/templates/project/{{ cookiecutter.repo_name }}/conf/logging.yml @@ -0,0 +1,41 @@ +version: 1 + +disable_existing_loggers: False + +formatters: + simple: + format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + +handlers: + console: + class: logging.StreamHandler + level: INFO + formatter: simple + stream: ext://sys.stdout + + info_file_handler: + class: logging.handlers.RotatingFileHandler + level: INFO + formatter: simple + filename: info.log + maxBytes: 10485760 # 10MB + backupCount: 20 + encoding: utf8 + delay: True + + rich: + class: kedro.logging.RichHandler + rich_tracebacks: True + # Advance options for customisation. + # See https://docs.kedro.org/en/stable/logging/logging.html#project-side-logging-configuration + # tracebacks_show_locals: False + +loggers: + kedro: + level: INFO + + {{ cookiecutter.python_package }}: + level: INFO + +root: + handlers: [rich, info_file_handler] diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/pyproject.toml b/kedro/templates/project/{{ cookiecutter.repo_name }}/pyproject.toml index dc3a98bbb9..8e914f4e4e 100644 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/pyproject.toml +++ b/kedro/templates/project/{{ cookiecutter.repo_name }}/pyproject.toml @@ -37,13 +37,3 @@ namespaces = false package_name = "{{ cookiecutter.python_package }}" project_name = "{{ cookiecutter.project_name }}" kedro_init_version = "{{ cookiecutter.kedro_version }}" - -[tool.pytest.ini_options] -addopts = """ ---cov-report term-missing \ ---cov src/{{ cookiecutter.python_package }} -ra""" - -[tool.coverage.report] -fail_under = 0 -show_missing = true -exclude_lines = ["pragma: no cover", "raise NotImplementedError"] diff --git a/kedro/templates/project/{{ cookiecutter.repo_name }}/requirements.txt b/kedro/templates/project/{{ cookiecutter.repo_name }}/requirements.txt index 02ae471d84..e1bf2945eb 100644 --- a/kedro/templates/project/{{ cookiecutter.repo_name }}/requirements.txt +++ b/kedro/templates/project/{{ cookiecutter.repo_name }}/requirements.txt @@ -1,4 +1,3 @@ -black~=22.0 ipython>=7.31.1, <8.0; python_version < '3.8' ipython~=8.10; python_version >= '3.8' jupyter~=1.0 @@ -6,8 +5,5 @@ jupyterlab_server>=2.11.1, <2.16.0 jupyterlab~=3.0, <3.6.0 kedro~={{ cookiecutter.kedro_version }} kedro-telemetry~=0.2.0 -pytest-cov~=3.0 -pytest-mock>=1.7.1, <2.0 -pytest~=7.2 # Pin problematic traitlets release - https://github.com/jupyter/notebook/issues/7048 traitlets<5.10.0 diff --git a/pyproject.toml b/pyproject.toml index 9b65021d1a..49e69feb61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -140,7 +140,8 @@ omit = [ "kedro/framework/hooks/specs.py", "tests/*", "kedro/io/core.py", # TODO: temp fix for removing datasets, need to resolve this before 0.19 - "kedro/runner/parallel_runner.py" + "kedro/runner/parallel_runner.py", + "*/site-packages/*" ] exclude_lines = ["pragma: no cover", "raise NotImplementedError"] diff --git a/tests/framework/cli/test_starters.py b/tests/framework/cli/test_starters.py index 7dde0b231d..26f82b96a7 100644 --- a/tests/framework/cli/test_starters.py +++ b/tests/framework/cli/test_starters.py @@ -16,8 +16,9 @@ TEMPLATE_PATH, KedroStarterSpec, ) +from kedro.templates.project.hooks.utils import parse_add_ons_input -FILES_IN_TEMPLATE = 27 +FILES_IN_TEMPLATE_WITH_NO_ADD_ONS = 14 @pytest.fixture @@ -38,25 +39,149 @@ def mock_cookiecutter(mocker): return mocker.patch("cookiecutter.main.cookiecutter") +def _clean_up_project(project_dir): + if project_dir.is_dir(): + shutil.rmtree(str(project_dir), ignore_errors=True) + + def _write_yaml(filepath: Path, config: dict): filepath.parent.mkdir(parents=True, exist_ok=True) yaml_str = yaml.dump(config) filepath.write_text(yaml_str) -def _make_cli_prompt_input(project_name="", repo_name="", python_package=""): - return "\n".join([project_name, repo_name, python_package]) +def _make_cli_prompt_input( + add_ons="none", project_name="", repo_name="", python_package="" +): + return "\n".join([add_ons, project_name, repo_name, python_package]) + + +def _get_expected_files(add_ons: str): + add_ons_template_files = { + "1": 0, + "2": 3, + "3": 1, + "4": 2, + "5": 8, + } # files added to template by each add-on + add_ons_list = parse_add_ons_input(add_ons) + + expected_files = FILES_IN_TEMPLATE_WITH_NO_ADD_ONS + + for add_on in add_ons_list: + expected_files = expected_files + add_ons_template_files[add_on] + + return expected_files + + +def _assert_requirements_ok( + result, + add_ons="none", + repo_name="new-kedro-project", + output_dir=".", +): + + assert result.exit_code == 0, result.output + assert "Change directory to the project generated in" in result.output + + root_path = (Path(output_dir) / repo_name).resolve() + requirements_file_path = root_path / "requirements.txt" + pyproject_file_path = root_path / "pyproject.toml" + + add_ons_list = parse_add_ons_input(add_ons) + + if "1" in add_ons_list: + with open(requirements_file_path) as requirements_file: + requirements = requirements_file.read() + + assert "black" in requirements + assert "ruff" in requirements + + with open(pyproject_file_path) as pyproject_file: + requirements = pyproject_file.read() + + assert ( + ( + """ +[tool.ruff] +select = [ + "F", # Pyflakes + "E", # Pycodestyle + "W", # Pycodestyle + "UP", # pyupgrade + "I", # isort + "PL", # Pylint +] +ignore = ["E501"] # Black takes care of line-too-long +""" + ) + in requirements + ) + + if "2" in add_ons_list: + with open(requirements_file_path) as requirements_file: + requirements = requirements_file.read() + + assert "pytest-cov~=3.0" in requirements + assert "pytest-mock>=1.7.1, <2.0" in requirements + assert "pytest~=7.2" in requirements + + with open(pyproject_file_path) as pyproject_file: + requirements = pyproject_file.read() + + assert ( + ( + """ +[tool.pytest.ini_options] +addopts = \"\"\" +--cov-report term-missing \\ +--cov src/{{ cookiecutter.python_package }} -ra +\"\"\" + +[tool.coverage.report] +fail_under = 0 +show_missing = true +exclude_lines = ["pragma: no cover", "raise NotImplementedError"] +""" + ) + in requirements + ) + + if "4" in add_ons_list: + with open(pyproject_file_path) as pyproject_file: + requirements = pyproject_file.read() + + assert ( + ( + """ +docs = [ + "docutils<0.18.0", + "sphinx~=3.4.3", + "sphinx_rtd_theme==0.5.1", + "nbsphinx==0.8.1", + "sphinx-autodoc-typehints==1.11.1", + "sphinx_copybutton==0.3.1", + "ipykernel>=5.3, <7.0", + "Jinja2<3.1.0", + "myst-parser~=0.17.2", +] +""" + ) + in requirements + ) # noqa: too-many-arguments def _assert_template_ok( result, + add_ons="none", project_name="New Kedro Project", repo_name="new-kedro-project", python_package="new_kedro_project", kedro_version=version, output_dir=".", ): + assert result.exit_code == 0, result.output assert "Change directory to the project generated in" in result.output @@ -65,7 +190,7 @@ def _assert_template_ok( p for p in full_path.rglob("*") if p.is_file() and p.name != ".DS_Store" ] - assert len(generated_files) == FILES_IN_TEMPLATE + assert len(generated_files) == _get_expected_files(add_ons) assert full_path.exists() assert (full_path / ".gitignore").is_file() assert project_name in (full_path / "README.md").read_text(encoding="utf-8") @@ -119,6 +244,44 @@ def test_starter_list_with_invalid_starter_plugin( assert expected in result.output +@pytest.mark.parametrize( + "input,expected", + [ + ("1", ["1"]), + ("1,2,3", ["1", "2", "3"]), + ("2-4", ["2", "3", "4"]), + ("3-3", ["3"]), + ("all", ["1", "2", "3", "4", "5"]), + ("none", []), + ], +) +def test_parse_add_ons_valid(input, expected): + result = parse_add_ons_input(input) + assert result == expected + + +@pytest.mark.parametrize( + "input", + ["5-2", "3-1"], +) +def test_parse_add_ons_invalid_range(input, capsys): + with pytest.raises(SystemExit): + parse_add_ons_input(input) + message = f"'{input}' is an invalid range for project add-ons.\nPlease ensure range values go from smaller to larger." + assert message in capsys.readouterr().err + + +@pytest.mark.parametrize( + "input,first_invalid", + [("0,3,5", "0"), ("1,3,6", "6"), ("0-4", "0"), ("3-6", "6")], +) +def test_parse_add_ons_invalid_selection(input, first_invalid, capsys): + with pytest.raises(SystemExit): + parse_add_ons_input(input) + message = f"'{first_invalid}' is not a valid selection.\nPlease select from the available add-ons: 1, 2, 3, 4, 5." + assert message in capsys.readouterr().err + + @pytest.mark.usefixtures("chdir_to_tmp") class TestNewFromUserPromptsValid: """Tests for running `kedro new` interactively.""" @@ -129,6 +292,7 @@ def test_default(self, fake_kedro_cli): fake_kedro_cli, ["new"], input=_make_cli_prompt_input() ) _assert_template_ok(result) + _clean_up_project(Path("./new-kedro-project")) def test_custom_project_name(self, fake_kedro_cli): result = CliRunner().invoke( @@ -142,6 +306,7 @@ def test_custom_project_name(self, fake_kedro_cli): repo_name="my-project", python_package="my_project", ) + _clean_up_project(Path("./my-project")) def test_custom_project_name_with_hyphen_and_underscore_and_number( self, fake_kedro_cli @@ -157,18 +322,21 @@ def test_custom_project_name_with_hyphen_and_underscore_and_number( repo_name="my-project--1", python_package="my_project__1", ) + _clean_up_project(Path("./my-project--1")) def test_no_prompts(self, fake_kedro_cli): shutil.copytree(TEMPLATE_PATH, "template") (Path("template") / "prompts.yml").unlink() result = CliRunner().invoke(fake_kedro_cli, ["new", "--starter", "template"]) _assert_template_ok(result) + _clean_up_project(Path("./new-kedro-project")) def test_empty_prompts(self, fake_kedro_cli): shutil.copytree(TEMPLATE_PATH, "template") _write_yaml(Path("template") / "prompts.yml", {}) result = CliRunner().invoke(fake_kedro_cli, ["new", "--starter", "template"]) _assert_template_ok(result) + _clean_up_project(Path("./new-kedro-project")) def test_custom_prompt_valid_input(self, fake_kedro_cli): shutil.copytree(TEMPLATE_PATH, "template") @@ -194,6 +362,7 @@ def test_custom_prompt_valid_input(self, fake_kedro_cli): repo_name="my-project", python_package="my_project", ) + _clean_up_project(Path("./my-project")) def test_custom_prompt_for_essential_variable(self, fake_kedro_cli): shutil.copytree(TEMPLATE_PATH, "template") @@ -219,6 +388,7 @@ def test_custom_prompt_for_essential_variable(self, fake_kedro_cli): repo_name="my_custom_repo", python_package="my_project", ) + _clean_up_project(Path("./my_custom_repo")) @pytest.mark.usefixtures("chdir_to_tmp") @@ -257,7 +427,7 @@ def test_invalid_project_name_special_characters(self, fake_kedro_cli): ) assert result.exit_code != 0 assert ( - "is an invalid value for Project Name.\nIt must contain only alphanumeric symbols" + "is an invalid value for project name.\nIt must contain only alphanumeric symbols" in result.output ) @@ -269,7 +439,7 @@ def test_invalid_project_name_too_short(self, fake_kedro_cli): ) assert result.exit_code != 0 assert ( - "is an invalid value for Project Name.\nIt must contain only alphanumeric symbols" + "is an invalid value for project name.\nIt must contain only alphanumeric symbols" in result.output ) @@ -302,6 +472,7 @@ class TestNewFromConfigFileValid: def test_required_keys_only(self, fake_kedro_cli): """Test project created from config.""" config = { + "add_ons": "none", "project_name": "My Project", "repo_name": "my-project", "python_package": "my_project", @@ -311,10 +482,12 @@ def test_required_keys_only(self, fake_kedro_cli): fake_kedro_cli, ["new", "-v", "--config", "config.yml"] ) _assert_template_ok(result, **config) + _clean_up_project(Path("./my-project")) def test_custom_required_keys(self, fake_kedro_cli): """Test project created from config.""" config = { + "add_ons": "none", "project_name": "Project X", "repo_name": "projectx", "python_package": "proj_x", @@ -324,10 +497,12 @@ def test_custom_required_keys(self, fake_kedro_cli): fake_kedro_cli, ["new", "-v", "--config", "config.yml"] ) _assert_template_ok(result, **config) + _clean_up_project(Path("./projectx")) def test_custom_kedro_version(self, fake_kedro_cli): """Test project created from config.""" config = { + "add_ons": "none", "project_name": "My Project", "repo_name": "my-project", "python_package": "my_project", @@ -338,10 +513,12 @@ def test_custom_kedro_version(self, fake_kedro_cli): fake_kedro_cli, ["new", "-v", "--config", "config.yml"] ) _assert_template_ok(result, **config) + _clean_up_project(Path("./my-project")) def test_custom_output_dir(self, fake_kedro_cli): """Test project created from config.""" config = { + "add_ons": "none", "project_name": "My Project", "repo_name": "my-project", "python_package": "my_project", @@ -353,10 +530,12 @@ def test_custom_output_dir(self, fake_kedro_cli): fake_kedro_cli, ["new", "-v", "--config", "config.yml"] ) _assert_template_ok(result, **config) + _clean_up_project(Path("./my-project")) def test_extra_keys_allowed(self, fake_kedro_cli): """Test project created from config.""" config = { + "add_ons": "none", "project_name": "My Project", "repo_name": "my-project", "python_package": "my_project", @@ -366,6 +545,7 @@ def test_extra_keys_allowed(self, fake_kedro_cli): fake_kedro_cli, ["new", "-v", "--config", "config.yml"] ) _assert_template_ok(result, **config) + _clean_up_project(Path("./my-project")) def test_no_prompts(self, fake_kedro_cli): config = { @@ -380,6 +560,7 @@ def test_no_prompts(self, fake_kedro_cli): fake_kedro_cli, ["new", "--starter", "template", "--config", "config.yml"] ) _assert_template_ok(result, **config) + _clean_up_project(Path("./my-project")) def test_empty_prompts(self, fake_kedro_cli): config = { @@ -394,6 +575,7 @@ def test_empty_prompts(self, fake_kedro_cli): fake_kedro_cli, ["new", "--starter", "template", "--config", "config.yml"] ) _assert_template_ok(result, **config) + _clean_up_project(Path("./my-project")) @pytest.mark.usefixtures("chdir_to_tmp") @@ -401,6 +583,7 @@ class TestNewFromConfigFileInvalid: def test_output_dir_does_not_exist(self, fake_kedro_cli): """Check the error if the output directory is invalid.""" config = { + "add_ons": "none", "project_name": "My Project", "repo_name": "my-project", "python_package": "my_project", @@ -414,6 +597,7 @@ def test_output_dir_does_not_exist(self, fake_kedro_cli): def test_config_missing_key(self, fake_kedro_cli): """Check the error if keys are missing from config file.""" config = { + "add_ons": "none", "python_package": "my_project", "repo_name": "my-project", } @@ -442,6 +626,41 @@ def test_config_bad_yaml(self, fake_kedro_cli): assert result.exit_code != 0 assert "Failed to generate project: could not load config" in result.output + def test_invalid_project_name_special_characters(self, fake_kedro_cli): + config = { + "add_ons": "none", + "project_name": "My $Project!", + "repo_name": "my-project", + "python_package": "my_project", + } + _write_yaml(Path("config.yml"), config) + result = CliRunner().invoke( + fake_kedro_cli, ["new", "-v", "--config", "config.yml"] + ) + + assert result.exit_code != 0 + assert ( + "is an invalid value for project name. It must contain only alphanumeric symbols, spaces, underscores and hyphens and be at least 2 characters long" + in result.output + ) + + def test_invalid_project_name_too_short(self, fake_kedro_cli): + config = { + "add_ons": "none", + "project_name": "P", + "repo_name": "my-project", + "python_package": "my_project", + } + _write_yaml(Path("config.yml"), config) + result = CliRunner().invoke( + fake_kedro_cli, ["new", "-v", "--config", "config.yml"] + ) + assert result.exit_code != 0 + assert ( + "is an invalid value for project name. It must contain only alphanumeric symbols, spaces, underscores and hyphens and be at least 2 characters long" + in result.output + ) + @pytest.mark.usefixtures("chdir_to_tmp") class TestNewWithStarterValid: @@ -453,6 +672,7 @@ def test_absolute_path(self, fake_kedro_cli): input=_make_cli_prompt_input(), ) _assert_template_ok(result) + _clean_up_project(Path("./new-kedro-project")) def test_relative_path(self, fake_kedro_cli): shutil.copytree(TEMPLATE_PATH, "template") @@ -462,6 +682,7 @@ def test_relative_path(self, fake_kedro_cli): input=_make_cli_prompt_input(), ) _assert_template_ok(result) + _clean_up_project(Path("./new-kedro-project")) def test_relative_path_directory(self, fake_kedro_cli): shutil.copytree(TEMPLATE_PATH, "template") @@ -471,6 +692,7 @@ def test_relative_path_directory(self, fake_kedro_cli): input=_make_cli_prompt_input(), ) _assert_template_ok(result) + _clean_up_project(Path("./new-kedro-project")) def test_alias(self, fake_kedro_cli, mock_determine_repo_dir, mock_cookiecutter): CliRunner().invoke( @@ -635,3 +857,79 @@ def test_directory_flag_with_starter_alias(self, fake_kedro_cli): ) assert result.exit_code != 0 assert "Cannot use the --directory flag with a --starter alias" in result.output + + +@pytest.mark.usefixtures("chdir_to_tmp") +class TestAddOnsFromUserPrompts: + @pytest.mark.parametrize( + "add_ons", + ["1", "2", "3", "4", "5", "none", "2,3,4", "3-5", "all"], + ) + def test_valid_add_ons(self, fake_kedro_cli, add_ons): + result = CliRunner().invoke( + fake_kedro_cli, + ["new"], + input=_make_cli_prompt_input(add_ons=add_ons), + ) + + _assert_template_ok(result, add_ons=add_ons) + _assert_requirements_ok(result, add_ons=add_ons) + _clean_up_project(Path("./new-kedro-project")) + + def test_invalid_add_ons(self, fake_kedro_cli): + result = CliRunner().invoke( + fake_kedro_cli, + ["new"], + input=_make_cli_prompt_input(add_ons="bad input"), + ) + + assert result.exit_code != 0 + assert "is an invalid value for project add-ons." in result.output + assert ( + "Invalid input. Please select valid options for add-ons using comma-separated values, ranges, or 'all/none'.\n" + in result.output + ) + + +@pytest.mark.usefixtures("chdir_to_tmp") +class TestAddOnsFromConfigFile: + @pytest.mark.parametrize( + "add_ons", + ["1", "2", "3", "4", "5", "none", "2,3,4", "3-5", "all"], + ) + def test_valid_add_ons(self, fake_kedro_cli, add_ons): + """Test project created from config.""" + config = { + "add_ons": add_ons, + "project_name": "My Project", + "repo_name": "my-project", + "python_package": "my_project", + } + _write_yaml(Path("config.yml"), config) + result = CliRunner().invoke( + fake_kedro_cli, ["new", "-v", "--config", "config.yml"] + ) + + _assert_template_ok(result, **config) + _assert_requirements_ok(result, add_ons=add_ons, repo_name="my-project") + _clean_up_project(Path("./my-project")) + + def test_invalid_add_ons(self, fake_kedro_cli): + """Test project created from config.""" + config = { + "add_ons": "bad input", + "project_name": "My Project", + "repo_name": "my-project", + "python_package": "my_project", + } + _write_yaml(Path("config.yml"), config) + result = CliRunner().invoke( + fake_kedro_cli, ["new", "-v", "--config", "config.yml"] + ) + + assert result.exit_code != 0 + assert "is an invalid value for project add-ons." in result.output + assert ( + "Please select valid options for add-ons using comma-separated values, ranges, or 'all/none'.\n" + in result.output + )