Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Tools: Minor issues/improvements to pyproject.toml #3426

Merged
merged 25 commits into from
Dec 20, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
93fded8
unique tool entires and QoL for pyproject.toml
SajidAlamQB Dec 14, 2023
39c1951
revert unique entries change
SajidAlamQB Dec 14, 2023
6abe298
Update cli.py template (#3428)
AhdraMeraliQB Dec 15, 2023
b643b23
Add logging about not using async for Sequential and Parallel runners…
merelcht Dec 15, 2023
f8c2a8e
Don't include requirements only needed for example pipeline (#3425)
SajidAlamQB Dec 18, 2023
848b4dc
Sort `requirements.txt` based on package name only (#3436)
deepyaman Dec 18, 2023
bf9cf82
Pass tools through as list
Dec 19, 2023
c96c5a0
Revert "Pass tools through as list"
Dec 19, 2023
8d16726
Remove duplicates
Dec 19, 2023
9a2d5d5
Fix for no add-ons selected
Dec 19, 2023
e587460
Merge branch 'main' into dev/tools-minor-bug-fixes
AhdraMeraliQB Dec 19, 2023
6f17b98
Merge branch 'main' into dev/tools-minor-bug-fixes
AhdraMeraliQB Dec 20, 2023
8e6f6b8
Lint
Dec 20, 2023
f31ae59
Add example as recognised key
Dec 20, 2023
36866fd
Add example as recognised key pt2
Dec 20, 2023
0021ae0
Merge branch 'main' into dev/tools-minor-bug-fixes
AhdraMeraliQB Dec 20, 2023
47ef1f2
Add example as recognised key pt3
Dec 20, 2023
9be2f4c
Merge branch 'dev/tools-minor-bug-fixes' of github.com:kedro-org/kedr…
Dec 20, 2023
6e29a8c
Add example as recognised key pt4
Dec 20, 2023
40558d4
Add example as recognised key pt5
Dec 20, 2023
4220d2a
Add example as recognised key pt6
Dec 20, 2023
3a2f1fe
Add example as recognised key pt7
Dec 20, 2023
4b78c64
Merge branch 'main' into dev/tools-minor-bug-fixes
AhdraMeraliQB Dec 20, 2023
7ccbad9
Merge branch 'main' into dev/tools-minor-bug-fixes
AhdraMeraliQB Dec 20, 2023
f0ca34f
Streamline condition
Dec 20, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions kedro/framework/cli/starters.py
Original file line number Diff line number Diff line change
Expand Up @@ -517,6 +517,8 @@ def _get_extra_context( # noqa: PLR0913
for tool in _parse_tools_input(tools) # type: ignore
]
extra_context["tools"] = str(extra_context["tools"])
else:
extra_context["tools"] = str(["None"])

extra_context["example_pipeline"] = (
_parse_yes_no_to_bool(
Expand All @@ -536,16 +538,14 @@ def _convert_tool_names_to_numbers(selected_tools: str | None) -> str | None:

Args:
selected_tools: a string containing the value for the --tools flag,
or None in case the flag wasn't used, i.e. lint,docs.
or None in case none were provided, i.e. lint,docs.

Returns:
String with the numbers corresponding to the desired tools, or
None in case the --tools flag was not used.
"""
if selected_tools is None:
if selected_tools is None or selected_tools.lower() == "none":
return None
if selected_tools.lower() == "none":
return ""
if selected_tools.lower() == "all":
return ",".join(NUMBER_TO_TOOLS_NAME.keys())

Expand All @@ -554,6 +554,10 @@ def _convert_tool_names_to_numbers(selected_tools: str | None) -> str | None:
tool_short_name = tool.strip()
if tool_short_name in TOOLS_SHORTNAME_TO_NUMBER:
tools.append(TOOLS_SHORTNAME_TO_NUMBER[tool_short_name])

# Remove duplicates if any
tools = sorted(list(set(tools)))

return ",".join(tools)


Expand Down
4 changes: 3 additions & 1 deletion kedro/framework/startup.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ class ProjectMetadata(NamedTuple):
source_dir: Path
kedro_init_version: str
tools: list
example_pipeline: str


def _version_mismatch_error(kedro_init_version) -> str:
Expand Down Expand Up @@ -103,6 +104,7 @@ def _get_project_metadata(project_path: Union[str, Path]) -> ProjectMetadata:
source_dir = Path(metadata_dict.get("source_dir", "src")).expanduser()
source_dir = (project_path / source_dir).resolve()
metadata_dict["tools"] = metadata_dict.get("tools")
metadata_dict["example_pipeline"] = metadata_dict.get("example_pipeline")

metadata_dict["source_dir"] = source_dir
metadata_dict["config_file"] = pyproject_toml
Expand All @@ -112,7 +114,7 @@ def _get_project_metadata(project_path: Union[str, Path]) -> ProjectMetadata:
try:
return ProjectMetadata(**metadata_dict)
except TypeError as exc:
expected_keys = mandatory_keys + ["source_dir", "tools"]
expected_keys = mandatory_keys + ["source_dir", "tools", "example_pipeline"]
raise RuntimeError(
f"Found unexpected keys in '{_PYPROJECT}'. Make sure "
f"it only contains the following keys: {expected_keys}."
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ namespaces = false
package_name = "{{ cookiecutter.python_package }}"
project_name = "{{ cookiecutter.project_name }}"
kedro_init_version = "{{ cookiecutter.kedro_version }}"
tools = "{{ cookiecutter.tools | default('') | string | replace('\"', '\\\"') }}"
tools = {{ cookiecutter.tools | default('') | string | replace('\"', '\\\"') }}
example_pipeline = "{{ cookiecutter.example_pipeline }}"

[tool.pytest.ini_options]
addopts = """
Expand Down
1 change: 1 addition & 0 deletions tests/framework/cli/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ def fake_metadata(fake_root_dir):
kedro_init_version=kedro_version,
source_dir=fake_root_dir / REPO_NAME / "src",
tools=None,
example_pipeline=None,
)
return metadata

Expand Down
11 changes: 10 additions & 1 deletion tests/framework/cli/test_starters.py
Original file line number Diff line number Diff line change
Expand Up @@ -1337,7 +1337,11 @@ def test_valid_tools_flag(self, fake_kedro_cli, tools, example_pipeline):
["new", "--tools", tools, "--example", example_pipeline],
input=_make_cli_prompt_input_without_tools(),
)

tools = _convert_tool_names_to_numbers(selected_tools=tools)
if not tools:
tools = ""

_assert_template_ok(result, tools=tools, example_pipeline=example_pipeline)
_assert_requirements_ok(result, tools=tools, repo_name="new-kedro-project")
assert (
Expand Down Expand Up @@ -1508,7 +1512,7 @@ def test_convert_tool_names_to_numbers_with_empty_string(self):
def test_convert_tool_names_to_numbers_with_none_string(self):
selected_tools = "none"
result = _convert_tool_names_to_numbers(selected_tools)
assert result == ""
assert result is None

def test_convert_tool_names_to_numbers_with_all_string(self):
result = _convert_tool_names_to_numbers("all")
Expand All @@ -1533,3 +1537,8 @@ def test_convert_tool_names_to_numbers_with_invalid_tools(self):
selected_tools = "invalid_tool1,invalid_tool2"
result = _convert_tool_names_to_numbers(selected_tools)
assert result == ""

def test_convert_tool_names_to_numbers_with_duplicates(self):
selected_tools = "lint,test,tests"
result = _convert_tool_names_to_numbers(selected_tools)
assert result == "1,2"
5 changes: 4 additions & 1 deletion tests/framework/test_startup.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ def test_valid_toml_file(self, mocker):
kedro_init_version=kedro_version,
project_path=self.project_path,
tools=None,
example_pipeline=None,
)
assert actual == expected

Expand Down Expand Up @@ -122,6 +123,7 @@ def test_valid_toml_file_with_project_version(self, mocker):
kedro_init_version=kedro_version,
project_path=self.project_path,
tools=None,
example_pipeline=None,
)
assert actual == expected

Expand All @@ -141,7 +143,7 @@ def test_toml_file_with_extra_keys(self, mocker):
pattern = (
"Found unexpected keys in 'pyproject.toml'. Make sure it "
"only contains the following keys: ['package_name', "
"'project_name', 'kedro_init_version', 'source_dir', 'tools']."
"'project_name', 'kedro_init_version', 'source_dir', 'tools', 'example_pipeline']."
)

with pytest.raises(RuntimeError, match=re.escape(pattern)):
Expand Down Expand Up @@ -301,6 +303,7 @@ def test_bootstrap_project(self, monkeypatch, tmp_path):
"kedro_init_version": kedro_version,
"source_dir": src_dir,
"tools": None,
"example_pipeline": None,
}
assert result == ProjectMetadata(**expected_metadata)
assert str(src_dir) in sys.path[0]
Expand Down
1 change: 1 addition & 0 deletions tests/ipython/test_ipython.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ def fake_metadata(tmp_path):
kedro_init_version=PROJECT_INIT_VERSION,
project_path=tmp_path,
tools=None,
example_pipeline=None,
)
return metadata

Expand Down
15 changes: 8 additions & 7 deletions tests/tools/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,14 @@ def fake_root_dir(tmp_path):
@pytest.fixture
def fake_metadata(fake_root_dir):
metadata = ProjectMetadata(
fake_root_dir / REPO_NAME / "pyproject.toml",
PACKAGE_NAME,
"CLI Tools Testing Project",
fake_root_dir / REPO_NAME,
kedro_version,
fake_root_dir / REPO_NAME / "src",
kedro_version,
source_dir=fake_root_dir / REPO_NAME / "src",
config_file=fake_root_dir / REPO_NAME / "pyproject.toml",
package_name=PACKAGE_NAME,
project_name="CLI Tools Testing Project",
kedro_init_version=kedro_version,
project_path=fake_root_dir / REPO_NAME,
tools=None,
example_pipeline=None,
)
return metadata

Expand Down