From 96ee2fef3dd9fd53e290ab3237d6d5c4ff2bcc6b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Apr 2023 06:55:17 -0700 Subject: [PATCH 001/279] Bump furo from 2023.3.23 to 2023.3.27 in /docs (#3636) Bumps [furo](https://github.com/pradyunsg/furo) from 2023.3.23 to 2023.3.27. - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2023.03.23...2023.03.27) --- updated-dependencies: - dependency-name: furo dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 9d059341b14..727a1512cec 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,4 +6,4 @@ Sphinx==6.1.3 docutils==0.19 sphinxcontrib-programoutput==0.17 sphinx_copybutton==0.5.1 -furo==2023.3.23 +furo==2023.3.27 From a552f7096a9f6e016c9bb1df1e0a77a17caeec1c Mon Sep 17 00:00:00 2001 From: Harutaka Kawamura Date: Mon, 3 Apr 2023 22:56:59 +0900 Subject: [PATCH 002/279] Fix an example for 'Improved parentheses management' in the (future of the) Black code style (#3635) --- docs/the_black_code_style/future_style.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index f5fc3644f18..bfab905b288 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -93,7 +93,6 @@ parentheses are now removed. For example: ```python my_dict = { - my_dict = { "a key in my dict": a_very_long_variable * and_a_very_long_function_call() / 100000.0, From f265ff5bcd0eafcde1e62df89fb8b62ad1439887 Mon Sep 17 00:00:00 2001 From: "Yilei \"Dolee\" Yang" Date: Thu, 13 Apr 2023 17:12:05 -0700 Subject: [PATCH 003/279] Explicitly annotate this with `Final[str]` to make it work in mypyc 1.0.0+. (#3645) --- src/blib2to3/pgen2/tokenize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blib2to3/pgen2/tokenize.py b/src/blib2to3/pgen2/tokenize.py index 257dbef4a19..a6353d154c9 100644 --- a/src/blib2to3/pgen2/tokenize.py +++ b/src/blib2to3/pgen2/tokenize.py @@ -425,7 +425,7 @@ def generate_tokens( logical line; continuation lines are included. """ lnum = parenlev = continued = 0 - numchars: Final = "0123456789" + numchars: Final[str] = "0123456789" contstr, needcont = "", 0 contline: Optional[str] = None indents = [0] From 02f81c6995db4688baedd3c63e4b9821c090f09c Mon Sep 17 00:00:00 2001 From: "Yilei \"Dolee\" Yang" Date: Fri, 14 Apr 2023 14:05:08 -0700 Subject: [PATCH 004/279] Fix two more mypyc issues with mypyc v1.2.0. (#3648) --- src/black/lines.py | 2 +- src/black/nodes.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/black/lines.py b/src/black/lines.py index bf4c12cb684..9d33bfa10b4 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -790,7 +790,7 @@ def is_line_short_enough( # noqa: C901 # store the leaves that contain parts of the MLS multiline_string_contexts: List[LN] = [] - max_level_to_update = math.inf # track the depth of the MLS + max_level_to_update: Union[int, float] = math.inf # track the depth of the MLS for i, leaf in enumerate(line.leaves): if max_level_to_update == math.inf: had_comma: Optional[int] = None diff --git a/src/black/nodes.py b/src/black/nodes.py index 4e9411b1b79..45070909df4 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -181,9 +181,9 @@ def whitespace(leaf: Leaf, *, complex_subscript: bool) -> str: # noqa: C901 `complex_subscript` signals whether the given leaf is part of a subscription which has non-trivial arguments, like arithmetic expressions or function calls. """ - NO: Final = "" - SPACE: Final = " " - DOUBLESPACE: Final = " " + NO: Final[str] = "" + SPACE: Final[str] = " " + DOUBLESPACE: Final[str] = " " t = leaf.type p = leaf.parent v = leaf.value From 3da73399552b17b70f59d23d32f2347d9a551188 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Apr 2023 07:17:11 -0700 Subject: [PATCH 005/279] Bump sphinx-copybutton from 0.5.1 to 0.5.2 in /docs (#3651) Bumps [sphinx-copybutton](https://github.com/executablebooks/sphinx-copybutton) from 0.5.1 to 0.5.2. - [Release notes](https://github.com/executablebooks/sphinx-copybutton/releases) - [Changelog](https://github.com/executablebooks/sphinx-copybutton/blob/master/CHANGELOG.md) - [Commits](https://github.com/executablebooks/sphinx-copybutton/compare/v0.5.1...v0.5.2) --- updated-dependencies: - dependency-name: sphinx-copybutton dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 727a1512cec..168f0c4ec91 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -5,5 +5,5 @@ Sphinx==6.1.3 # Older versions break Sphinx even though they're declared to be supported. docutils==0.19 sphinxcontrib-programoutput==0.17 -sphinx_copybutton==0.5.1 +sphinx_copybutton==0.5.2 furo==2023.3.27 From 4b76a548153ed5e58e61c90d0f2e4d817fc19537 Mon Sep 17 00:00:00 2001 From: James Braza Date: Wed, 19 Apr 2023 03:24:03 -0700 Subject: [PATCH 006/279] Document black-jupyter hook (#3650) Co-authored-by: Jelle Zijlstra --- docs/integrations/source_version_control.md | 24 +++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/docs/integrations/source_version_control.md b/docs/integrations/source_version_control.md index de521833609..8b8fd658e0e 100644 --- a/docs/integrations/source_version_control.md +++ b/docs/integrations/source_version_control.md @@ -14,7 +14,7 @@ repos: # supported by your project here, or alternatively use # pre-commit's default_language_version, see # https://pre-commit.com/#top_level-default_language_version - language_version: python3.9 + language_version: python3.11 ``` Feel free to switch out the `rev` value to something else, like another @@ -22,11 +22,27 @@ Feel free to switch out the `rev` value to something else, like another branches or other mutable refs since the hook [won't auto update as you may expect][pre-commit-mutable-rev]. -If you want support for Jupyter Notebooks as well, then replace `id: black` with -`id: black-jupyter`. +## Jupyter Notebooks + +There is an alternate hook `black-jupyter` that expands the targets of `black` to +include Jupyter Notebooks. To use this hook, simply replace the hook's `id: black` with +`id: black-jupyter` in the `.pre-commit-config.yaml`: + +```yaml +repos: + - repo: https://github.com/psf/black + rev: 23.3.0 + hooks: + - id: black-jupyter + # It is recommended to specify the latest version of Python + # supported by your project here, or alternatively use + # pre-commit's default_language_version, see + # https://pre-commit.com/#top_level-default_language_version + language_version: python3.11 +``` ```{note} -The `black-jupyter` hook is only available from version 21.8b0 and onwards. +The `black-jupyter` hook became available in version 21.8b0. ``` [black-tags]: https://github.com/psf/black/tags From de65741b8d49d78fa2675ef79b799cd35e92e7c1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Apr 2023 04:46:34 -0700 Subject: [PATCH 007/279] Bump pypa/cibuildwheel from 2.12.1 to 2.12.3 (#3657) Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.12.1 to 2.12.3. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.12.1...v2.12.3) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pypi_upload.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index d5797c7d230..4b59c481bb0 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -58,7 +58,7 @@ jobs: - uses: actions/checkout@v3 - name: Build wheels via cibuildwheel - uses: pypa/cibuildwheel@v2.12.1 + uses: pypa/cibuildwheel@v2.12.3 env: CIBW_ARCHS_MACOS: "${{ matrix.macos_arch }}" From e712e48e06420d9240ce95c81acfcf6f11d14c83 Mon Sep 17 00:00:00 2001 From: "Yilei \"Dolee\" Yang" Date: Fri, 28 Apr 2023 11:10:01 -0700 Subject: [PATCH 008/279] Do not wrap implicitly concatenated strings used as func args in parens (#3640) --- CHANGES.md | 3 + src/black/__init__.py | 6 +- src/black/mode.py | 6 +- src/black/parsing.py | 8 +- src/black/trans.py | 43 ++++-- tests/data/preview/cantfit.py | 12 +- tests/data/preview/long_strings.py | 54 +++----- .../data/preview/long_strings__regression.py | 22 ++- tests/test_black.py | 126 ++++++------------ 9 files changed, 113 insertions(+), 167 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 7c76bca4f6a..c7ecc396214 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,9 @@ +- Implicitly concatenated strings used as function args are no longer wrapped inside + parentheses (#3640) + ### Configuration diff --git a/src/black/__init__.py b/src/black/__init__.py index 4ebf28821c3..871e9a0d7c8 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -503,10 +503,8 @@ def main( # noqa: C901 user_level_config = str(find_user_pyproject_toml()) if config == user_level_config: out( - ( - "Using configuration from user-level config at " - f"'{user_level_config}'." - ), + "Using configuration from user-level config at " + f"'{user_level_config}'.", fg="blue", ) elif config_source in ( diff --git a/src/black/mode.py b/src/black/mode.py index 0511676ce53..3e37a588e52 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -188,10 +188,8 @@ class Mode: def __post_init__(self) -> None: if self.experimental_string_processing: warn( - ( - "`experimental string processing` has been included in `preview`" - " and deprecated. Use `preview` instead." - ), + "`experimental string processing` has been included in `preview`" + " and deprecated. Use `preview` instead.", Deprecated, ) diff --git a/src/black/parsing.py b/src/black/parsing.py index eaa3c367e54..70ed99c1549 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -29,11 +29,9 @@ except ImportError: if sys.version_info < (3, 8) and not _IS_PYPY: print( - ( - "The typed_ast package is required but not installed.\n" - "You can upgrade to Python 3.8+ or install typed_ast with\n" - "`python3 -m pip install typed-ast`." - ), + "The typed_ast package is required but not installed.\n" + "You can upgrade to Python 3.8+ or install typed_ast with\n" + "`python3 -m pip install typed-ast`.", file=sys.stderr, ) sys.exit(1) diff --git a/src/black/trans.py b/src/black/trans.py index 95695f32b14..1e28ed0656e 100644 --- a/src/black/trans.py +++ b/src/black/trans.py @@ -1186,19 +1186,33 @@ def _prefer_paren_wrap_match(LL: List[Leaf]) -> Optional[int]: if LL[0].type != token.STRING: return None - # If the string is surrounded by commas (or is the first/last child)... - prev_sibling = LL[0].prev_sibling - next_sibling = LL[0].next_sibling - if not prev_sibling and not next_sibling and parent_type(LL[0]) == syms.atom: - # If it's an atom string, we need to check the parent atom's siblings. - parent = LL[0].parent - assert parent is not None # For type checkers. - prev_sibling = parent.prev_sibling - next_sibling = parent.next_sibling - if (not prev_sibling or prev_sibling.type == token.COMMA) and ( - not next_sibling or next_sibling.type == token.COMMA + matching_nodes = [ + syms.listmaker, + syms.dictsetmaker, + syms.testlist_gexp, + ] + # If the string is an immediate child of a list/set/tuple literal... + if ( + parent_type(LL[0]) in matching_nodes + or parent_type(LL[0].parent) in matching_nodes ): - return 0 + # And the string is surrounded by commas (or is the first/last child)... + prev_sibling = LL[0].prev_sibling + next_sibling = LL[0].next_sibling + if ( + not prev_sibling + and not next_sibling + and parent_type(LL[0]) == syms.atom + ): + # If it's an atom string, we need to check the parent atom's siblings. + parent = LL[0].parent + assert parent is not None # For type checkers. + prev_sibling = parent.prev_sibling + next_sibling = parent.next_sibling + if (not prev_sibling or prev_sibling.type == token.COMMA) and ( + not next_sibling or next_sibling.type == token.COMMA + ): + return 0 return None @@ -1811,8 +1825,9 @@ class StringParenWrapper(BaseStringSplitter, CustomSplitMapMixin): * The line is an lambda expression and the value is a string. OR * The line starts with an "atom" string that prefers to be wrapped in - parens. It's preferred to be wrapped when the string is surrounded by - commas (or is the first/last child). + parens. It's preferred to be wrapped when it's is an immediate child of + a list/set/tuple literal, AND the string is surrounded by commas (or is + the first/last child). Transformations: The chosen string is wrapped in parentheses and then split at the LPAR. diff --git a/tests/data/preview/cantfit.py b/tests/data/preview/cantfit.py index cade382e30d..0849374f776 100644 --- a/tests/data/preview/cantfit.py +++ b/tests/data/preview/cantfit.py @@ -79,14 +79,10 @@ ) # long arguments normal_name = normal_function_name( - ( - "but with super long string arguments that on their own exceed the line limit" - " so there's no way it can ever fit" - ), - ( - "eggs with spam and eggs and spam with eggs with spam and eggs and spam with" - " eggs with spam and eggs and spam with eggs" - ), + "but with super long string arguments that on their own exceed the line limit so" + " there's no way it can ever fit", + "eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs" + " with spam and eggs and spam with eggs", this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it=0, ) string_variable_name = "a string that is waaaaaaaayyyyyyyy too long, even in parens, there's nothing you can do" # noqa diff --git a/tests/data/preview/long_strings.py b/tests/data/preview/long_strings.py index c68da3a8632..059148729d5 100644 --- a/tests/data/preview/long_strings.py +++ b/tests/data/preview/long_strings.py @@ -323,10 +323,8 @@ def foo(): y = "Short string" print( - ( - "This is a really long string inside of a print statement with extra arguments" - " attached at the end of it." - ), + "This is a really long string inside of a print statement with extra arguments" + " attached at the end of it.", x, y, z, @@ -501,15 +499,13 @@ def foo(): ) bad_split_func1( - ( - "But what should happen when code has already " - "been formatted but in the wrong way? Like " - "with a space at the end instead of the " - "beginning. Or what about when it is split too " - "soon? In the case of a split that is too " - "short, black will try to honer the custom " - "split." - ), + "But what should happen when code has already " + "been formatted but in the wrong way? Like " + "with a space at the end instead of the " + "beginning. Or what about when it is split too " + "soon? In the case of a split that is too " + "short, black will try to honer the custom " + "split.", xxx, yyy, zzz, @@ -612,11 +608,9 @@ def foo(): ) arg_comment_string = print( - ( # This comment gets thrown to the top. - "Long lines with inline comments which are apart of (and not the only member" - " of) an argument list should have their comments appended to the reformatted" - " string's enclosing left parentheses." - ), + "Long lines with inline comments which are apart of (and not the only member of) an" + " argument list should have their comments appended to the reformatted string's" + " enclosing left parentheses.", # This comment gets thrown to the top. "Arg #2", "Arg #3", "Arg #4", @@ -676,31 +670,23 @@ def foo(): ) func_with_bad_comma( - ( - "This is a really long string argument to a function that has a trailing comma" - " which should NOT be there." - ), + "This is a really long string argument to a function that has a trailing comma" + " which should NOT be there.", ) func_with_bad_comma( - ( # comment after comma - "This is a really long string argument to a function that has a trailing comma" - " which should NOT be there." - ), + "This is a really long string argument to a function that has a trailing comma" + " which should NOT be there.", # comment after comma ) func_with_bad_comma( - ( - "This is a really long string argument to a function that has a trailing comma" - " which should NOT be there." - ), + "This is a really long string argument to a function that has a trailing comma" + " which should NOT be there.", ) func_with_bad_comma( - ( # comment after comma - "This is a really long string argument to a function that has a trailing comma" - " which should NOT be there." - ), + "This is a really long string argument to a function that has a trailing comma" + " which should NOT be there.", # comment after comma ) func_with_bad_parens_that_wont_fit_in_one_line( diff --git a/tests/data/preview/long_strings__regression.py b/tests/data/preview/long_strings__regression.py index eead8c204a9..5f0646e6029 100644 --- a/tests/data/preview/long_strings__regression.py +++ b/tests/data/preview/long_strings__regression.py @@ -715,11 +715,9 @@ class A: def foo(): some_func_call( "xxxxxxxxxx", - ( - "xx {xxxxxxxxxxx}/xxxxxxxxxxx.xxx xxxx.xxx && xxxxxx -x " - '"xxxx xxxxxxx xxxxxx xxxx; xxxx xxxxxx_xxxxx xxxxxx xxxx; ' - "xxxx.xxxx_xxxxxx(['xxxx.xxx'], xxxx.xxxxxxx().xxxxxxxxxx)\" " - ), + "xx {xxxxxxxxxxx}/xxxxxxxxxxx.xxx xxxx.xxx && xxxxxx -x " + '"xxxx xxxxxxx xxxxxx xxxx; xxxx xxxxxx_xxxxx xxxxxx xxxx; ' + "xxxx.xxxx_xxxxxx(['xxxx.xxx'], xxxx.xxxxxxx().xxxxxxxxxx)\" ", None, ("xxxxxxxxxxx",), ), @@ -728,11 +726,9 @@ def foo(): class A: def foo(): some_func_call( - ( - "xx {xxxxxxxxxxx}/xxxxxxxxxxx.xxx xxxx.xxx && xxxxxx -x " - "xxxx, ('xxxxxxx xxxxxx xxxx, xxxx') xxxxxx_xxxxx xxxxxx xxxx; " - "xxxx.xxxx_xxxxxx(['xxxx.xxx'], xxxx.xxxxxxx().xxxxxxxxxx)\" " - ), + "xx {xxxxxxxxxxx}/xxxxxxxxxxx.xxx xxxx.xxx && xxxxxx -x " + "xxxx, ('xxxxxxx xxxxxx xxxx, xxxx') xxxxxx_xxxxx xxxxxx xxxx; " + "xxxx.xxxx_xxxxxx(['xxxx.xxx'], xxxx.xxxxxxx().xxxxxxxxxx)\" ", None, ("xxxxxxxxxxx",), ), @@ -850,10 +846,8 @@ def foo(): ) lpar_and_rpar_have_comments = func_call( # LPAR Comment - ( # Comma Comment - "Long really ridiculous type of string that shouldn't really even exist at all." - " I mean commmme onnn!!!" - ), + "Long really ridiculous type of string that shouldn't really even exist at all. I" + " mean commmme onnn!!!", # Comma Comment ) # RPAR Comment cmd_fstring = ( diff --git a/tests/test_black.py b/tests/test_black.py index e5e17777715..00de5b745e7 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -567,10 +567,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(err_lines[-1], "error: cannot format e1: boom") self.assertEqual( unstyle(str(report)), - ( - "1 file reformatted, 2 files left unchanged, 1 file failed to" - " reformat." - ), + "1 file reformatted, 2 files left unchanged, 1 file failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.done(Path("f3"), black.Changed.YES) @@ -579,10 +577,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(out_lines[-1], "reformatted f3") self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 2 files left unchanged, 1 file failed to" - " reformat." - ), + "2 files reformatted, 2 files left unchanged, 1 file failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.failed(Path("e2"), "boom") @@ -591,10 +587,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(err_lines[-1], "error: cannot format e2: boom") self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 2 files left unchanged, 2 files failed to" - " reformat." - ), + "2 files reformatted, 2 files left unchanged, 2 files failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.path_ignored(Path("wat"), "no match") @@ -603,10 +597,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(out_lines[-1], "wat ignored: no match") self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 2 files left unchanged, 2 files failed to" - " reformat." - ), + "2 files reformatted, 2 files left unchanged, 2 files failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.done(Path("f4"), black.Changed.NO) @@ -615,28 +607,22 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(out_lines[-1], "f4 already well formatted, good job.") self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 3 files left unchanged, 2 files failed to" - " reformat." - ), + "2 files reformatted, 3 files left unchanged, 2 files failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.check = True self.assertEqual( unstyle(str(report)), - ( - "2 files would be reformatted, 3 files would be left unchanged, 2" - " files would fail to reformat." - ), + "2 files would be reformatted, 3 files would be left unchanged, 2" + " files would fail to reformat.", ) report.check = False report.diff = True self.assertEqual( unstyle(str(report)), - ( - "2 files would be reformatted, 3 files would be left unchanged, 2" - " files would fail to reformat." - ), + "2 files would be reformatted, 3 files would be left unchanged, 2" + " files would fail to reformat.", ) def test_report_quiet(self) -> None: @@ -678,10 +664,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(err_lines[-1], "error: cannot format e1: boom") self.assertEqual( unstyle(str(report)), - ( - "1 file reformatted, 2 files left unchanged, 1 file failed to" - " reformat." - ), + "1 file reformatted, 2 files left unchanged, 1 file failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.done(Path("f3"), black.Changed.YES) @@ -689,10 +673,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(len(err_lines), 1) self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 2 files left unchanged, 1 file failed to" - " reformat." - ), + "2 files reformatted, 2 files left unchanged, 1 file failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.failed(Path("e2"), "boom") @@ -701,10 +683,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(err_lines[-1], "error: cannot format e2: boom") self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 2 files left unchanged, 2 files failed to" - " reformat." - ), + "2 files reformatted, 2 files left unchanged, 2 files failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.path_ignored(Path("wat"), "no match") @@ -712,10 +692,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(len(err_lines), 2) self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 2 files left unchanged, 2 files failed to" - " reformat." - ), + "2 files reformatted, 2 files left unchanged, 2 files failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.done(Path("f4"), black.Changed.NO) @@ -723,28 +701,22 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(len(err_lines), 2) self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 3 files left unchanged, 2 files failed to" - " reformat." - ), + "2 files reformatted, 3 files left unchanged, 2 files failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.check = True self.assertEqual( unstyle(str(report)), - ( - "2 files would be reformatted, 3 files would be left unchanged, 2" - " files would fail to reformat." - ), + "2 files would be reformatted, 3 files would be left unchanged, 2" + " files would fail to reformat.", ) report.check = False report.diff = True self.assertEqual( unstyle(str(report)), - ( - "2 files would be reformatted, 3 files would be left unchanged, 2" - " files would fail to reformat." - ), + "2 files would be reformatted, 3 files would be left unchanged, 2" + " files would fail to reformat.", ) def test_report_normal(self) -> None: @@ -788,10 +760,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(err_lines[-1], "error: cannot format e1: boom") self.assertEqual( unstyle(str(report)), - ( - "1 file reformatted, 2 files left unchanged, 1 file failed to" - " reformat." - ), + "1 file reformatted, 2 files left unchanged, 1 file failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.done(Path("f3"), black.Changed.YES) @@ -800,10 +770,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(out_lines[-1], "reformatted f3") self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 2 files left unchanged, 1 file failed to" - " reformat." - ), + "2 files reformatted, 2 files left unchanged, 1 file failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.failed(Path("e2"), "boom") @@ -812,10 +780,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(err_lines[-1], "error: cannot format e2: boom") self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 2 files left unchanged, 2 files failed to" - " reformat." - ), + "2 files reformatted, 2 files left unchanged, 2 files failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.path_ignored(Path("wat"), "no match") @@ -823,10 +789,8 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(len(err_lines), 2) self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 2 files left unchanged, 2 files failed to" - " reformat." - ), + "2 files reformatted, 2 files left unchanged, 2 files failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.done(Path("f4"), black.Changed.NO) @@ -834,28 +798,22 @@ def err(msg: str, **kwargs: Any) -> None: self.assertEqual(len(err_lines), 2) self.assertEqual( unstyle(str(report)), - ( - "2 files reformatted, 3 files left unchanged, 2 files failed to" - " reformat." - ), + "2 files reformatted, 3 files left unchanged, 2 files failed to" + " reformat.", ) self.assertEqual(report.return_code, 123) report.check = True self.assertEqual( unstyle(str(report)), - ( - "2 files would be reformatted, 3 files would be left unchanged, 2" - " files would fail to reformat." - ), + "2 files would be reformatted, 3 files would be left unchanged, 2" + " files would fail to reformat.", ) report.check = False report.diff = True self.assertEqual( unstyle(str(report)), - ( - "2 files would be reformatted, 3 files would be left unchanged, 2" - " files would fail to reformat." - ), + "2 files would be reformatted, 3 files would be left unchanged, 2" + " files would fail to reformat.", ) def test_lib2to3_parse(self) -> None: From a07871b9cd1b5a1469271be6aaac5766f5a0e0fc Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 3 May 2023 08:43:20 -0700 Subject: [PATCH 009/279] Fix new mypy error in blib2to3 (#3674) See python/mypy#15174 --- src/blib2to3/pgen2/tokenize.py | 11 ++++++----- tests/data/simple_cases/fstring.py | 4 ++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/blib2to3/pgen2/tokenize.py b/src/blib2to3/pgen2/tokenize.py index a6353d154c9..82ac5130bad 100644 --- a/src/blib2to3/pgen2/tokenize.py +++ b/src/blib2to3/pgen2/tokenize.py @@ -163,7 +163,6 @@ def _combinations(*l): '"""': double3prog, **{f"{prefix}'''": single3prog for prefix in _strprefixes}, **{f'{prefix}"""': double3prog for prefix in _strprefixes}, - **{prefix: None for prefix in _strprefixes}, } triple_quoted: Final = ( @@ -599,11 +598,13 @@ def generate_tokens( ): if token[-1] == "\n": # continued string strstart = (lnum, start) - endprog = ( - endprogs[initial] - or endprogs[token[1]] - or endprogs[token[2]] + maybe_endprog = ( + endprogs.get(initial) + or endprogs.get(token[1]) + or endprogs.get(token[2]) ) + assert maybe_endprog is not None, f"endprog not found for {token}" + endprog = maybe_endprog contstr, needcont = line[start:], 1 contline = line break diff --git a/tests/data/simple_cases/fstring.py b/tests/data/simple_cases/fstring.py index 4b33231c01c..60560309376 100644 --- a/tests/data/simple_cases/fstring.py +++ b/tests/data/simple_cases/fstring.py @@ -7,6 +7,8 @@ f"\"{f'{nested} inner'}\" outer" f"space between opening braces: { {a for a in (1, 2, 3)}}" f'Hello \'{tricky + "example"}\'' +f"Tried directories {str(rootdirs)} \ +but none started with prefix {parentdir_prefix}" # output @@ -19,3 +21,5 @@ f"\"{f'{nested} inner'}\" outer" f"space between opening braces: { {a for a in (1, 2, 3)}}" f'Hello \'{tricky + "example"}\'' +f"Tried directories {str(rootdirs)} \ +but none started with prefix {parentdir_prefix}" From eb32729ab562c010d9f644a3bbb2305196b2deb7 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 3 May 2023 10:26:57 -0700 Subject: [PATCH 010/279] blib2to3: add a few annotations (#3675) --- src/blib2to3/pgen2/tokenize.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/blib2to3/pgen2/tokenize.py b/src/blib2to3/pgen2/tokenize.py index 82ac5130bad..2d0cc4324ce 100644 --- a/src/blib2to3/pgen2/tokenize.py +++ b/src/blib2to3/pgen2/tokenize.py @@ -34,6 +34,7 @@ Iterator, List, Optional, + Set, Text, Tuple, Pattern, @@ -66,19 +67,19 @@ del token -def group(*choices): +def group(*choices: str) -> str: return "(" + "|".join(choices) + ")" -def any(*choices): +def any(*choices: str) -> str: return group(*choices) + "*" -def maybe(*choices): +def maybe(*choices: str) -> str: return group(*choices) + "?" -def _combinations(*l): +def _combinations(*l: str) -> Set[str]: return set(x + y for x in l for y in l + ("",) if x.casefold() != y.casefold()) @@ -187,15 +188,19 @@ class StopTokenizing(Exception): pass -def printtoken(type, token, xxx_todo_changeme, xxx_todo_changeme1, line): # for testing - (srow, scol) = xxx_todo_changeme - (erow, ecol) = xxx_todo_changeme1 +Coord = Tuple[int, int] + + +def printtoken( + type: int, token: Text, srow_col: Coord, erow_col: Coord, line: Text +) -> None: # for testing + (srow, scol) = srow_col + (erow, ecol) = erow_col print( "%d,%d-%d,%d:\t%s\t%s" % (srow, scol, erow, ecol, tok_name[type], repr(token)) ) -Coord = Tuple[int, int] TokenEater = Callable[[int, Text, Coord, Coord, Text], None] @@ -219,7 +224,7 @@ def tokenize(readline: Callable[[], Text], tokeneater: TokenEater = printtoken) # backwards compatible interface -def tokenize_loop(readline, tokeneater): +def tokenize_loop(readline: Callable[[], Text], tokeneater: TokenEater) -> None: for token_info in generate_tokens(readline): tokeneater(*token_info) @@ -229,7 +234,6 @@ def tokenize_loop(readline, tokeneater): class Untokenizer: - tokens: List[Text] prev_row: int prev_col: int @@ -603,7 +607,9 @@ def generate_tokens( or endprogs.get(token[1]) or endprogs.get(token[2]) ) - assert maybe_endprog is not None, f"endprog not found for {token}" + assert ( + maybe_endprog is not None + ), f"endprog not found for {token}" endprog = maybe_endprog contstr, needcont = line[start:], 1 contline = line @@ -632,7 +638,6 @@ def generate_tokens( if token in ("def", "for"): if stashed and stashed[0] == NAME and stashed[1] == "async": - if token == "def": async_def = True async_def_indent = indents[-1] From 64887aab032c0fd64f9238cdab6684f2fc0c7f33 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 May 2023 06:36:24 -0700 Subject: [PATCH 011/279] Bump peter-evans/create-or-update-comment from 2.1.1 to 3.0.1 (#3683) Bumps [peter-evans/create-or-update-comment](https://github.com/peter-evans/create-or-update-comment) from 2.1.1 to 3.0.1. - [Release notes](https://github.com/peter-evans/create-or-update-comment/releases) - [Commits](https://github.com/peter-evans/create-or-update-comment/compare/67dcc547d311b736a8e6c5c236542148a47adc3d...ca08ebd5dc95aa0cd97021e9708fcd6b87138c9b) --- updated-dependencies: - dependency-name: peter-evans/create-or-update-comment dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/diff_shades_comment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/diff_shades_comment.yml b/.github/workflows/diff_shades_comment.yml index bb81ca4f0d6..70ab7ff4f7a 100644 --- a/.github/workflows/diff_shades_comment.yml +++ b/.github/workflows/diff_shades_comment.yml @@ -41,7 +41,7 @@ jobs: - name: Create or update PR comment if: steps.metadata.outputs.needs-comment == 'true' - uses: peter-evans/create-or-update-comment@67dcc547d311b736a8e6c5c236542148a47adc3d + uses: peter-evans/create-or-update-comment@ca08ebd5dc95aa0cd97021e9708fcd6b87138c9b with: comment-id: ${{ steps.find-comment.outputs.comment-id }} issue-number: ${{ steps.metadata.outputs.pr-number }} From c97b9c55b488a478afe171537c0e4c0f10631ca1 Mon Sep 17 00:00:00 2001 From: Matthieu Simon Date: Mon, 15 May 2023 23:35:39 +0200 Subject: [PATCH 012/279] [github action] display black result in job summary (#3688) * send output to $GITHUB_STEP_SUMMARY * update CHANGES.md * update CHANGES.md with PR number * implement PR feedback * fix pre-commit issues (prettier/trailing whitespace) --- CHANGES.md | 2 ++ action.yml | 7 ++++--- action/main.py | 20 +++++++++++++++----- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c7ecc396214..f9bec185ff5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -45,6 +45,8 @@ +- Update GitHub Action to display black output in the job summary (#3688) + ### Documentation +- `.pytest_cache`, `.ruff_cache` and `.vscode` are now excluded by default (#3691) + ### Packaging diff --git a/src/black/const.py b/src/black/const.py index 0e13f31517d..ee466679c70 100644 --- a/src/black/const.py +++ b/src/black/const.py @@ -1,4 +1,4 @@ DEFAULT_LINE_LENGTH = 88 -DEFAULT_EXCLUDES = r"/(\.direnv|\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|venv|\.svn|\.ipynb_checkpoints|_build|buck-out|build|dist|__pypackages__)/" # noqa: B950 +DEFAULT_EXCLUDES = r"/(\.direnv|\.eggs|\.git|\.hg|\.ipynb_checkpoints|\.mypy_cache|\.nox|\.pytest_cache|\.ruff_cache|\.tox|\.svn|\.venv|\.vscode|__pypackages__|_build|buck-out|build|dist|venv)/" # noqa: B950 DEFAULT_INCLUDES = r"(\.pyi?|\.ipynb)$" STDIN_PLACEHOLDER = "__BLACK_STDIN_FILENAME__" From 2fd9d8b339e1e2e1b93956c6d68b2b358b3fc29d Mon Sep 17 00:00:00 2001 From: Jonathan Berthias Date: Fri, 19 May 2023 01:57:17 +0200 Subject: [PATCH 015/279] Remove blank lines before class docstring (#3692) --- CHANGES.md | 1 + src/black/lines.py | 2 + src/black/mode.py | 1 + .../preview/no_blank_line_before_docstring.py | 58 +++++++++++++++++++ 4 files changed, 62 insertions(+) create mode 100644 tests/data/preview/no_blank_line_before_docstring.py diff --git a/CHANGES.md b/CHANGES.md index a1bc7d59020..6a9923f8d8d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,6 +16,7 @@ - Implicitly concatenated strings used as function args are no longer wrapped inside parentheses (#3640) +- Remove blank lines between a class definition and its docstring (#3692) ### Configuration diff --git a/src/black/lines.py b/src/black/lines.py index 9d33bfa10b4..daf0444d24e 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -634,6 +634,8 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: and self.previous_line.is_class and current_line.is_triple_quoted_string ): + if Preview.no_blank_line_before_class_docstring in current_line.mode: + return 0, 1 return before, 1 if self.previous_line and self.previous_line.opens_block: diff --git a/src/black/mode.py b/src/black/mode.py index 3e37a588e52..a5841edb30a 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -158,6 +158,7 @@ class Preview(Enum): hex_codes_in_unicode_sequences = auto() improved_async_statements_handling = auto() multiline_string_handling = auto() + no_blank_line_before_class_docstring = auto() prefer_splitting_right_hand_side_of_assignments = auto() # NOTE: string_processing requires wrap_long_dict_values_in_parens # for https://github.com/psf/black/issues/3117 to be fixed. diff --git a/tests/data/preview/no_blank_line_before_docstring.py b/tests/data/preview/no_blank_line_before_docstring.py new file mode 100644 index 00000000000..a37362de100 --- /dev/null +++ b/tests/data/preview/no_blank_line_before_docstring.py @@ -0,0 +1,58 @@ +def line_before_docstring(): + + """Please move me up""" + + +class LineBeforeDocstring: + + """Please move me up""" + + +class EvenIfThereIsAMethodAfter: + + """I'm the docstring""" + def method(self): + pass + + +class TwoLinesBeforeDocstring: + + + """I want to be treated the same as if I were closer""" + + +class MultilineDocstringsAsWell: + + """I'm so far + + and on so many lines... + """ + + +# output + + +def line_before_docstring(): + """Please move me up""" + + +class LineBeforeDocstring: + """Please move me up""" + + +class EvenIfThereIsAMethodAfter: + """I'm the docstring""" + + def method(self): + pass + + +class TwoLinesBeforeDocstring: + """I want to be treated the same as if I were closer""" + + +class MultilineDocstringsAsWell: + """I'm so far + + and on so many lines... + """ From eedfc3832290b3a32825b3c0f2dfa3f3d7ee9d1c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 19 May 2023 13:00:29 -0400 Subject: [PATCH 016/279] Avoid EncodingWarning in blib2to3 (#3696) --- src/blib2to3/pgen2/pgen.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/blib2to3/pgen2/pgen.py b/src/blib2to3/pgen2/pgen.py index 631682a77c9..b5ebc7b3e42 100644 --- a/src/blib2to3/pgen2/pgen.py +++ b/src/blib2to3/pgen2/pgen.py @@ -30,7 +30,6 @@ class PgenGrammar(grammar.Grammar): class ParserGenerator(object): - filename: Path stream: IO[Text] generator: Iterator[GoodTokenInfo] @@ -39,7 +38,7 @@ class ParserGenerator(object): def __init__(self, filename: Path, stream: Optional[IO[Text]] = None) -> None: close_stream = None if stream is None: - stream = open(filename) + stream = open(filename, encoding="utf-8") close_stream = stream.close self.filename = filename self.stream = stream From cd02c2809b193e17aa7c43f8dd0fae4695898184 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 May 2023 11:47:58 -0400 Subject: [PATCH 017/279] Bump furo from 2023.3.27 to 2023.5.20 in /docs (#3698) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 168f0c4ec91..7b26d089b01 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,4 +6,4 @@ Sphinx==6.1.3 docutils==0.19 sphinxcontrib-programoutput==0.17 sphinx_copybutton==0.5.2 -furo==2023.3.27 +furo==2023.5.20 From c99417ffe8aa51015f08a96220072fa0dbcce51d Mon Sep 17 00:00:00 2001 From: Deepyaman Datta Date: Wed, 24 May 2023 22:52:59 -0400 Subject: [PATCH 018/279] Change example from `%%writeline` to `%%writefile` (#3673) --- docs/faq.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index a6a422c2fec..8941ca3fe4d 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -57,8 +57,8 @@ _Black_ is timid about formatting Jupyter Notebooks. Cells containing any of the following will not be formatted: - automagics (e.g. `pip install black`) -- non-Python cell magics (e.g. `%%writeline`). These can be added with the flag - `--python-cell-magics`, e.g. `black --python-cell-magics writeline hello.ipynb`. +- non-Python cell magics (e.g. `%%writefile`). These can be added with the flag + `--python-cell-magics`, e.g. `black --python-cell-magics writefile hello.ipynb`. - multiline magics, e.g.: ```python From f95b43d6fa9883a87574f1d69d0a433422c19377 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Thu, 25 May 2023 04:53:27 +0200 Subject: [PATCH 019/279] docs: update note on GitHub .git-blame-ignore-revs support (#3655) --- docs/guides/introducing_black_to_your_project.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/guides/introducing_black_to_your_project.md b/docs/guides/introducing_black_to_your_project.md index 9ae40a1928e..53bb0d9fcd6 100644 --- a/docs/guides/introducing_black_to_your_project.md +++ b/docs/guides/introducing_black_to_your_project.md @@ -46,7 +46,5 @@ $ git config blame.ignoreRevsFile .git-blame-ignore-revs **The one caveat is that some online Git-repositories like GitLab do not yet support ignoring revisions using their native blame UI.** So blame information will be cluttered with a reformatting commit on those platforms. (If you'd like this feature, there's an -open issue for [GitLab](https://gitlab.com/gitlab-org/gitlab/-/issues/31423)). This is -however supported by -[GitHub](https://docs.github.com/en/repositories/working-with-files/using-files/viewing-a-file#ignore-commits-in-the-blame-view), -currently in beta. +open issue for [GitLab](https://gitlab.com/gitlab-org/gitlab/-/issues/31423)). +[GitHub supports `.git-blame-ignore-revs`](https://docs.github.com/en/repositories/working-with-files/using-files/viewing-a-file#ignore-commits-in-the-blame-view) by default in blame views however. From 3decbd6db9f120e8d7c8fa86b5b2f64f7861da0c Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 24 May 2023 19:55:12 -0700 Subject: [PATCH 020/279] Document each configuration option in more detail (#2839) --- docs/the_black_code_style/current_style.md | 8 + docs/the_black_code_style/future_style.md | 2 + docs/usage_and_configuration/the_basics.md | 270 ++++++++++++++++----- scripts/check_version_in_basics_example.py | 17 +- 4 files changed, 230 insertions(+), 67 deletions(-) diff --git a/docs/the_black_code_style/current_style.md b/docs/the_black_code_style/current_style.md index 83f8785cc55..e2625f9e16e 100644 --- a/docs/the_black_code_style/current_style.md +++ b/docs/the_black_code_style/current_style.md @@ -140,6 +140,8 @@ If you're reaching for backslashes, that's a clear signal that you can do better slightly refactor your code. I hope some of the examples above show you that there are many ways in which you can do it. +(labels/line-length)= + ### Line length You probably noticed the peculiar default line length. _Black_ defaults to 88 characters @@ -273,6 +275,8 @@ A pre-existing trailing comma informs _Black_ to always explode contents of the bracket pair into one item per line. Read more about this in the [Pragmatism](#pragmatism) section below. +(labels/strings)= + ### Strings _Black_ prefers double quotes (`"` and `"""`) over single quotes (`'` and `'''`). It @@ -457,6 +461,8 @@ there were not many users anyway. Not many edge cases were reported. As a mature _Black_ does make some exceptions to rules it otherwise holds. This section documents what those exceptions are and why this is the case. +(labels/magic-trailing-comma)= + ### The magic trailing comma _Black_ in general does not take existing formatting into account. @@ -493,6 +499,8 @@ default by (among others) GitHub and Visual Studio Code, differentiates between r-strings and R-strings. The former are syntax highlighted as regular expressions while the latter are treated as true raw strings with no special semantics. +(labels/ast-changes)= + ### AST before and after formatting When run with `--safe` (the default), _Black_ checks that the code before and after is diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index bfab905b288..861bb64bff4 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -47,6 +47,8 @@ with contextlib.ExitStack() as exit_stack: ... ``` +(labels/preview-style)= + ## Preview style Experimental, potentially disruptive style changes are gathered under the `--preview` diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index b101e179d0e..48619c6bbe8 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -26,62 +26,106 @@ python -m black {source_file_or_directory} ### Command line options -The CLI options of _Black_ can be displayed by expanding the view below or by running -`black --help`. While _Black_ has quite a few knobs these days, it is still opinionated -so style options are deliberately limited and rarely added. +The CLI options of _Black_ can be displayed by running `black --help`. All options are +also covered in more detail below. -
+While _Black_ has quite a few knobs these days, it is still opinionated so style options +are deliberately limited and rarely added. -CLI reference +Note that all command-line options listed above can also be configured using a +`pyproject.toml` file (more on that below). -```{program-output} black --help +#### `-c`, `--code` +Format the code passed in as a string. + +```console +$ black --code "print ( 'hello, world' )" +print("hello, world") ``` -
+#### `-l`, `--line-length` -Note that all command-line options listed above can also be configured using a -`pyproject.toml` file (more on that below). +How many characters per line to allow. The default is 88. -### Code input alternatives +See also [the style documentation](labels/line-length). -#### Standard Input +#### `-t`, `--target-version` -_Black_ supports formatting code via stdin, with the result being printed to stdout. -Just let _Black_ know with `-` as the path. +Python versions that should be supported by Black's output. You should include all +versions that your code supports. If you support Python 3.7 through 3.10, you should +write: ```console -$ echo "print ( 'hello, world' )" | black - -print("hello, world") -reformatted - -All done! ✨ 🍰 ✨ -1 file reformatted. +$ black -t py37 -t py38 -t py39 -t py310 ``` -**Tip:** if you need _Black_ to treat stdin input as a file passed directly via the CLI, -use `--stdin-filename`. Useful to make sure _Black_ will respect the `--force-exclude` -option on some editors that rely on using stdin. +In a [configuration file](#configuration-via-a-file), you can write: -#### As a string +```toml +target-versions = ["py37", "py38", "py39", "py310"] +``` -You can also pass code as a string using the `-c` / `--code` option. +_Black_ uses this option to decide what grammar to use to parse your code. In addition, +it may use it to decide what style to use. For example, support for a trailing comma +after `*args` in a function call was added in Python 3.5, so _Black_ will add this comma +only if the target versions are all Python 3.5 or higher: ```console -$ black --code "print ( 'hello, world' )" -print("hello, world") +$ black --line-length=10 --target-version=py35 -c 'f(a, *args)' +f( + a, + *args, +) +$ black --line-length=10 --target-version=py34 -c 'f(a, *args)' +f( + a, + *args +) +$ black --line-length=10 --target-version=py34 --target-version=py35 -c 'f(a, *args)' +f( + a, + *args +) ``` -### Writeback and reporting +#### `--pyi` -By default _Black_ reformats the files given and/or found in place. Sometimes you need -_Black_ to just tell you what it _would_ do without actually rewriting the Python files. +Format all input files like typing stubs regardless of file extension. This is useful +when piping source on standard input. -There's two variations to this mode that are independently enabled by their respective -flags. Both variations can be enabled at once. +#### `--ipynb` + +Format all input files like Jupyter Notebooks regardless of file extension. This is +useful when piping source on standard input. + +#### `--python-cell-magics` + +When processing Jupyter Notebooks, add the given magic to the list of known python- +magics. Useful for formatting cells with custom python magics. + +#### `-S, --skip-string-normalization` + +By default, _Black_ uses double quotes for all strings and normalizes string prefixes, +as described in [the style documentation](labels/strings). If this option is given, +strings are left unchanged instead. + +#### `-C, --skip-magic-trailing-comma` + +By default, _Black_ uses existing trailing commas as an indication that short lines +should be left separate, as described in +[the style documentation](labels/magic-trailing-comma). If this option is given, the +magic trailing comma is ignored. + +#### `--preview` + +Enable potentially disruptive style changes that may be added to Black's main +functionality in the next major release. Read more about +[our preview style](labels/preview-style). (labels/exit-code)= -#### Exit code +#### `--check` Passing `--check` will make _Black_ exit with: @@ -111,12 +155,12 @@ $ echo $? 123 ``` -#### Diffs +#### `--diff` Passing `--diff` will make _Black_ print out diffs that indicate what changes _Black_ would've made. They are printed to stdout so capturing them is simple. -If you'd like colored diffs, you can enable them with the `--color`. +If you'd like colored diffs, you can enable them with `--color`. ```console $ black test.py --diff @@ -130,22 +174,92 @@ All done! ✨ 🍰 ✨ 1 file would be reformatted. ``` -### Output verbosity +#### `--color` / `--no-color` -_Black_ in general tries to produce the right amount of output, balancing between -usefulness and conciseness. By default, _Black_ emits files modified and error messages, -plus a short summary. +Show (or do not show) colored diff. Only applies when `--diff` is given. + +#### `--fast` / `--safe` + +By default, _Black_ performs [an AST safety check](labels/ast-changes) after formatting +your code. The `--fast` flag turns off this check and the `--safe` flag explicitly +enables it. + +#### `--required-version` + +Require a specific version of _Black_ to be running. This is useful for ensuring that +all contributors to your project are using the same version, because different versions +of _Black_ may format code a little differently. This option can be set in a +configuration file for consistent results across environments. ```console -$ black src/ +$ black --version +black, 23.3.0 (compiled: yes) +$ black --required-version 23.3.0 -c "format = 'this'" +format = "this" +$ black --required-version 31.5b2 -c "still = 'beta?!'" +Oh no! 💥 💔 💥 The required version does not match the running version! +``` + +You can also pass just the major version: + +```console +$ black --required-version 22 -c "format = 'this'" +format = "this" +$ black --required-version 31 -c "still = 'beta?!'" +Oh no! 💥 💔 💥 The required version does not match the running version! +``` + +Because of our [stability policy](../the_black_code_style/index.md), this will guarantee +stable formatting, but still allow you to take advantage of improvements that do not +affect formatting. + +#### `--include` + +A regular expression that matches files and directories that should be included on +recursive searches. An empty value means all files are included regardless of the name. +Use forward slashes for directories on all platforms (Windows, too). Exclusions are +calculated first, inclusions later. + +#### `--exclude` + +A regular expression that matches files and directories that should be excluded on +recursive searches. An empty value means no paths are excluded. Use forward slashes for +directories on all platforms (Windows, too). Exclusions are calculated first, inclusions +later. + +#### `--extend-exclude` + +Like `--exclude`, but adds additional files and directories on top of the excluded ones. +Useful if you simply want to add to the default. + +#### `--force-exclude` + +Like `--exclude`, but files and directories matching this regex will be excluded even +when they are passed explicitly as arguments. This is useful when invoking _Black_ +programmatically on changed files, such as in a pre-commit hook or editor plugin. + +#### `--stdin-filename` + +The name of the file when passing it through stdin. Useful to make sure Black will +respect the `--force-exclude` option on some editors that rely on using stdin. + +#### `-W`, `--workers` + +When _Black_ formats multiple files, it may use a process pool to speed up formatting. +This option controls the number of parallel workers. + +#### `-q`, `--quiet` + +Passing `-q` / `--quiet` will cause _Black_ to stop emitting all non-critical output. +Error messages will still be emitted (which can silenced by `2>/dev/null`). + +```console +$ black src/ -q error: cannot format src/black_primer/cli.py: Cannot parse: 5:6: mport asyncio -reformatted src/black_primer/lib.py -reformatted src/blackd/__init__.py -reformatted src/black/__init__.py -Oh no! 💥 💔 💥 -3 files reformatted, 2 files left unchanged, 1 file failed to reformat. ``` +#### `-v`, `--verbose` + Passing `-v` / `--verbose` will cause _Black_ to also emit messages about files that were not changed or were ignored due to exclusion patterns. If _Black_ is using a configuration file, a blue message detailing which one it is using will be emitted. @@ -164,35 +278,73 @@ Oh no! 💥 💔 💥 3 files reformatted, 2 files left unchanged, 1 file failed to reformat ``` -Passing `-q` / `--quiet` will cause _Black_ to stop emitting all non-critial output. -Error messages will still be emitted (which can silenced by `2>/dev/null`). +#### `--version` + +You can check the version of _Black_ you have installed using the `--version` flag. ```console -$ black src/ -q -error: cannot format src/black_primer/cli.py: Cannot parse: 5:6: mport asyncio +$ black --version +black, 23.3.0 ``` -### Versions +#### `--config` -You can check the version of _Black_ you have installed using the `--version` flag. +Read configuration options from a configuration file. See +[below](#configuration-via-a-file) for more details on the configuration file. + +#### `-h`, `--help` + +Show available command-line options and exit. + +### Code input alternatives + +_Black_ supports formatting code via stdin, with the result being printed to stdout. +Just let _Black_ know with `-` as the path. ```console -$ black --version -black, version 23.3.0 +$ echo "print ( 'hello, world' )" | black - +print("hello, world") +reformatted - +All done! ✨ 🍰 ✨ +1 file reformatted. ``` -An option to require a specific version to be running is also provided. +**Tip:** if you need _Black_ to treat stdin input as a file passed directly via the CLI, +use `--stdin-filename`. Useful to make sure _Black_ will respect the `--force-exclude` +option on some editors that rely on using stdin. + +You can also pass code as a string using the `-c` / `--code` option. + +### Writeback and reporting + +By default _Black_ reformats the files given and/or found in place. Sometimes you need +_Black_ to just tell you what it _would_ do without actually rewriting the Python files. + +There's two variations to this mode that are independently enabled by their respective +flags: + +- `--check` (exit with code 1 if any file would be reformatted) +- `--diff` (print a diff instead of reformatting files) + +Both variations can be enabled at once. + +### Output verbosity + +_Black_ in general tries to produce the right amount of output, balancing between +usefulness and conciseness. By default, _Black_ emits files modified and error messages, +plus a short summary. ```console -$ black --required-version 21.9b0 -c "format = 'this'" -format = "this" -$ black --required-version 31.5b2 -c "still = 'beta?!'" -Oh no! 💥 💔 💥 The required version does not match the running version! +$ black src/ +error: cannot format src/black_primer/cli.py: Cannot parse: 5:6: mport asyncio +reformatted src/black_primer/lib.py +reformatted src/blackd/__init__.py +reformatted src/black/__init__.py +Oh no! 💥 💔 💥 +3 files reformatted, 2 files left unchanged, 1 file failed to reformat. ``` -This is useful for example when running _Black_ in multiple environments that haven't -necessarily installed the correct version. This option can be set in a configuration -file for consistent results across environments. +The `--quiet` and `--verbose` flags control output verbosity. ## Configuration via a file diff --git a/scripts/check_version_in_basics_example.py b/scripts/check_version_in_basics_example.py index c62780d97ab..7f559b3aee1 100644 --- a/scripts/check_version_in_basics_example.py +++ b/scripts/check_version_in_basics_example.py @@ -20,20 +20,21 @@ def main(changes: str, the_basics: str) -> None: the_basics_html = commonmark.commonmark(the_basics) the_basics_soup = BeautifulSoup(the_basics_html, "html.parser") - (version_example,) = [ + version_examples = [ code_block.string for code_block in the_basics_soup.find_all(class_="language-console") if "$ black --version" in code_block.string ] for tag in tags: - if tag in version_example and tag != latest_tag: - print( - "Please set the version in the ``black --version`` " - "example from ``the_basics.md`` to be the latest one.\n" - f"Expected {latest_tag}, got {tag}.\n" - ) - sys.exit(1) + for version_example in version_examples: + if tag in version_example and tag != latest_tag: + print( + "Please set the version in the ``black --version`` " + "examples from ``the_basics.md`` to be the latest one.\n" + f"Expected {latest_tag}, got {tag}.\n" + ) + sys.exit(1) if __name__ == "__main__": From c42178690e96f3bf061ad44f70dec52b1d8a299a Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 24 May 2023 21:06:08 -0700 Subject: [PATCH 021/279] Fix docs formatting (#3704) --- docs/guides/introducing_black_to_your_project.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/guides/introducing_black_to_your_project.md b/docs/guides/introducing_black_to_your_project.md index 53bb0d9fcd6..71a566fbda1 100644 --- a/docs/guides/introducing_black_to_your_project.md +++ b/docs/guides/introducing_black_to_your_project.md @@ -46,5 +46,6 @@ $ git config blame.ignoreRevsFile .git-blame-ignore-revs **The one caveat is that some online Git-repositories like GitLab do not yet support ignoring revisions using their native blame UI.** So blame information will be cluttered with a reformatting commit on those platforms. (If you'd like this feature, there's an -open issue for [GitLab](https://gitlab.com/gitlab-org/gitlab/-/issues/31423)). -[GitHub supports `.git-blame-ignore-revs`](https://docs.github.com/en/repositories/working-with-files/using-files/viewing-a-file#ignore-commits-in-the-blame-view) by default in blame views however. +open issue for [GitLab](https://gitlab.com/gitlab-org/gitlab/-/issues/31423)). +[GitHub supports `.git-blame-ignore-revs`](https://docs.github.com/en/repositories/working-with-files/using-files/viewing-a-file#ignore-commits-in-the-blame-view) +by default in blame views however. From a4032dce645b83e1faccc7274864869ddfe279fc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 May 2023 06:33:41 -0700 Subject: [PATCH 022/279] Bump pypa/cibuildwheel from 2.12.3 to 2.13.0 (#3710) Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.12.3 to 2.13.0. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.12.3...v2.13.0) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pypi_upload.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index 4b59c481bb0..20787f71cee 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -58,7 +58,7 @@ jobs: - uses: actions/checkout@v3 - name: Build wheels via cibuildwheel - uses: pypa/cibuildwheel@v2.12.3 + uses: pypa/cibuildwheel@v2.13.0 env: CIBW_ARCHS_MACOS: "${{ matrix.macos_arch }}" From a538ab76636bbe71b7fbfeaf56fd8e61805df38f Mon Sep 17 00:00:00 2001 From: jmcb Date: Wed, 31 May 2023 22:29:31 +0100 Subject: [PATCH 023/279] blackd: show default values for options (#3712) * blackd: show default values for options Reference: https://click.palletsprojects.com/en/8.1.x/api/#click.Option * Fix spacing in CHANGES.md --- CHANGES.md | 3 +++ src/blackd/__init__.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6a9923f8d8d..762a799a1d1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -44,6 +44,9 @@ +- The `blackd` argument parser now shows the default values for options in their help + text (#3712) + ### Integrations diff --git a/src/blackd/__init__.py b/src/blackd/__init__.py index ba4750b8298..d331ad000bc 100644 --- a/src/blackd/__init__.py +++ b/src/blackd/__init__.py @@ -59,9 +59,15 @@ class InvalidVariantHeader(Exception): @click.command(context_settings={"help_option_names": ["-h", "--help"]}) @click.option( - "--bind-host", type=str, help="Address to bind the server to.", default="localhost" + "--bind-host", + type=str, + help="Address to bind the server to.", + default="localhost", + show_default=True, +) +@click.option( + "--bind-port", type=int, help="Port to listen on", default=45484, show_default=True ) -@click.option("--bind-port", type=int, help="Port to listen on", default=45484) @click.version_option(version=black.__version__) def main(bind_host: str, bind_port: int) -> None: logging.basicConfig(level=logging.INFO) From 3aad6e385bfbd4348b2e13695cb6741806951160 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 1 Jun 2023 18:37:08 -0700 Subject: [PATCH 024/279] Add support for PEP 695 syntax (#3703) --- CHANGES.md | 2 ++ pyproject.toml | 2 ++ src/black/__init__.py | 3 ++ src/black/linegen.py | 12 +++++++ src/black/mode.py | 21 ++++++++++++ src/blib2to3/Grammar.txt | 13 +++++-- src/blib2to3/pygram.py | 6 ++++ tests/data/py_312/type_aliases.py | 13 +++++++ tests/data/py_312/type_params.py | 57 +++++++++++++++++++++++++++++++ tests/test_black.py | 30 +++++++++++++--- tests/test_format.py | 7 ++++ 11 files changed, 159 insertions(+), 7 deletions(-) create mode 100644 tests/data/py_312/type_aliases.py create mode 100644 tests/data/py_312/type_params.py diff --git a/CHANGES.md b/CHANGES.md index 762a799a1d1..fb3dea8c348 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -32,6 +32,8 @@ +- Add support for the new PEP 695 syntax in Python 3.12 (#3703) + ### Performance diff --git a/pyproject.toml b/pyproject.toml index 435626ac8f4..6803a627e9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -214,4 +214,6 @@ filterwarnings = [ # aiohttp is using deprecated cgi modules - Safe to remove when fixed: # https://github.com/aio-libs/aiohttp/issues/6905 '''ignore:'cgi' is deprecated and slated for removal in Python 3.13:DeprecationWarning''', + # Work around https://github.com/pytest-dev/pytest/issues/10977 for Python 3.12 + '''ignore:(Attribute s|Attribute n|ast.Str|ast.Bytes|ast.NameConstant|ast.Num) is deprecated and will be removed in Python 3.14:DeprecationWarning''' ] diff --git a/src/black/__init__.py b/src/black/__init__.py index 871e9a0d7c8..8a759aa493a 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1275,6 +1275,9 @@ def get_features_used( # noqa: C901 ): features.add(Feature.VARIADIC_GENERICS) + elif n.type in (syms.type_stmt, syms.typeparams): + features.add(Feature.TYPE_PARAMS) + return features diff --git a/src/black/linegen.py b/src/black/linegen.py index b6b83da26f7..0091cbb3bd1 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -215,6 +215,18 @@ def visit_stmt( yield from self.visit(child) + def visit_typeparams(self, node: Node) -> Iterator[Line]: + yield from self.visit_default(node) + node.children[0].prefix = "" + + def visit_typevartuple(self, node: Node) -> Iterator[Line]: + yield from self.visit_default(node) + node.children[1].prefix = "" + + def visit_paramspec(self, node: Node) -> Iterator[Line]: + yield from self.visit_default(node) + node.children[1].prefix = "" + def visit_dictsetmaker(self, node: Node) -> Iterator[Line]: if Preview.wrap_long_dict_values_in_parens in self.mode: for i, child in enumerate(node.children): diff --git a/src/black/mode.py b/src/black/mode.py index a5841edb30a..1091494afac 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -30,6 +30,7 @@ class TargetVersion(Enum): PY39 = 9 PY310 = 10 PY311 = 11 + PY312 = 12 class Feature(Enum): @@ -51,6 +52,7 @@ class Feature(Enum): VARIADIC_GENERICS = 15 DEBUG_F_STRINGS = 16 PARENTHESIZED_CONTEXT_MANAGERS = 17 + TYPE_PARAMS = 18 FORCE_OPTIONAL_PARENTHESES = 50 # __future__ flags @@ -143,6 +145,25 @@ class Feature(Enum): Feature.EXCEPT_STAR, Feature.VARIADIC_GENERICS, }, + TargetVersion.PY312: { + Feature.F_STRINGS, + Feature.DEBUG_F_STRINGS, + Feature.NUMERIC_UNDERSCORES, + Feature.TRAILING_COMMA_IN_CALL, + Feature.TRAILING_COMMA_IN_DEF, + Feature.ASYNC_KEYWORDS, + Feature.FUTURE_ANNOTATIONS, + Feature.ASSIGNMENT_EXPRESSIONS, + Feature.RELAXED_DECORATORS, + Feature.POS_ONLY_ARGUMENTS, + Feature.UNPACKING_ON_FLOW, + Feature.ANN_ASSIGN_EXTENDED_RHS, + Feature.PARENTHESIZED_CONTEXT_MANAGERS, + Feature.PATTERN_MATCHING, + Feature.EXCEPT_STAR, + Feature.VARIADIC_GENERICS, + Feature.TYPE_PARAMS, + }, } diff --git a/src/blib2to3/Grammar.txt b/src/blib2to3/Grammar.txt index bd8a452a386..e48e66363fb 100644 --- a/src/blib2to3/Grammar.txt +++ b/src/blib2to3/Grammar.txt @@ -12,11 +12,17 @@ file_input: (NEWLINE | stmt)* ENDMARKER single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE eval_input: testlist NEWLINE* ENDMARKER +typevar: NAME [':' expr] +paramspec: '**' NAME +typevartuple: '*' NAME +typeparam: typevar | paramspec | typevartuple +typeparams: '[' typeparam (',' typeparam)* [','] ']' + decorator: '@' namedexpr_test NEWLINE decorators: decorator+ decorated: decorators (classdef | funcdef | async_funcdef) async_funcdef: ASYNC funcdef -funcdef: 'def' NAME parameters ['->' test] ':' suite +funcdef: 'def' NAME [typeparams] parameters ['->' test] ':' suite parameters: '(' [typedargslist] ')' # The following definition for typedarglist is equivalent to this set of rules: @@ -74,7 +80,7 @@ vfplist: vfpdef (',' vfpdef)* [','] stmt: simple_stmt | compound_stmt simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE -small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | +small_stmt: (type_stmt | expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | exec_stmt | assert_stmt) expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) | ('=' (yield_expr|testlist_star_expr))*) @@ -105,6 +111,7 @@ dotted_name: NAME ('.' NAME)* global_stmt: ('global' | 'nonlocal') NAME (',' NAME)* exec_stmt: 'exec' expr ['in' test [',' test]] assert_stmt: 'assert' test [',' test] +type_stmt: "type" NAME [typeparams] '=' expr compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt | match_stmt async_stmt: ASYNC (funcdef | with_stmt | for_stmt) @@ -174,7 +181,7 @@ dictsetmaker: ( ((test ':' asexpr_test | '**' expr) ((test [':=' test] | star_expr) (comp_for | (',' (test [':=' test] | star_expr))* [','])) ) -classdef: 'class' NAME ['(' [arglist] ')'] ':' suite +classdef: 'class' NAME [typeparams] ['(' [arglist] ')'] ':' suite arglist: argument (',' argument)* [','] diff --git a/src/blib2to3/pygram.py b/src/blib2to3/pygram.py index 99012cdd9cb..15702e4059e 100644 --- a/src/blib2to3/pygram.py +++ b/src/blib2to3/pygram.py @@ -95,6 +95,7 @@ class _python_symbols(Symbols): old_test: int or_test: int parameters: int + paramspec: int pass_stmt: int pattern: int patterns: int @@ -126,7 +127,12 @@ class _python_symbols(Symbols): tname_star: int trailer: int try_stmt: int + type_stmt: int typedargslist: int + typeparam: int + typeparams: int + typevar: int + typevartuple: int varargslist: int vfpdef: int vfplist: int diff --git a/tests/data/py_312/type_aliases.py b/tests/data/py_312/type_aliases.py new file mode 100644 index 00000000000..84e07e50fe2 --- /dev/null +++ b/tests/data/py_312/type_aliases.py @@ -0,0 +1,13 @@ +type A=int +type Gen[T]=list[T] + +type = aliased +print(type(42)) + +# output + +type A = int +type Gen[T] = list[T] + +type = aliased +print(type(42)) diff --git a/tests/data/py_312/type_params.py b/tests/data/py_312/type_params.py new file mode 100644 index 00000000000..5f8ec43267c --- /dev/null +++ b/tests/data/py_312/type_params.py @@ -0,0 +1,57 @@ +def func [T ](): pass +async def func [ T ] (): pass +class C[ T ] : pass + +def all_in[T : int,U : (bytes, str),* Ts,**P](): pass + +def really_long[WhatIsTheLongestTypeVarNameYouCanThinkOfEnoughToMakeBlackSplitThisLine](): pass + +def even_longer[WhatIsTheLongestTypeVarNameYouCanThinkOfEnoughToMakeBlackSplitThisLine: WhatIfItHadABound](): pass + +def it_gets_worse[WhatIsTheLongestTypeVarNameYouCanThinkOfEnoughToMakeBlackSplitThisLine, ItCouldBeGenericOverMultipleTypeVars](): pass + +def magic[Trailing, Comma,](): pass + +# output + + +def func[T](): + pass + + +async def func[T](): + pass + + +class C[T]: + pass + + +def all_in[T: int, U: (bytes, str), *Ts, **P](): + pass + + +def really_long[ + WhatIsTheLongestTypeVarNameYouCanThinkOfEnoughToMakeBlackSplitThisLine +](): + pass + + +def even_longer[ + WhatIsTheLongestTypeVarNameYouCanThinkOfEnoughToMakeBlackSplitThisLine: WhatIfItHadABound +](): + pass + + +def it_gets_worse[ + WhatIsTheLongestTypeVarNameYouCanThinkOfEnoughToMakeBlackSplitThisLine, + ItCouldBeGenericOverMultipleTypeVars, +](): + pass + + +def magic[ + Trailing, + Comma, +](): + pass diff --git a/tests/test_black.py b/tests/test_black.py index 00de5b745e7..42b0161d156 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -271,6 +271,15 @@ def test_pep_572_version_detection(self) -> None: versions = black.detect_target_versions(root) self.assertIn(black.TargetVersion.PY38, versions) + def test_pep_695_version_detection(self) -> None: + for file in ("type_aliases", "type_params"): + source, _ = read_data("py_312", file) + root = black.lib2to3_parse(source) + features = black.get_features_used(root) + self.assertIn(black.Feature.TYPE_PARAMS, features) + versions = black.detect_target_versions(root) + self.assertIn(black.TargetVersion.PY312, versions) + def test_expression_ff(self) -> None: source, expected = read_data("simple_cases", "expression.py") tmp_file = Path(black.dump_to_file(source)) @@ -1533,14 +1542,25 @@ def test_infer_target_version(self) -> None: for version, expected in [ ("3.6", [TargetVersion.PY36]), ("3.11.0rc1", [TargetVersion.PY311]), - (">=3.10", [TargetVersion.PY310, TargetVersion.PY311]), - (">=3.10.6", [TargetVersion.PY310, TargetVersion.PY311]), + (">=3.10", [TargetVersion.PY310, TargetVersion.PY311, TargetVersion.PY312]), + ( + ">=3.10.6", + [TargetVersion.PY310, TargetVersion.PY311, TargetVersion.PY312], + ), ("<3.6", [TargetVersion.PY33, TargetVersion.PY34, TargetVersion.PY35]), (">3.7,<3.10", [TargetVersion.PY38, TargetVersion.PY39]), - (">3.7,!=3.8,!=3.9", [TargetVersion.PY310, TargetVersion.PY311]), + ( + ">3.7,!=3.8,!=3.9", + [TargetVersion.PY310, TargetVersion.PY311, TargetVersion.PY312], + ), ( "> 3.9.4, != 3.10.3", - [TargetVersion.PY39, TargetVersion.PY310, TargetVersion.PY311], + [ + TargetVersion.PY39, + TargetVersion.PY310, + TargetVersion.PY311, + TargetVersion.PY312, + ], ), ( "!=3.3,!=3.4", @@ -1552,6 +1572,7 @@ def test_infer_target_version(self) -> None: TargetVersion.PY39, TargetVersion.PY310, TargetVersion.PY311, + TargetVersion.PY312, ], ), ( @@ -1566,6 +1587,7 @@ def test_infer_target_version(self) -> None: TargetVersion.PY39, TargetVersion.PY310, TargetVersion.PY311, + TargetVersion.PY312, ], ), ("==3.8.*", [TargetVersion.PY38]), diff --git a/tests/test_format.py b/tests/test_format.py index 5a7b3bb6762..8e0ada99cba 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -134,6 +134,13 @@ def test_python_311(filename: str) -> None: assert_format(source, expected, mode, minimum_version=(3, 11)) +@pytest.mark.parametrize("filename", all_data_cases("py_312")) +def test_python_312(filename: str) -> None: + source, expected = read_data("py_312", filename) + mode = black.Mode(target_versions={black.TargetVersion.PY312}) + assert_format(source, expected, mode, minimum_version=(3, 12)) + + @pytest.mark.parametrize("filename", all_data_cases("fast")) def test_fast_cases(filename: str) -> None: source, expected = read_data("fast", filename) From 898915d5569f503c278ef53cb6f10e003034943c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Sat, 10 Jun 2023 19:54:21 +0300 Subject: [PATCH 025/279] Use aware datetimes to represent UTC (#3728) Avoids a Python 3.12 deprecation warning. Subtle difference: previously, timestamps in diff filenames had the `+0000` separated from the timestamp by space. With this, the space is there no more, and there is a colon, as in `+00:00`. --- CHANGES.md | 2 ++ docs/usage_and_configuration/the_basics.md | 4 ++-- src/black/__init__.py | 18 +++++++++--------- src/blackd/__init__.py | 10 +++++----- tests/test_black.py | 10 +++++----- tests/test_blackd.py | 2 +- 6 files changed, 24 insertions(+), 22 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fb3dea8c348..658faad3a78 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -42,6 +42,8 @@ +- Use aware UTC datetimes internally, avoids deprecation warning on Python 3.12 (#3728) + ### _Blackd_ diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 48619c6bbe8..6f3a3cff30c 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -164,8 +164,8 @@ If you'd like colored diffs, you can enable them with `--color`. ```console $ black test.py --diff ---- test.py 2021-03-08 22:23:40.848954 +0000 -+++ test.py 2021-03-08 22:23:47.126319 +0000 +--- test.py 2021-03-08 22:23:40.848954+00:00 ++++ test.py 2021-03-08 22:23:47.126319+00:00 @@ -1 +1 @@ -print ( 'hello, world' ) +print("hello, world") diff --git a/src/black/__init__.py b/src/black/__init__.py index 8a759aa493a..dbcb559f09d 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -7,7 +7,7 @@ import traceback from contextlib import contextmanager from dataclasses import replace -from datetime import datetime +from datetime import datetime, timezone from enum import Enum from json.decoder import JSONDecodeError from pathlib import Path @@ -807,7 +807,7 @@ def format_file_in_place( elif src.suffix == ".ipynb": mode = replace(mode, is_ipynb=True) - then = datetime.utcfromtimestamp(src.stat().st_mtime) + then = datetime.fromtimestamp(src.stat().st_mtime, timezone.utc) header = b"" with open(src, "rb") as buf: if mode.skip_source_first_line: @@ -828,9 +828,9 @@ def format_file_in_place( with open(src, "w", encoding=encoding, newline=newline) as f: f.write(dst_contents) elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF): - now = datetime.utcnow() - src_name = f"{src}\t{then} +0000" - dst_name = f"{src}\t{now} +0000" + now = datetime.now(timezone.utc) + src_name = f"{src}\t{then}" + dst_name = f"{src}\t{now}" if mode.is_ipynb: diff_contents = ipynb_diff(src_contents, dst_contents, src_name, dst_name) else: @@ -868,7 +868,7 @@ def format_stdin_to_stdout( write a diff to stdout. The `mode` argument is passed to :func:`format_file_contents`. """ - then = datetime.utcnow() + then = datetime.now(timezone.utc) if content is None: src, encoding, newline = decode_bytes(sys.stdin.buffer.read()) @@ -893,9 +893,9 @@ def format_stdin_to_stdout( dst += "\n" f.write(dst) elif write_back in (WriteBack.DIFF, WriteBack.COLOR_DIFF): - now = datetime.utcnow() - src_name = f"STDIN\t{then} +0000" - dst_name = f"STDOUT\t{now} +0000" + now = datetime.now(timezone.utc) + src_name = f"STDIN\t{then}" + dst_name = f"STDOUT\t{now}" d = diff(src, dst, src_name, dst_name) if write_back == WriteBack.COLOR_DIFF: d = color_diff(d) diff --git a/src/blackd/__init__.py b/src/blackd/__init__.py index d331ad000bc..c1b69feed63 100644 --- a/src/blackd/__init__.py +++ b/src/blackd/__init__.py @@ -1,7 +1,7 @@ import asyncio import logging from concurrent.futures import Executor, ProcessPoolExecutor -from datetime import datetime +from datetime import datetime, timezone from functools import partial from multiprocessing import freeze_support from typing import Set, Tuple @@ -138,7 +138,7 @@ async def handle(request: web.Request, executor: Executor) -> web.Response: req_bytes = await request.content.read() charset = request.charset if request.charset is not None else "utf8" req_str = req_bytes.decode(charset) - then = datetime.utcnow() + then = datetime.now(timezone.utc) header = "" if skip_source_first_line: @@ -165,9 +165,9 @@ async def handle(request: web.Request, executor: Executor) -> web.Response: # Only output the diff in the HTTP response only_diff = bool(request.headers.get(DIFF_HEADER, False)) if only_diff: - now = datetime.utcnow() - src_name = f"In\t{then} +0000" - dst_name = f"Out\t{now} +0000" + now = datetime.now(timezone.utc) + src_name = f"In\t{then}" + dst_name = f"Out\t{now}" loop = asyncio.get_event_loop() formatted_str = await loop.run_in_executor( executor, diff --git a/tests/test_black.py b/tests/test_black.py index 42b0161d156..5f2e6f5b14c 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -207,8 +207,8 @@ def test_piping(self) -> None: def test_piping_diff(self) -> None: diff_header = re.compile( - r"(STDIN|STDOUT)\t\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d\.\d\d\d\d\d\d " - r"\+\d\d\d\d" + r"(STDIN|STDOUT)\t\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d\.\d\d\d\d\d\d" + r"\+\d\d:\d\d" ) source, _ = read_data("simple_cases", "expression.py") expected, _ = read_data("simple_cases", "expression.diff") @@ -300,7 +300,7 @@ def test_expression_diff(self) -> None: tmp_file = Path(black.dump_to_file(source)) diff_header = re.compile( rf"{re.escape(str(tmp_file))}\t\d\d\d\d-\d\d-\d\d " - r"\d\d:\d\d:\d\d\.\d\d\d\d\d\d \+\d\d\d\d" + r"\d\d:\d\d:\d\d\.\d\d\d\d\d\d\+\d\d:\d\d" ) try: result = BlackRunner().invoke( @@ -411,7 +411,7 @@ def test_skip_magic_trailing_comma(self) -> None: tmp_file = Path(black.dump_to_file(source)) diff_header = re.compile( rf"{re.escape(str(tmp_file))}\t\d\d\d\d-\d\d-\d\d " - r"\d\d:\d\d:\d\d\.\d\d\d\d\d\d \+\d\d\d\d" + r"\d\d:\d\d:\d\d\.\d\d\d\d\d\d\+\d\d:\d\d" ) try: result = BlackRunner().invoke( @@ -1750,7 +1750,7 @@ def test_bpo_2142_workaround(self) -> None: tmp_file = Path(black.dump_to_file(source, ensure_final_newline=False)) diff_header = re.compile( rf"{re.escape(str(tmp_file))}\t\d\d\d\d-\d\d-\d\d " - r"\d\d:\d\d:\d\d\.\d\d\d\d\d\d \+\d\d\d\d" + r"\d\d:\d\d:\d\d\.\d\d\d\d\d\d\+\d\d:\d\d" ) try: result = BlackRunner().invoke(black.main, ["--diff", str(tmp_file)]) diff --git a/tests/test_blackd.py b/tests/test_blackd.py index 5b6461f7685..325bd7dd5aa 100644 --- a/tests/test_blackd.py +++ b/tests/test_blackd.py @@ -114,7 +114,7 @@ async def test_blackd_pyi(self) -> None: @unittest_run_loop async def test_blackd_diff(self) -> None: diff_header = re.compile( - r"(In|Out)\t\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d\.\d\d\d\d\d\d \+\d\d\d\d" + r"(In|Out)\t\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d\.\d\d\d\d\d\d\+\d\d:\d\d" ) source, _ = read_data("miscellaneous", "blackd_diff") From c76e0b03ec706619cc76bbd45e9c355c18462e2e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 10:05:49 -0700 Subject: [PATCH 026/279] Bump pypa/cibuildwheel from 2.13.0 to 2.13.1 (#3729) Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.13.0 to 2.13.1. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.13.0...v2.13.1) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pypi_upload.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index 20787f71cee..06600fcbc45 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -58,7 +58,7 @@ jobs: - uses: actions/checkout@v3 - name: Build wheels via cibuildwheel - uses: pypa/cibuildwheel@v2.13.0 + uses: pypa/cibuildwheel@v2.13.1 env: CIBW_ARCHS_MACOS: "${{ matrix.macos_arch }}" From 688f78d380706fd20776b31eb1ead98ba3f45839 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 10:08:45 -0700 Subject: [PATCH 027/279] Bump peter-evans/create-or-update-comment from 3.0.1 to 3.0.2 (#3730) Bumps [peter-evans/create-or-update-comment](https://github.com/peter-evans/create-or-update-comment) from 3.0.1 to 3.0.2. - [Release notes](https://github.com/peter-evans/create-or-update-comment/releases) - [Commits](https://github.com/peter-evans/create-or-update-comment/compare/ca08ebd5dc95aa0cd97021e9708fcd6b87138c9b...c6c9a1a66007646a28c153e2a8580a5bad27bcfa) --- updated-dependencies: - dependency-name: peter-evans/create-or-update-comment dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/diff_shades_comment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/diff_shades_comment.yml b/.github/workflows/diff_shades_comment.yml index 95dd9b01dba..22c293f91d2 100644 --- a/.github/workflows/diff_shades_comment.yml +++ b/.github/workflows/diff_shades_comment.yml @@ -41,7 +41,7 @@ jobs: - name: Create or update PR comment if: steps.metadata.outputs.needs-comment == 'true' - uses: peter-evans/create-or-update-comment@ca08ebd5dc95aa0cd97021e9708fcd6b87138c9b + uses: peter-evans/create-or-update-comment@c6c9a1a66007646a28c153e2a8580a5bad27bcfa with: comment-id: ${{ steps.find-comment.outputs.comment-id }} issue-number: ${{ steps.metadata.outputs.pr-number }} From 35722dff623f3cdf5018e3f1183cd4e02e91caa8 Mon Sep 17 00:00:00 2001 From: Alwyn Kik Date: Mon, 12 Jun 2023 21:20:31 +0200 Subject: [PATCH 028/279] Max line length with bugbear (#3731) * Make phrasing for flake8 users more concise max-line-length should be 80 with flake8-bugbear Fixes #3716 * Re-add rationale and an explanation for disabling E203 * Run pre-commit --- docs/the_black_code_style/current_style.md | 50 +++++++++++----------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/docs/the_black_code_style/current_style.md b/docs/the_black_code_style/current_style.md index e2625f9e16e..0fb59fe5aae 100644 --- a/docs/the_black_code_style/current_style.md +++ b/docs/the_black_code_style/current_style.md @@ -160,33 +160,35 @@ harder to work with line lengths exceeding 100 characters. It also adversely aff side-by-side diff review on typical screen resolutions. Long lines also make it harder to present code neatly in documentation or talk slides. -If you're using Flake8, you can bump `max-line-length` to 88 and mostly forget about it. -However, it's better if you use [Bugbear](https://github.com/PyCQA/flake8-bugbear)'s -B950 warning instead of E501, and bump the max line length to 88 (or the `--line-length` -you used for black), which will align more with black's _"try to respect -`--line-length`, but don't become crazy if you can't"_. You'd do it like this: - -```ini -[flake8] -max-line-length = 88 -... -select = C,E,F,W,B,B950 -extend-ignore = E203, E501 -``` +#### Flake8 -Explanation of why E203 is disabled can be found further in this documentation. And if -you're curious about the reasoning behind B950, -[Bugbear's documentation](https://github.com/PyCQA/flake8-bugbear#opinionated-warnings) -explains it. The tl;dr is "it's like highway speed limits, we won't bother you if you -overdo it by a few km/h". +If you use Flake8, you have a few options: -**If you're looking for a minimal, black-compatible flake8 configuration:** +1. Recommended is using [Bugbear](https://github.com/PyCQA/flake8-bugbear) and enabling + its B950 check instead of using Flake8's E501, because it aligns with Black's 10% + rule. Install Bugbear and use the following config: -```ini -[flake8] -max-line-length = 88 -extend-ignore = E203 -``` + ```ini + [flake8] + max-line-length = 80 + ... + select = C,E,F,W,B,B950 + extend-ignore = E203, E501 + ``` + + The rationale for E950 is explained in + [Bugbear's documentation](https://github.com/PyCQA/flake8-bugbear#opinionated-warnings). + +2. For a minimally compatible config: + + ```ini + [flake8] + max-line-length = 88 + extend-ignore = E203 + ``` + +An explanation of why E203 is disabled can be found in the [Slices section](#slices) of +this page. ### Empty lines From 01b8d3d4095ebdb91d0d39012a517931625c63cb Mon Sep 17 00:00:00 2001 From: "Yilei \"Dolee\" Yang" Date: Thu, 15 Jun 2023 17:08:26 -0700 Subject: [PATCH 029/279] Do not add trailing commas to return type annotations using PEP 604 unions (#3735) Fix #3638: Do not add trailing commas to return type annotations using PEP 604 unions. --- CHANGES.md | 3 +++ src/black/linegen.py | 7 +++++++ tests/data/simple_cases/pep_604.py | 25 +++++++++++++++++++++++++ 3 files changed, 35 insertions(+) create mode 100644 tests/data/simple_cases/pep_604.py diff --git a/CHANGES.md b/CHANGES.md index 658faad3a78..fd4d911287d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,9 @@ +- Fix a bug where an illegal trailing comma was added to return type annotations using + PEP 604 unions (#3735) + ### Preview style diff --git a/src/black/linegen.py b/src/black/linegen.py index 0091cbb3bd1..ad21307c311 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -918,6 +918,13 @@ def bracket_split_build_line( ) if isinstance(node, Node) and isinstance(node.prev_sibling, Leaf) ) + # Except the false negatives above for PEP 604 unions where we + # can't add the comma. + and not ( + leaves[0].parent + and leaves[0].parent.next_sibling + and leaves[0].parent.next_sibling.type == token.VBAR + ) ) if original.is_import or no_commas: diff --git a/tests/data/simple_cases/pep_604.py b/tests/data/simple_cases/pep_604.py new file mode 100644 index 00000000000..b68d59d6440 --- /dev/null +++ b/tests/data/simple_cases/pep_604.py @@ -0,0 +1,25 @@ +def some_very_long_name_function() -> my_module.Asdf | my_module.AnotherType | my_module.YetAnotherType | None: + pass + + +def some_very_long_name_function() -> my_module.Asdf | my_module.AnotherType | my_module.YetAnotherType | my_module.EvenMoreType | None: + pass + + +# output + + +def some_very_long_name_function() -> ( + my_module.Asdf | my_module.AnotherType | my_module.YetAnotherType | None +): + pass + + +def some_very_long_name_function() -> ( + my_module.Asdf + | my_module.AnotherType + | my_module.YetAnotherType + | my_module.EvenMoreType + | None +): + pass From e7783e9ab27e95cd9b50c2ef84174f4e1c9d60cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jun 2023 06:58:20 -0700 Subject: [PATCH 030/279] Bump myst-parser from 1.0.0 to 2.0.0 in /docs (#3738) Bumps [myst-parser](https://github.com/executablebooks/MyST-Parser) from 1.0.0 to 2.0.0. - [Release notes](https://github.com/executablebooks/MyST-Parser/releases) - [Changelog](https://github.com/executablebooks/MyST-Parser/blob/master/CHANGELOG.md) - [Commits](https://github.com/executablebooks/MyST-Parser/compare/v1.0.0...v2.0.0) --- updated-dependencies: - dependency-name: myst-parser dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 7b26d089b01..f1b47c69413 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,6 +1,6 @@ # Used by ReadTheDocs; pinned requirements for stability. -myst-parser==1.0.0 +myst-parser==2.0.0 Sphinx==6.1.3 # Older versions break Sphinx even though they're declared to be supported. docutils==0.19 From d1248ca9beaf0ba526d265f4108836d89cf551b7 Mon Sep 17 00:00:00 2001 From: rdrll <13176405+rdrll@users.noreply.github.com> Date: Tue, 20 Jun 2023 07:06:03 -0700 Subject: [PATCH 031/279] Doc: updating url link (#3739) --- docs/contributing/index.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/contributing/index.md b/docs/contributing/index.md index f56e57c9e90..3314c8eaa39 100644 --- a/docs/contributing/index.md +++ b/docs/contributing/index.md @@ -24,7 +24,8 @@ not very). This is deliberate. _Black_ aims to provide a consistent style and ta opportunities for arguing about style. Bug reports and fixes are always welcome! Please follow the -[issue template on GitHub](https://github.com/psf/black/issues/new) for best results. +[issue templates on GitHub](https://github.com/psf/black/issues/new/choose) for best +results. Before you suggest a new feature or configuration knob, ask yourself why you want it. If it enables better integration with some workflow, fixes an inconsistency, speeds things From 453828d17d50e4bf8bdef972fa6815258e380034 Mon Sep 17 00:00:00 2001 From: Renan Santos Date: Fri, 23 Jun 2023 01:21:49 -0300 Subject: [PATCH 032/279] Fix not honouring pyproject.toml when using stdin and calling black from parent directory (#3719) Co-authored-by: Renan Rodrigues --- CHANGES.md | 2 ++ src/black/__init__.py | 5 ++++- src/black/files.py | 6 ++++-- tests/test_black.py | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fd4d911287d..e1028c17681 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -26,6 +26,8 @@ - `.pytest_cache`, `.ruff_cache` and `.vscode` are now excluded by default (#3691) +- Fix black not honouring `pyproject.toml` settings when running `--stdin-filename` and + the `pyproject.toml` found isn't in the current working directory (#3719) ### Packaging diff --git a/src/black/__init__.py b/src/black/__init__.py index dbcb559f09d..60a339cdd12 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -127,7 +127,9 @@ def read_pyproject_toml( otherwise. """ if not value: - value = find_pyproject_toml(ctx.params.get("src", ())) + value = find_pyproject_toml( + ctx.params.get("src", ()), ctx.params.get("stdin_filename", None) + ) if value is None: return None @@ -362,6 +364,7 @@ def validate_regex( @click.option( "--stdin-filename", type=str, + is_eager=True, help=( "The name of the file when passing it through stdin. Useful to make " "sure Black will respect --force-exclude option on some " diff --git a/src/black/files.py b/src/black/files.py index 8c0131126b7..65b2d0a8402 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -89,9 +89,11 @@ def find_project_root( return directory, "file system root" -def find_pyproject_toml(path_search_start: Tuple[str, ...]) -> Optional[str]: +def find_pyproject_toml( + path_search_start: Tuple[str, ...], stdin_filename: Optional[str] = None +) -> Optional[str]: """Find the absolute filepath to a pyproject.toml if it exists""" - path_project_root, _ = find_project_root(path_search_start) + path_project_root, _ = find_project_root(path_search_start, stdin_filename) path_pyproject_toml = path_project_root / "pyproject.toml" if path_pyproject_toml.is_file(): return str(path_pyproject_toml) diff --git a/tests/test_black.py b/tests/test_black.py index 5f2e6f5b14c..abb304a246d 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -104,6 +104,7 @@ class FakeContext(click.Context): def __init__(self) -> None: self.default_map: Dict[str, Any] = {} + self.params: Dict[str, Any] = {} # Dummy root, since most of the tests don't care about it self.obj: Dict[str, Any] = {"root": PROJECT_ROOT} @@ -1620,6 +1621,39 @@ def test_read_pyproject_toml(self) -> None: self.assertEqual(config["exclude"], r"\.pyi?$") self.assertEqual(config["include"], r"\.py?$") + def test_read_pyproject_toml_from_stdin(self) -> None: + with TemporaryDirectory() as workspace: + root = Path(workspace) + + src_dir = root / "src" + src_dir.mkdir() + + src_pyproject = src_dir / "pyproject.toml" + src_pyproject.touch() + + test_toml_file = THIS_DIR / "test.toml" + src_pyproject.write_text(test_toml_file.read_text()) + + src_python = src_dir / "foo.py" + src_python.touch() + + fake_ctx = FakeContext() + fake_ctx.params["src"] = ("-",) + fake_ctx.params["stdin_filename"] = str(src_python) + + with change_directory(root): + black.read_pyproject_toml(fake_ctx, FakeParameter(), None) + + config = fake_ctx.default_map + self.assertEqual(config["verbose"], "1") + self.assertEqual(config["check"], "no") + self.assertEqual(config["diff"], "y") + self.assertEqual(config["color"], "True") + self.assertEqual(config["line_length"], "79") + self.assertEqual(config["target_version"], ["py36", "py37", "py38"]) + self.assertEqual(config["exclude"], r"\.pyi?$") + self.assertEqual(config["include"], r"\.py?$") + @pytest.mark.incompatible_with_mypyc def test_find_project_root(self) -> None: with TemporaryDirectory() as workspace: From c732a1f13a4b75d84dfd74eda0a57a55936d2e22 Mon Sep 17 00:00:00 2001 From: Stian Jensen Date: Fri, 23 Jun 2023 06:22:28 +0200 Subject: [PATCH 033/279] Build with mypyc 1.3 (#3697) Several new versions of mypyc has been released since the last upgrade, and they include some performance improvements which could make the compiled version of Black run faster. https://mypy-lang.org/news.html The latest version of hatch-mypyc allows being installed next the 1.x series of mypy. --- CHANGES.md | 2 ++ pyproject.toml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e1028c17681..460f9c95114 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -33,6 +33,8 @@ +- Upgrade mypyc from 0.991 to 1.3 (#3697) + ### Parser diff --git a/pyproject.toml b/pyproject.toml index 6803a627e9a..d44623fdbd3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -119,8 +119,8 @@ sources = ["src"] [tool.hatch.build.targets.wheel.hooks.mypyc] enable-by-default = false dependencies = [ - "hatch-mypyc>=0.13.0", - "mypy==0.991", + "hatch-mypyc>=0.16.0", + "mypy==1.3", # Required stubs to be removed when the packages support PEP 561 themselves "types-typed-ast>=1.4.2", ] From 7be273531852a16a407fcb66d5efeede0f7ca474 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sat, 24 Jun 2023 16:06:12 -0700 Subject: [PATCH 034/279] Allow specifying `--workers` via environment variable (#3743) --- CHANGES.md | 2 ++ docs/usage_and_configuration/the_basics.md | 16 +++++++++++++++- src/black/__init__.py | 5 ++++- src/black/concurrency.py | 3 ++- 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 460f9c95114..9717867df4e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -25,6 +25,8 @@ +- The `--workers` argument to Black can now be specified via the `BLACK_NUM_WORKERS` + environment variable (#3743) - `.pytest_cache`, `.ruff_cache` and `.vscode` are now excluded by default (#3691) - Fix black not honouring `pyproject.toml` settings when running `--stdin-filename` and the `pyproject.toml` found isn't in the current working directory (#3719) diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 6f3a3cff30c..2a461487210 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -246,7 +246,8 @@ respect the `--force-exclude` option on some editors that rely on using stdin. #### `-W`, `--workers` When _Black_ formats multiple files, it may use a process pool to speed up formatting. -This option controls the number of parallel workers. +This option controls the number of parallel workers. This can also be specified via the +`BLACK_NUM_WORKERS` environment variable. #### `-q`, `--quiet` @@ -296,6 +297,19 @@ Read configuration options from a configuration file. See Show available command-line options and exit. +### Environment variable options + +_Black_ supports the following configuration via environment variables. + +#### `BLACK_CACHE_DIR` + +The directory where _Black_ should store its cache. + +#### `BLACK_NUM_WORKERS` + +The number of parallel workers _Black_ should use. The command line option `-W` / +`--workers` takes precedence over this environment variable. + ### Code input alternatives _Black_ supports formatting code via stdin, with the result being printed to stdout. diff --git a/src/black/__init__.py b/src/black/__init__.py index 60a339cdd12..3451c86b508 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -376,7 +376,10 @@ def validate_regex( "--workers", type=click.IntRange(min=1), default=None, - help="Number of parallel workers [default: number of CPUs in the system]", + help=( + "Number of parallel workers [default: BLACK_NUM_WORKERS environment variable " + "or number of CPUs in the system]" + ), ) @click.option( "-q", diff --git a/src/black/concurrency.py b/src/black/concurrency.py index 1598f51e43f..893eba6675a 100644 --- a/src/black/concurrency.py +++ b/src/black/concurrency.py @@ -80,7 +80,8 @@ def reformat_many( executor: Executor if workers is None: - workers = os.cpu_count() or 1 + workers = int(os.environ.get("BLACK_NUM_WORKERS", 0)) + workers = workers or os.cpu_count() or 1 if sys.platform == "win32": # Work around https://bugs.python.org/issue26903 workers = min(workers, 60) From 93989e995da7fa22c26abf8cc805273940265f5b Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sat, 24 Jun 2023 17:27:47 -0700 Subject: [PATCH 035/279] Integrate verbose logging with get_sources (#3749) Currently the verbose logging for "Sources to be formatted" is a little suspect in that it is a completely different code path from `get_sources`. This can result in bugs like https://github.com/psf/black/pull/3216#issuecomment-1213557359 and generally limits the value of these logs. This does change the "when" of this log, but the colours help separate it from the even more verbose logs. --- CHANGES.md | 1 + src/black/__init__.py | 34 +++++++++++++--------------------- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9717867df4e..a9d4d9d63c9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -52,6 +52,7 @@ - Use aware UTC datetimes internally, avoids deprecation warning on Python 3.12 (#3728) +- Change verbose logging to exactly mirror _Black_'s logic for source discovery (#3749) ### _Blackd_ diff --git a/src/black/__init__.py b/src/black/__init__.py index 3451c86b508..222cb3ca03d 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -484,26 +484,6 @@ def main( # noqa: C901 fg="blue", ) - normalized = [ - ( - (source, source) - if source == "-" - else (normalize_path_maybe_ignore(Path(source), root), source) - ) - for source in src - ] - srcs_string = ", ".join( - [ - ( - f'"{_norm}"' - if _norm - else f'\033[31m"{source} (skipping - invalid)"\033[34m' - ) - for _norm, source in normalized - ] - ) - out(f"Sources to be formatted: {srcs_string}", fg="blue") - if config: config_source = ctx.get_parameter_source("config") user_level_config = str(find_user_pyproject_toml()) @@ -654,9 +634,15 @@ def get_sources( is_stdin = False if is_stdin or p.is_file(): - normalized_path = normalize_path_maybe_ignore(p, ctx.obj["root"], report) + normalized_path: Optional[str] = normalize_path_maybe_ignore( + p, ctx.obj["root"], report + ) if normalized_path is None: + if verbose: + out(f'Skipping invalid source: "{normalized_path}"', fg="red") continue + if verbose: + out(f'Found input source: "{normalized_path}"', fg="blue") normalized_path = "/" + normalized_path # Hard-exclude any files that matches the `--force-exclude` regex. @@ -679,6 +665,9 @@ def get_sources( sources.add(p) elif p.is_dir(): p = root / normalize_path_maybe_ignore(p, ctx.obj["root"], report) + if verbose: + out(f'Found input source directory: "{p}"', fg="blue") + if using_default_exclude: gitignore = { root: root_gitignore, @@ -699,9 +688,12 @@ def get_sources( ) ) elif s == "-": + if verbose: + out("Found input source stdin", fg="blue") sources.add(p) else: err(f"invalid path: {s}") + return sources From e1036119f264a846bb049fad8404df318bc2f455 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 25 Jun 2023 06:53:26 -0700 Subject: [PATCH 036/279] Check self format for the whole repo (#3750) `black .` is changing things in gallery and scripts for me --- .github/workflows/test.yml | 2 +- gallery/gallery.py | 6 ++---- scripts/make_width_table.py | 6 ++---- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3ca2a469147..608c58af2ee 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -103,4 +103,4 @@ jobs: python -m pip install -e ".[uvloop]" - name: Format ourselves - run: python -m black --check src/ + run: python -m black --check . diff --git a/gallery/gallery.py b/gallery/gallery.py index 38e52e34795..ba5d6f65fbe 100755 --- a/gallery/gallery.py +++ b/gallery/gallery.py @@ -243,11 +243,9 @@ def format_repos(repos: Tuple[Path, ...], options: Namespace) -> None: def main() -> None: - parser = ArgumentParser( - description="""Black Gallery is a script that + parser = ArgumentParser(description="""Black Gallery is a script that automates the process of applying different Black versions to a selected - PyPI package and seeing the results between versions.""" - ) + PyPI package and seeing the results between versions.""") group = parser.add_mutually_exclusive_group(required=True) group.add_argument("-p", "--pypi-package", help="PyPI package to download.") diff --git a/scripts/make_width_table.py b/scripts/make_width_table.py index 09aca9c34b5..89c202553d3 100644 --- a/scripts/make_width_table.py +++ b/scripts/make_width_table.py @@ -49,8 +49,7 @@ def make_width_table() -> Iterable[Tuple[int, int, int]]: def main() -> None: table_path = join(dirname(__file__), "..", "src", "black", "_width_table.py") with open(table_path, "w") as f: - f.write( - f"""# Generated by {basename(__file__)} + f.write(f"""# Generated by {basename(__file__)} # wcwidth {wcwidth.__version__} # Unicode {wcwidth.list_versions()[-1]} import sys @@ -62,8 +61,7 @@ def main() -> None: from typing import Final WIDTH_TABLE: Final[List[Tuple[int, int, int]]] = [ -""" - ) +""") for triple in make_width_table(): f.write(f" {triple!r},\n") f.write("]\n") From 31b3b6701d2cfae072900f9d45dc8f1737ab282b Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 26 Jun 2023 17:47:55 -0700 Subject: [PATCH 037/279] Decrease cost of ipynb code path when unneeded (#3748) IPython is a very expensive import, like, at least 300ms. I'd also venture that it's much more common than tokenize-rt, which is like 30ms. I work in a repo where I use black, have IPython installed and there happen to be a couple notebooks (that we don't want formatted). I know I can force exclude ipynb, but this change doesn't really have a cost. --- CHANGES.md | 2 ++ src/black/handle_ipynb_magics.py | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index a9d4d9d63c9..6fa0e4b7cc0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -47,6 +47,8 @@ +- Avoid importing `IPython` in a case where we wouldn't need it (#3748) + ### Output diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index 9e1af757c32..2b6b9209211 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -58,8 +58,13 @@ class Replacement: @lru_cache() def jupyter_dependencies_are_installed(*, verbose: bool, quiet: bool) -> bool: try: - import IPython # noqa:F401 + # isort: off + # tokenize_rt is less commonly installed than IPython + # and IPython is expensive to import import tokenize_rt # noqa:F401 + import IPython # noqa:F401 + + # isort: on except ModuleNotFoundError: if verbose or not quiet: msg = ( From 63481bb9264a8c3756577089414d222da9c7fab0 Mon Sep 17 00:00:00 2001 From: rdrll <13176405+rdrll@users.noreply.github.com> Date: Tue, 27 Jun 2023 07:23:39 -0700 Subject: [PATCH 038/279] Fix a magical comment caused internal error (#3740) `is_type_comment` now specifically deals with general type comments for a leaf. `is_type_ignore_comment` now handles type comments contains ignore annotation for a leaf `is_type_ignore_comment_string` used to determine if a string is an ignore type comment --- CHANGES.md | 2 + src/black/linegen.py | 10 ++++- src/black/lines.py | 5 ++- src/black/nodes.py | 23 +++++++++-- ...ine_consecutive_open_parentheses_ignore.py | 41 +++++++++++++++++++ 5 files changed, 73 insertions(+), 8 deletions(-) create mode 100644 tests/data/simple_cases/multiline_consecutive_open_parentheses_ignore.py diff --git a/CHANGES.md b/CHANGES.md index 6fa0e4b7cc0..2dfed8a0dc4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,6 +12,8 @@ - Fix a bug where an illegal trailing comma was added to return type annotations using PEP 604 unions (#3735) +- Fix a bug where multi-line open parenthesis magic comment like `type: ignore` were not + correctly parsed (#3740) ### Preview style diff --git a/src/black/linegen.py b/src/black/linegen.py index ad21307c311..5ef3bbd1705 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -49,6 +49,7 @@ is_stub_body, is_stub_suite, is_tuple_containing_walrus, + is_type_ignore_comment_string, is_vararg, is_walrus_assignment, is_yield, @@ -1399,8 +1400,13 @@ def maybe_make_parens_invisible_in_atom( if is_lpar_token(first) and is_rpar_token(last): middle = node.children[1] # make parentheses invisible - first.value = "" - last.value = "" + if ( + # If the prefix of `middle` includes a type comment with + # ignore annotation, then we do not remove the parentheses + not is_type_ignore_comment_string(middle.prefix.strip()) + ): + first.value = "" + last.value = "" maybe_make_parens_invisible_in_atom( middle, parent=parent, diff --git a/src/black/lines.py b/src/black/lines.py index daf0444d24e..ea8fe520756 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -28,6 +28,7 @@ is_multiline_string, is_one_sequence_between, is_type_comment, + is_type_ignore_comment, is_with_or_async_with_stmt, replace_child, syms, @@ -251,7 +252,7 @@ def contains_uncollapsable_type_comments(self) -> bool: for comment in comments: if is_type_comment(comment): if comment_seen or ( - not is_type_comment(comment, " ignore") + not is_type_ignore_comment(comment) and leaf_id not in ignored_ids ): return True @@ -288,7 +289,7 @@ def contains_unsplittable_type_ignore(self) -> bool: # line. for node in self.leaves[-2:]: for comment in self.comments.get(id(node), []): - if is_type_comment(comment, " ignore"): + if is_type_ignore_comment(comment): return True return False diff --git a/src/black/nodes.py b/src/black/nodes.py index 45070909df4..b019b0c6440 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -816,12 +816,27 @@ def is_async_stmt_or_funcdef(leaf: Leaf) -> bool: ) -def is_type_comment(leaf: Leaf, suffix: str = "") -> bool: - """Return True if the given leaf is a special comment. - Only returns true for type comments for now.""" +def is_type_comment(leaf: Leaf) -> bool: + """Return True if the given leaf is a type comment. This function should only + be used for general type comments (excluding ignore annotations, which should + use `is_type_ignore_comment`). Note that general type comments are no longer + used in modern version of Python, this function may be deprecated in the future.""" t = leaf.type v = leaf.value - return t in {token.COMMENT, STANDALONE_COMMENT} and v.startswith("# type:" + suffix) + return t in {token.COMMENT, STANDALONE_COMMENT} and v.startswith("# type:") + + +def is_type_ignore_comment(leaf: Leaf) -> bool: + """Return True if the given leaf is a type comment with ignore annotation.""" + t = leaf.type + v = leaf.value + return t in {token.COMMENT, STANDALONE_COMMENT} and is_type_ignore_comment_string(v) + + +def is_type_ignore_comment_string(value: str) -> bool: + """Return True if the given string match with type comment with + ignore annotation.""" + return value.startswith("# type: ignore") def wrap_in_parentheses(parent: Node, child: LN, *, visible: bool = True) -> None: diff --git a/tests/data/simple_cases/multiline_consecutive_open_parentheses_ignore.py b/tests/data/simple_cases/multiline_consecutive_open_parentheses_ignore.py new file mode 100644 index 00000000000..6ec8bb45408 --- /dev/null +++ b/tests/data/simple_cases/multiline_consecutive_open_parentheses_ignore.py @@ -0,0 +1,41 @@ +# This is a regression test. Issue #3737 + +a = ( # type: ignore + int( # type: ignore + int( # type: ignore + int( # type: ignore + 6 + ) + ) + ) +) + +b = ( + int( + 6 + ) +) + +print( "111") # type: ignore +print( "111" ) # type: ignore +print( "111" ) # type: ignore + + +# output + + +# This is a regression test. Issue #3737 + +a = ( # type: ignore + int( # type: ignore + int( # type: ignore + int(6) # type: ignore + ) + ) +) + +b = int(6) + +print("111") # type: ignore +print("111") # type: ignore +print("111") # type: ignore \ No newline at end of file From f01aaa63a0ff792b932205cbb539f70aca769d7d Mon Sep 17 00:00:00 2001 From: rdrll <13176405+rdrll@users.noreply.github.com> Date: Wed, 28 Jun 2023 13:45:56 -0700 Subject: [PATCH 039/279] Doc: Developer reference update (#3755) --- CHANGES.md | 3 + .../reference/reference_classes.rst | 161 +++++++++++++++++- .../reference/reference_exceptions.rst | 10 +- .../reference/reference_summary.rst | 7 +- src/black/handle_ipynb_magics.py | 3 +- src/black/trans.py | 95 ++++++----- 6 files changed, 220 insertions(+), 59 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2dfed8a0dc4..ba7947a3c7d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -76,6 +76,9 @@ +- Updated the _classes_ and _exceptions_ documentation in Developer reference to match + the latest ccode base. (#3755) + ## 23.3.0 ### Highlights diff --git a/docs/contributing/reference/reference_classes.rst b/docs/contributing/reference/reference_classes.rst index 3931e0e0072..29b25003af2 100644 --- a/docs/contributing/reference/reference_classes.rst +++ b/docs/contributing/reference/reference_classes.rst @@ -3,6 +3,9 @@ *Contents are subject to change.* +Black Classes +~~~~~~~~~~~~~~ + .. currentmodule:: black :class:`BracketTracker` @@ -18,6 +21,12 @@ :members: :special-members: __str__, __bool__ +:class:`RHSResult` +------------------------- + +.. autoclass:: black.lines.RHSResult + :members: + :class:`LinesBlock` ------------------------- @@ -43,6 +52,12 @@ .. autoclass:: black.comments.ProtoComment :members: +:class:`Mode` +--------------------- + +.. autoclass:: black.mode.Mode + :members: + :class:`Report` --------------- @@ -50,6 +65,20 @@ :members: :special-members: __str__ +:class:`Ok` +--------------- + +.. autoclass:: black.rusty.Ok + :show-inheritance: + :members: + +:class:`Err` +--------------- + +.. autoclass:: black.rusty.Err + :show-inheritance: + :members: + :class:`Visitor` ---------------- @@ -57,20 +86,115 @@ :show-inheritance: :members: -Enums -===== +:class:`StringTransformer` +---------------------------- -:class:`Changed` ----------------- +.. autoclass:: black.trans.StringTransformer + :show-inheritance: + :members: -.. autoclass:: black.Changed +:class:`CustomSplit` +---------------------------- + +.. autoclass:: black.trans.CustomSplit + :members: + +:class:`CustomSplitMapMixin` +----------------------------- + +.. autoclass:: black.trans.CustomSplitMapMixin :show-inheritance: :members: -:class:`Mode` ------------------ +:class:`StringMerger` +---------------------- -.. autoclass:: black.Mode +.. autoclass:: black.trans.StringMerger + :show-inheritance: + :members: + +:class:`StringParenStripper` +----------------------------- + +.. autoclass:: black.trans.StringParenStripper + :show-inheritance: + :members: + +:class:`BaseStringSplitter` +----------------------------- + +.. autoclass:: black.trans.BaseStringSplitter + :show-inheritance: + :members: + +:class:`StringSplitter` +----------------------------- + +.. autoclass:: black.trans.StringSplitter + :show-inheritance: + :members: + +:class:`StringParenWrapper` +----------------------------- + +.. autoclass:: black.trans.StringParenWrapper + :show-inheritance: + :members: + +:class:`StringParser` +----------------------------- + +.. autoclass:: black.trans.StringParser + :members: + +:class:`DebugVisitor` +------------------------ + +.. autoclass:: black.debug.DebugVisitor + :show-inheritance: + :members: + +:class:`Replacement` +------------------------ + +.. autoclass:: black.handle_ipynb_magics.Replacement + :members: + +:class:`CellMagic` +------------------------ + +.. autoclass:: black.handle_ipynb_magics.CellMagic + :members: + +:class:`CellMagicFinder` +------------------------ + +.. autoclass:: black.handle_ipynb_magics.CellMagicFinder + :show-inheritance: + :members: + +:class:`OffsetAndMagic` +------------------------ + +.. autoclass:: black.handle_ipynb_magics.OffsetAndMagic + :members: + +:class:`MagicFinder` +------------------------ + +.. autoclass:: black.handle_ipynb_magics.MagicFinder + :show-inheritance: + :members: + +Enum Classes +~~~~~~~~~~~~~ + +Classes inherited from Python `Enum `_ class. + +:class:`Changed` +---------------- + +.. autoclass:: black.report.Changed :show-inheritance: :members: @@ -80,3 +204,24 @@ Enums .. autoclass:: black.WriteBack :show-inheritance: :members: + +:class:`TargetVersion` +---------------------- + +.. autoclass:: black.mode.TargetVersion + :show-inheritance: + :members: + +:class:`Feature` +------------------ + +.. autoclass:: black.mode.Feature + :show-inheritance: + :members: + +:class:`Preview` +------------------ + +.. autoclass:: black.mode.Preview + :show-inheritance: + :members: diff --git a/docs/contributing/reference/reference_exceptions.rst b/docs/contributing/reference/reference_exceptions.rst index aafe61e5017..ab46ebdb628 100644 --- a/docs/contributing/reference/reference_exceptions.rst +++ b/docs/contributing/reference/reference_exceptions.rst @@ -5,8 +5,14 @@ .. currentmodule:: black +.. autoexception:: black.trans.CannotTransform + .. autoexception:: black.linegen.CannotSplit -.. autoexception:: black.NothingChanged +.. autoexception:: black.brackets.BracketMatchError + +.. autoexception:: black.report.NothingChanged + +.. autoexception:: black.parsing.InvalidInput -.. autoexception:: black.InvalidInput +.. autoexception:: black.mode.Deprecated diff --git a/docs/contributing/reference/reference_summary.rst b/docs/contributing/reference/reference_summary.rst index f6ff4681557..c6163d897b6 100644 --- a/docs/contributing/reference/reference_summary.rst +++ b/docs/contributing/reference/reference_summary.rst @@ -3,8 +3,11 @@ Developer reference .. note:: - The documentation here is quite outdated and has been neglected. Many objects worthy - of inclusion aren't documented. Contributions are appreciated! + As of June 2023, the documentation of *Black classes* and *Black exceptions* + has been updated to the latest available version. + + The documentation of *Black functions* is quite outdated and has been neglected. Many + functions worthy of inclusion aren't documented. Contributions are appreciated! *Contents are subject to change.* diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index 2b6b9209211..4324819beaf 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -335,7 +335,8 @@ class CellMagicFinder(ast.NodeVisitor): For example, - %%time\nfoo() + %%time\n + foo() would have been transformed to diff --git a/src/black/trans.py b/src/black/trans.py index 1e28ed0656e..4d40cb4bdf6 100644 --- a/src/black/trans.py +++ b/src/black/trans.py @@ -205,11 +205,11 @@ def do_match(self, line: Line) -> TMatchResult: """ Returns: * Ok(string_indices) such that for each index, `line.leaves[index]` - is our target string if a match was able to be made. For - transformers that don't result in more lines (e.g. StringMerger, - StringParenStripper), multiple matches and transforms are done at - once to reduce the complexity. - OR + is our target string if a match was able to be made. For + transformers that don't result in more lines (e.g. StringMerger, + StringParenStripper), multiple matches and transforms are done at + once to reduce the complexity. + OR * Err(CannotTransform), if no match could be made. """ @@ -220,12 +220,12 @@ def do_transform( """ Yields: * Ok(new_line) where new_line is the new transformed line. - OR + OR * Err(CannotTransform) if the transformation failed for some reason. The - `do_match(...)` template method should usually be used to reject - the form of the given Line, but in some cases it is difficult to - know whether or not a Line meets the StringTransformer's - requirements until the transformation is already midway. + `do_match(...)` template method should usually be used to reject + the form of the given Line, but in some cases it is difficult to + know whether or not a Line meets the StringTransformer's + requirements until the transformation is already midway. Side Effects: This method should NOT mutate @line directly, but it MAY mutate the @@ -335,8 +335,8 @@ def pop_custom_splits(self, string: str) -> List[CustomSplit]: Returns: * A list of the custom splits that are mapped to @string, if any - exist. - OR + exist. + OR * [], otherwise. Side Effects: @@ -365,14 +365,14 @@ class StringMerger(StringTransformer, CustomSplitMapMixin): Requirements: (A) The line contains adjacent strings such that ALL of the validation checks listed in StringMerger._validate_msg(...)'s docstring pass. - OR + OR (B) The line contains a string which uses line continuation backslashes. Transformations: Depending on which of the two requirements above where met, either: (A) The string group associated with the target string is merged. - OR + OR (B) All line-continuation backslashes are removed from the target string. Collaborations: @@ -965,17 +965,20 @@ class BaseStringSplitter(StringTransformer): Requirements: * The target string value is responsible for the line going over the - line length limit. It follows that after all of black's other line - split methods have been exhausted, this line (or one of the resulting - lines after all line splits are performed) would still be over the - line_length limit unless we split this string. - AND + line length limit. It follows that after all of black's other line + split methods have been exhausted, this line (or one of the resulting + lines after all line splits are performed) would still be over the + line_length limit unless we split this string. + AND + * The target string is NOT a "pointless" string (i.e. a string that has - no parent or siblings). - AND + no parent or siblings). + AND + * The target string is not followed by an inline comment that appears - to be a pragma. - AND + to be a pragma. + AND + * The target string is not a multiline (i.e. triple-quote) string. """ @@ -1027,7 +1030,7 @@ def _validate(self, line: Line, string_idx: int) -> TResult[None]: Returns: * Ok(None), if ALL of the requirements are met. - OR + OR * Err(CannotTransform), if ANY of the requirements are NOT met. """ LL = line.leaves @@ -1299,9 +1302,9 @@ class StringSplitter(BaseStringSplitter, CustomSplitMapMixin): Requirements: * The line consists ONLY of a single string (possibly prefixed by a - string operator [e.g. '+' or '==']), MAYBE a string trailer, and MAYBE - a trailing comma. - AND + string operator [e.g. '+' or '==']), MAYBE a string trailer, and MAYBE + a trailing comma. + AND * All of the requirements listed in BaseStringSplitter's docstring. Transformations: @@ -1808,26 +1811,26 @@ class StringParenWrapper(BaseStringSplitter, CustomSplitMapMixin): addition to the requirements listed below: * The line is a return/yield statement, which returns/yields a string. - OR + OR * The line is part of a ternary expression (e.g. `x = y if cond else - z`) such that the line starts with `else `, where is - some string. - OR + z`) such that the line starts with `else `, where is + some string. + OR * The line is an assert statement, which ends with a string. - OR + OR * The line is an assignment statement (e.g. `x = ` or `x += - `) such that the variable is being assigned the value of some - string. - OR + `) such that the variable is being assigned the value of some + string. + OR * The line is a dictionary key assignment where some valid key is being - assigned the value of some string. - OR + assigned the value of some string. + OR * The line is an lambda expression and the value is a string. - OR + OR * The line starts with an "atom" string that prefers to be wrapped in - parens. It's preferred to be wrapped when it's is an immediate child of - a list/set/tuple literal, AND the string is surrounded by commas (or is - the first/last child). + parens. It's preferred to be wrapped when it's is an immediate child of + a list/set/tuple literal, AND the string is surrounded by commas (or is + the first/last child). Transformations: The chosen string is wrapped in parentheses and then split at the LPAR. @@ -2273,7 +2276,7 @@ def parse(self, leaves: List[Leaf], string_idx: int) -> int: Returns: The index directly after the last leaf which is apart of the string trailer, if a "trailer" exists. - OR + OR @string_idx + 1, if no string "trailer" exists. """ assert leaves[string_idx].type == token.STRING @@ -2287,11 +2290,11 @@ def _next_state(self, leaf: Leaf) -> bool: """ Pre-conditions: * On the first call to this function, @leaf MUST be the leaf that - was directly after the string leaf in question (e.g. if our target - string is `line.leaves[i]` then the first call to this method must - be `line.leaves[i + 1]`). + was directly after the string leaf in question (e.g. if our target + string is `line.leaves[i]` then the first call to this method must + be `line.leaves[i + 1]`). * On the next call to this function, the leaf parameter passed in - MUST be the leaf directly following @leaf. + MUST be the leaf directly following @leaf. Returns: True iff @leaf is apart of the string's trailer. From 839ef35dc1d72bb6eceac9fa809f095e2edcd12b Mon Sep 17 00:00:00 2001 From: rdrll <13176405+rdrll@users.noreply.github.com> Date: Fri, 30 Jun 2023 07:07:42 -0700 Subject: [PATCH 040/279] CI Test: Deprecating 'set-output' command (#3757) --- CHANGES.md | 2 ++ scripts/diff_shades_gha_helper.py | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index ba7947a3c7d..d7928a30c5c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -70,6 +70,8 @@ - Update GitHub Action to display black output in the job summary (#3688) +- Deprecated `set-output` command in CI test to keep up to date with GitHub's + deprecation announcement (#3757) ### Documentation diff --git a/scripts/diff_shades_gha_helper.py b/scripts/diff_shades_gha_helper.py index b5fea5a817d..994fbe05045 100644 --- a/scripts/diff_shades_gha_helper.py +++ b/scripts/diff_shades_gha_helper.py @@ -52,7 +52,13 @@ def set_output(name: str, value: str) -> None: print(f"[INFO]: setting '{name}' to '{value}'") else: print(f"[INFO]: setting '{name}' to [{len(value)} chars]") - print(f"::set-output name={name}::{value}") + + # Originally the `set-output` workflow command was used here, now replaced + # by setting variables through the `GITHUB_OUTPUT` environment variable + # to stay up to date with GitHub's update. + if "GITHUB_OUTPUT" in os.environ: + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + print(f"{name}={value}", file=f) def http_get(url: str, *, is_json: bool = True, **kwargs: Any) -> Any: From 8e618f386995fa89434834e6a793a1057e58112a Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Tue, 4 Jul 2023 16:38:39 -0700 Subject: [PATCH 041/279] Enable `PYTHONWARNDEFAULTENCODING = 1` in CI (#3763) --- CHANGES.md | 3 ++ tests/test_black.py | 87 +++++++++++++++++------------------------- tests/test_ipynb.py | 9 ++--- tests/test_no_ipynb.py | 3 +- tox.ini | 4 +- 5 files changed, 44 insertions(+), 62 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d7928a30c5c..acb5a822674 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -69,6 +69,9 @@ +- Black is now tested with + [`PYTHONWARNDEFAULTENCODING = 1`](https://docs.python.org/3/library/io.html#io-encoding-warning) + (#3763) - Update GitHub Action to display black output in the job summary (#3688) - Deprecated `set-output` command in CI test to keep up to date with GitHub's deprecation announcement (#3757) diff --git a/tests/test_black.py b/tests/test_black.py index abb304a246d..dee6ead50d0 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -149,8 +149,7 @@ def test_empty_ff(self) -> None: tmp_file = Path(black.dump_to_file()) try: self.assertFalse(ff(tmp_file, write_back=black.WriteBack.YES)) - with open(tmp_file, encoding="utf8") as f: - actual = f.read() + actual = tmp_file.read_text(encoding="utf-8") finally: os.unlink(tmp_file) self.assertFormatEqual(expected, actual) @@ -178,7 +177,7 @@ def test_one_empty_line_ff(self) -> None: ff(tmp_file, mode=mode, write_back=black.WriteBack.YES) ) with open(tmp_file, "rb") as f: - actual = f.read().decode("utf8") + actual = f.read().decode("utf-8") finally: os.unlink(tmp_file) self.assertFormatEqual(expected, actual) @@ -198,7 +197,7 @@ def test_piping(self) -> None: f"--line-length={black.DEFAULT_LINE_LENGTH}", f"--config={EMPTY_CONFIG}", ], - input=BytesIO(source.encode("utf8")), + input=BytesIO(source.encode("utf-8")), ) self.assertEqual(result.exit_code, 0) self.assertFormatEqual(expected, result.output) @@ -221,7 +220,7 @@ def test_piping_diff(self) -> None: f"--config={EMPTY_CONFIG}", ] result = BlackRunner().invoke( - black.main, args, input=BytesIO(source.encode("utf8")) + black.main, args, input=BytesIO(source.encode("utf-8")) ) self.assertEqual(result.exit_code, 0) actual = diff_header.sub(DETERMINISTIC_HEADER, result.output) @@ -239,7 +238,7 @@ def test_piping_diff_with_color(self) -> None: f"--config={EMPTY_CONFIG}", ] result = BlackRunner().invoke( - black.main, args, input=BytesIO(source.encode("utf8")) + black.main, args, input=BytesIO(source.encode("utf-8")) ) actual = result.output # Again, the contents are checked in a different test, so only look for colors. @@ -286,8 +285,7 @@ def test_expression_ff(self) -> None: tmp_file = Path(black.dump_to_file(source)) try: self.assertTrue(ff(tmp_file, write_back=black.WriteBack.YES)) - with open(tmp_file, encoding="utf8") as f: - actual = f.read() + actual = tmp_file.read_text(encoding="utf-8") finally: os.unlink(tmp_file) self.assertFormatEqual(expected, actual) @@ -390,8 +388,7 @@ def test_skip_source_first_line(self) -> None: black.main, [str(tmp_file), "-x", f"--config={EMPTY_CONFIG}"] ) self.assertEqual(result.exit_code, 0) - with open(tmp_file, encoding="utf8") as f: - actual = f.read() + actual = tmp_file.read_text(encoding="utf-8") self.assertFormatEqual(source, actual) def test_skip_source_first_line_when_mixing_newlines(self) -> None: @@ -1081,7 +1078,7 @@ def test_works_in_mono_process_only_environment(self) -> None: (workspace / "one.py").resolve(), (workspace / "two.py").resolve(), ]: - f.write_text('print("hello")\n') + f.write_text('print("hello")\n', encoding="utf-8") self.invokeBlack([str(workspace)]) @event_loop() @@ -1118,11 +1115,9 @@ def test_single_file_force_pyi(self) -> None: contents, expected = read_data("miscellaneous", "force_pyi") with cache_dir() as workspace: path = (workspace / "file.py").resolve() - with open(path, "w") as fh: - fh.write(contents) + path.write_text(contents, encoding="utf-8") self.invokeBlack([str(path), "--pyi"]) - with open(path, "r") as fh: - actual = fh.read() + actual = path.read_text(encoding="utf-8") # verify cache with --pyi is separate pyi_cache = black.read_cache(pyi_mode) self.assertIn(str(path), pyi_cache) @@ -1143,12 +1138,10 @@ def test_multi_file_force_pyi(self) -> None: (workspace / "file2.py").resolve(), ] for path in paths: - with open(path, "w") as fh: - fh.write(contents) + path.write_text(contents, encoding="utf-8") self.invokeBlack([str(p) for p in paths] + ["--pyi"]) for path in paths: - with open(path, "r") as fh: - actual = fh.read() + actual = path.read_text(encoding="utf-8") self.assertEqual(actual, expected) # verify cache with --pyi is separate pyi_cache = black.read_cache(pyi_mode) @@ -1160,7 +1153,7 @@ def test_multi_file_force_pyi(self) -> None: def test_pipe_force_pyi(self) -> None: source, expected = read_data("miscellaneous", "force_pyi") result = CliRunner().invoke( - black.main, ["-", "-q", "--pyi"], input=BytesIO(source.encode("utf8")) + black.main, ["-", "-q", "--pyi"], input=BytesIO(source.encode("utf-8")) ) self.assertEqual(result.exit_code, 0) actual = result.output @@ -1172,11 +1165,9 @@ def test_single_file_force_py36(self) -> None: source, expected = read_data("miscellaneous", "force_py36") with cache_dir() as workspace: path = (workspace / "file.py").resolve() - with open(path, "w") as fh: - fh.write(source) + path.write_text(source, encoding="utf-8") self.invokeBlack([str(path), *PY36_ARGS]) - with open(path, "r") as fh: - actual = fh.read() + actual = path.read_text(encoding="utf-8") # verify cache with --target-version is separate py36_cache = black.read_cache(py36_mode) self.assertIn(str(path), py36_cache) @@ -1195,12 +1186,10 @@ def test_multi_file_force_py36(self) -> None: (workspace / "file2.py").resolve(), ] for path in paths: - with open(path, "w") as fh: - fh.write(source) + path.write_text(source, encoding="utf-8") self.invokeBlack([str(p) for p in paths] + PY36_ARGS) for path in paths: - with open(path, "r") as fh: - actual = fh.read() + actual = path.read_text(encoding="utf-8") self.assertEqual(actual, expected) # verify cache with --target-version is separate pyi_cache = black.read_cache(py36_mode) @@ -1214,7 +1203,7 @@ def test_pipe_force_py36(self) -> None: result = CliRunner().invoke( black.main, ["-", "-q", "--target-version=py36"], - input=BytesIO(source.encode("utf8")), + input=BytesIO(source.encode("utf-8")), ) self.assertEqual(result.exit_code, 0) actual = result.output @@ -1443,11 +1432,11 @@ def test_preserves_line_endings_via_stdin(self) -> None: contents = nl.join(["def f( ):", " pass"]) runner = BlackRunner() result = runner.invoke( - black.main, ["-", "--fast"], input=BytesIO(contents.encode("utf8")) + black.main, ["-", "--fast"], input=BytesIO(contents.encode("utf-8")) ) self.assertEqual(result.exit_code, 0) output = result.stdout_bytes - self.assertIn(nl.encode("utf8"), output) + self.assertIn(nl.encode("utf-8"), output) if nl == "\n": self.assertNotIn(b"\r\n", output) @@ -1631,8 +1620,8 @@ def test_read_pyproject_toml_from_stdin(self) -> None: src_pyproject = src_dir / "pyproject.toml" src_pyproject.touch() - test_toml_file = THIS_DIR / "test.toml" - src_pyproject.write_text(test_toml_file.read_text()) + test_toml_content = (THIS_DIR / "test.toml").read_text(encoding="utf-8") + src_pyproject.write_text(test_toml_content, encoding="utf-8") src_python = src_dir / "foo.py" src_python.touch() @@ -1985,10 +1974,10 @@ def test_cache_broken_file(self) -> None: mode = DEFAULT_MODE with cache_dir() as workspace: cache_file = get_cache_file(mode) - cache_file.write_text("this is not a pickle") + cache_file.write_text("this is not a pickle", encoding="utf-8") assert black.read_cache(mode) == {} src = (workspace / "test.py").resolve() - src.write_text("print('hello')") + src.write_text("print('hello')", encoding="utf-8") invokeBlack([str(src)]) cache = black.read_cache(mode) assert str(src) in cache @@ -1997,10 +1986,10 @@ def test_cache_single_file_already_cached(self) -> None: mode = DEFAULT_MODE with cache_dir() as workspace: src = (workspace / "test.py").resolve() - src.write_text("print('hello')") + src.write_text("print('hello')", encoding="utf-8") black.write_cache({}, [src], mode) invokeBlack([str(src)]) - assert src.read_text() == "print('hello')" + assert src.read_text(encoding="utf-8") == "print('hello')" @event_loop() def test_cache_multiple_files(self) -> None: @@ -2009,17 +1998,13 @@ def test_cache_multiple_files(self) -> None: "concurrent.futures.ProcessPoolExecutor", new=ThreadPoolExecutor ): one = (workspace / "one.py").resolve() - with one.open("w") as fobj: - fobj.write("print('hello')") + one.write_text("print('hello')", encoding="utf-8") two = (workspace / "two.py").resolve() - with two.open("w") as fobj: - fobj.write("print('hello')") + two.write_text("print('hello')", encoding="utf-8") black.write_cache({}, [one], mode) invokeBlack([str(workspace)]) - with one.open("r") as fobj: - assert fobj.read() == "print('hello')" - with two.open("r") as fobj: - assert fobj.read() == 'print("hello")\n' + assert one.read_text(encoding="utf-8") == "print('hello')" + assert two.read_text(encoding="utf-8") == 'print("hello")\n' cache = black.read_cache(mode) assert str(one) in cache assert str(two) in cache @@ -2029,8 +2014,7 @@ def test_no_cache_when_writeback_diff(self, color: bool) -> None: mode = DEFAULT_MODE with cache_dir() as workspace: src = (workspace / "test.py").resolve() - with src.open("w") as fobj: - fobj.write("print('hello')") + src.write_text("print('hello')", encoding="utf-8") with patch("black.read_cache") as read_cache, patch( "black.write_cache" ) as write_cache: @@ -2049,8 +2033,7 @@ def test_output_locking_when_writeback_diff(self, color: bool) -> None: with cache_dir() as workspace: for tag in range(0, 4): src = (workspace / f"test{tag}.py").resolve() - with src.open("w") as fobj: - fobj.write("print('hello')") + src.write_text("print('hello')", encoding="utf-8") with patch( "black.concurrency.Manager", wraps=multiprocessing.Manager ) as mgr: @@ -2120,11 +2103,9 @@ def test_failed_formatting_does_not_get_cached(self) -> None: "concurrent.futures.ProcessPoolExecutor", new=ThreadPoolExecutor ): failing = (workspace / "failing.py").resolve() - with failing.open("w") as fobj: - fobj.write("not actually python") + failing.write_text("not actually python", encoding="utf-8") clean = (workspace / "clean.py").resolve() - with clean.open("w") as fobj: - fobj.write('print("hello")\n') + clean.write_text('print("hello")\n', encoding="utf-8") invokeBlack([str(workspace)], exit_code=123) cache = black.read_cache(mode) assert str(failing) not in cache diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 7aa2e91dd00..91e7901125b 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -439,8 +439,7 @@ def test_cache_isnt_written_if_no_jupyter_deps_single( jupyter_dependencies_are_installed.cache_clear() nb = get_case_path("jupyter", "notebook_trailing_newline.ipynb") tmp_nb = tmp_path / "notebook.ipynb" - with open(nb) as src, open(tmp_nb, "w") as dst: - dst.write(src.read()) + tmp_nb.write_bytes(nb.read_bytes()) monkeypatch.setattr( "black.jupyter_dependencies_are_installed", lambda verbose, quiet: False ) @@ -465,8 +464,7 @@ def test_cache_isnt_written_if_no_jupyter_deps_dir( jupyter_dependencies_are_installed.cache_clear() nb = get_case_path("jupyter", "notebook_trailing_newline.ipynb") tmp_nb = tmp_path / "notebook.ipynb" - with open(nb) as src, open(tmp_nb, "w") as dst: - dst.write(src.read()) + tmp_nb.write_bytes(nb.read_bytes()) monkeypatch.setattr( "black.files.jupyter_dependencies_are_installed", lambda verbose, quiet: False ) @@ -483,8 +481,7 @@ def test_cache_isnt_written_if_no_jupyter_deps_dir( def test_ipynb_flag(tmp_path: pathlib.Path) -> None: nb = get_case_path("jupyter", "notebook_trailing_newline.ipynb") tmp_nb = tmp_path / "notebook.a_file_extension_which_is_definitely_not_ipynb" - with open(nb) as src, open(tmp_nb, "w") as dst: - dst.write(src.read()) + tmp_nb.write_bytes(nb.read_bytes()) result = runner.invoke( main, [ diff --git a/tests/test_no_ipynb.py b/tests/test_no_ipynb.py index b63ecde8896..12c820def39 100644 --- a/tests/test_no_ipynb.py +++ b/tests/test_no_ipynb.py @@ -27,8 +27,7 @@ def test_ipynb_diff_with_no_change_dir(tmp_path: pathlib.Path) -> None: runner = CliRunner() nb = get_case_path("jupyter", "notebook_trailing_newline.ipynb") tmp_nb = tmp_path / "notebook.ipynb" - with open(nb) as src, open(tmp_nb, "w") as dst: - dst.write(src.read()) + tmp_nb.write_bytes(nb.read_bytes()) result = runner.invoke(main, [str(tmp_path)]) expected_output = ( "Skipping .ipynb files as Jupyter dependencies are not installed.\n" diff --git a/tox.ini b/tox.ini index 4934514264b..f8e1a785331 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,9 @@ isolated_build = true envlist = {,ci-}py{37,38,39,310,311,py3},fuzz,run_self [testenv] -setenv = PYTHONPATH = {toxinidir}/src +setenv = + PYTHONPATH = {toxinidir}/src + PYTHONWARNDEFAULTENCODING = 1 skip_install = True # We use `recreate=True` because otherwise, on the second run of `tox -e py`, # the `no_jupyter` tests would run with the jupyter extra dependencies installed. From cf4cc2981900565ab931aada176abf08a1f5782d Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Tue, 4 Jul 2023 22:45:57 -0700 Subject: [PATCH 042/279] Better error message for invalid exclude types (#3764) --- CHANGES.md | 2 ++ src/black/__init__.py | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index acb5a822674..bfa021617cb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -32,6 +32,8 @@ - `.pytest_cache`, `.ruff_cache` and `.vscode` are now excluded by default (#3691) - Fix black not honouring `pyproject.toml` settings when running `--stdin-filename` and the `pyproject.toml` found isn't in the current working directory (#3719) +- Black will now error if `exclude` and `extend-exclude` have invalid data types in + `pyproject.toml`, instead of silently doing the wrong thing (#3764) ### Packaging diff --git a/src/black/__init__.py b/src/black/__init__.py index 222cb3ca03d..b6611bef84b 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -157,6 +157,16 @@ def read_pyproject_toml( "target-version", "Config key target-version must be a list" ) + exclude = config.get("exclude") + if exclude is not None and not isinstance(exclude, str): + raise click.BadOptionUsage("exclude", "Config key exclude must be a string") + + extend_exclude = config.get("extend_exclude") + if extend_exclude is not None and not isinstance(extend_exclude, str): + raise click.BadOptionUsage( + "extend-exclude", "Config key extend-exclude must be a string" + ) + default_map: Dict[str, Any] = {} if ctx.default_map: default_map.update(ctx.default_map) From b4dca26c7d93f930bbd5a7b552807370b60d4298 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Wed, 5 Jul 2023 10:08:04 -0700 Subject: [PATCH 043/279] Drop support for Python 3.7 (#3765) --- .github/workflows/fuzz.yml | 2 +- .github/workflows/test.yml | 2 +- CHANGES.md | 3 + docs/integrations/editors.md | 54 ---------------- mypy.ini | 2 +- pyproject.toml | 10 +-- scripts/make_width_table.py | 8 +-- src/black/_width_table.py | 8 +-- src/black/brackets.py | 8 +-- src/black/comments.py | 8 +-- src/black/mode.py | 8 +-- src/black/nodes.py | 6 +- src/black/parsing.py | 109 +++++++-------------------------- src/black/strings.py | 10 +-- src/black/trans.py | 8 +-- src/blib2to3/pgen2/token.py | 5 +- src/blib2to3/pgen2/tokenize.py | 5 +- 17 files changed, 43 insertions(+), 213 deletions(-) diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 373e1500ee9..4439148a1c7 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 608c58af2ee..92d7d411510 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,7 +31,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "pypy-3.7", "pypy-3.8"] + python-version: ["3.8", "3.9", "3.10", "3.11", "pypy-3.8"] os: [ubuntu-latest, macOS-latest, windows-latest] steps: diff --git a/CHANGES.md b/CHANGES.md index bfa021617cb..24ca54a82ac 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,9 @@ +- Runtime support for Python 3.7 has been removed. Formatting 3.7 code will still be + supported until further notice (#3765) + ### Stable style diff --git a/docs/integrations/editors.md b/docs/integrations/editors.md index 74c6a283ab8..ff563068e79 100644 --- a/docs/integrations/editors.md +++ b/docs/integrations/editors.md @@ -334,60 +334,6 @@ To run _Black_ on a key press (e.g. F9 below), add this: nnoremap :Black ``` -#### Troubleshooting - -**How to get Vim with Python 3.6?** On Ubuntu 17.10 Vim comes with Python 3.6 by -default. On macOS with Homebrew run: `brew install vim`. When building Vim from source, -use: `./configure --enable-python3interp=yes`. There's many guides online how to do -this. - -**I get an import error when using _Black_ from a virtual environment**: If you get an -error message like this: - -```text -Traceback (most recent call last): - File "", line 63, in - File "/home/gui/.vim/black/lib/python3.7/site-packages/black.py", line 45, in - from typed_ast import ast3, ast27 - File "/home/gui/.vim/black/lib/python3.7/site-packages/typed_ast/ast3.py", line 40, in - from typed_ast import _ast3 -ImportError: /home/gui/.vim/black/lib/python3.7/site-packages/typed_ast/_ast3.cpython-37m-x86_64-linux-gnu.so: undefined symbool: PyExc_KeyboardInterrupt -``` - -Then you need to install `typed_ast` directly from the source code. The error happens -because `pip` will download [Python wheels](https://pythonwheels.com/) if they are -available. Python wheels are a new standard of distributing Python packages and packages -that have Cython and extensions written in C are already compiled, so the installation -is much more faster. The problem here is that somehow the Python environment inside Vim -does not match with those already compiled C extensions and these kind of errors are the -result. Luckily there is an easy fix: installing the packages from the source code. - -The package that causes problems is: - -- [typed-ast](https://pypi.org/project/typed-ast/) - -Now remove those two packages: - -```console -$ pip uninstall typed-ast -y -``` - -And now you can install them with: - -```console -$ pip install --no-binary :all: typed-ast -``` - -The C extensions will be compiled and now Vim's Python environment will match. Note that -you need to have the GCC compiler and the Python development files installed (on -Ubuntu/Debian do `sudo apt-get install build-essential python3-dev`). - -If you later want to update _Black_, you should do it like this: - -```console -$ pip install -U black --no-binary typed-ast -``` - ### With ALE 1. Install [`ale`](https://github.com/dense-analysis/ale) diff --git a/mypy.ini b/mypy.ini index 58bb7536173..95ec22d65be 100644 --- a/mypy.ini +++ b/mypy.ini @@ -2,7 +2,7 @@ # Specify the target platform details in config, so your developers are # free to run mypy on Windows, Linux, or macOS and get consistent # results. -python_version=3.7 +python_version=3.8 mypy_path=src diff --git a/pyproject.toml b/pyproject.toml index d44623fdbd3..2d8da88f6c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ build-backend = "hatchling.build" name = "black" description = "The uncompromising code formatter." license = { text = "MIT" } -requires-python = ">=3.7" +requires-python = ">=3.8" authors = [ { name = "Łukasz Langa", email = "lukasz@langa.pl" }, ] @@ -69,7 +69,6 @@ dependencies = [ "pathspec>=0.9.0", "platformdirs>=2", "tomli>=1.1.0; python_version < '3.11'", - "typed-ast>=1.4.2; python_version < '3.8' and implementation_name == 'cpython'", "typing_extensions>=3.10.0.0; python_version < '3.10'", ] dynamic = ["readme", "version"] @@ -121,8 +120,6 @@ enable-by-default = false dependencies = [ "hatch-mypyc>=0.16.0", "mypy==1.3", - # Required stubs to be removed when the packages support PEP 561 themselves - "types-typed-ast>=1.4.2", ] require-runtime-dependencies = true exclude = [ @@ -145,7 +142,7 @@ options = { debug_level = "0" } [tool.cibuildwheel] build-verbosity = 1 # So these are the environments we target: -# - Python: CPython 3.7+ only +# - Python: CPython 3.8+ only # - Architecture (64-bit only): amd64 / x86_64, universal2, and arm64 # - OS: Linux (no musl), Windows, and macOS build = "cp3*-*" @@ -208,9 +205,6 @@ filterwarnings = [ # this is mitigated by a try/catch in https://github.com/psf/black/pull/3198/ # this ignore can be removed when support for aiohttp 3.x is dropped. '''ignore:Middleware decorator is deprecated since 4\.0 and its behaviour is default, you can simply remove this decorator:DeprecationWarning''', - # this is mitigated by https://github.com/python/cpython/issues/79071 in python 3.8+ - # this ignore can be removed when support for 3.7 is dropped. - '''ignore:Bare functions are deprecated, use async ones:DeprecationWarning''', # aiohttp is using deprecated cgi modules - Safe to remove when fixed: # https://github.com/aio-libs/aiohttp/issues/6905 '''ignore:'cgi' is deprecated and slated for removal in Python 3.13:DeprecationWarning''', diff --git a/scripts/make_width_table.py b/scripts/make_width_table.py index 89c202553d3..30fd32c34b0 100644 --- a/scripts/make_width_table.py +++ b/scripts/make_width_table.py @@ -52,13 +52,7 @@ def main() -> None: f.write(f"""# Generated by {basename(__file__)} # wcwidth {wcwidth.__version__} # Unicode {wcwidth.list_versions()[-1]} -import sys -from typing import List, Tuple - -if sys.version_info < (3, 8): - from typing_extensions import Final -else: - from typing import Final +from typing import Final, List, Tuple WIDTH_TABLE: Final[List[Tuple[int, int, int]]] = [ """) diff --git a/src/black/_width_table.py b/src/black/_width_table.py index 6923f597687..f3304e48ed0 100644 --- a/src/black/_width_table.py +++ b/src/black/_width_table.py @@ -1,13 +1,7 @@ # Generated by make_width_table.py # wcwidth 0.2.6 # Unicode 15.0.0 -import sys -from typing import List, Tuple - -if sys.version_info < (3, 8): - from typing_extensions import Final -else: - from typing import Final +from typing import Final, List, Tuple WIDTH_TABLE: Final[List[Tuple[int, int, int]]] = [ (0, 0, 0), diff --git a/src/black/brackets.py b/src/black/brackets.py index 343f0608d50..85dac6edd1e 100644 --- a/src/black/brackets.py +++ b/src/black/brackets.py @@ -1,13 +1,7 @@ """Builds on top of nodes.py to track brackets.""" -import sys from dataclasses import dataclass, field -from typing import Dict, Iterable, List, Optional, Sequence, Set, Tuple, Union - -if sys.version_info < (3, 8): - from typing_extensions import Final -else: - from typing import Final +from typing import Dict, Final, Iterable, List, Optional, Sequence, Set, Tuple, Union from black.nodes import ( BRACKET, diff --git a/src/black/comments.py b/src/black/comments.py index 619123ab4be..226968bff98 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -1,13 +1,7 @@ import re -import sys from dataclasses import dataclass from functools import lru_cache -from typing import Iterator, List, Optional, Union - -if sys.version_info >= (3, 8): - from typing import Final -else: - from typing_extensions import Final +from typing import Final, Iterator, List, Optional, Union from black.nodes import ( CLOSING_BRACKETS, diff --git a/src/black/mode.py b/src/black/mode.py index 1091494afac..4d979afd84d 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -4,19 +4,13 @@ chosen by the user. """ -import sys from dataclasses import dataclass, field from enum import Enum, auto from hashlib import sha256 from operator import attrgetter -from typing import Dict, Set +from typing import Dict, Final, Set from warnings import warn -if sys.version_info < (3, 8): - from typing_extensions import Final -else: - from typing import Final - from black.const import DEFAULT_LINE_LENGTH diff --git a/src/black/nodes.py b/src/black/nodes.py index b019b0c6440..ef42278d83f 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -3,12 +3,8 @@ """ import sys -from typing import Generic, Iterator, List, Optional, Set, Tuple, TypeVar, Union +from typing import Final, Generic, Iterator, List, Optional, Set, Tuple, TypeVar, Union -if sys.version_info >= (3, 8): - from typing import Final -else: - from typing_extensions import Final if sys.version_info >= (3, 10): from typing import TypeGuard else: diff --git a/src/black/parsing.py b/src/black/parsing.py index 70ed99c1549..455c5eed968 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -2,14 +2,8 @@ Parse Python code and perform AST validation. """ import ast -import platform import sys -from typing import Any, Iterable, Iterator, List, Set, Tuple, Type, Union - -if sys.version_info < (3, 8): - from typing_extensions import Final -else: - from typing import Final +from typing import Final, Iterable, Iterator, List, Set, Tuple from black.mode import VERSION_TO_FEATURES, Feature, TargetVersion, supports_feature from black.nodes import syms @@ -20,25 +14,6 @@ from blib2to3.pgen2.tokenize import TokenError from blib2to3.pytree import Leaf, Node -ast3: Any - -_IS_PYPY = platform.python_implementation() == "PyPy" - -try: - from typed_ast import ast3 -except ImportError: - if sys.version_info < (3, 8) and not _IS_PYPY: - print( - "The typed_ast package is required but not installed.\n" - "You can upgrade to Python 3.8+ or install typed_ast with\n" - "`python3 -m pip install typed-ast`.", - file=sys.stderr, - ) - sys.exit(1) - else: - ast3 = ast - - PY2_HINT: Final = "Python 2 support was removed in version 22.0." @@ -147,31 +122,14 @@ def lib2to3_unparse(node: Node) -> str: def parse_single_version( src: str, version: Tuple[int, int], *, type_comments: bool -) -> Union[ast.AST, ast3.AST]: +) -> ast.AST: filename = "" - # typed-ast is needed because of feature version limitations in the builtin ast 3.8> - if sys.version_info >= (3, 8) and version >= (3,): - return ast.parse( - src, filename, feature_version=version, type_comments=type_comments - ) - - if _IS_PYPY: - # PyPy 3.7 doesn't support type comment tracking which is not ideal, but there's - # not much we can do as typed-ast won't work either. - if sys.version_info >= (3, 8): - return ast3.parse(src, filename, type_comments=type_comments) - else: - return ast3.parse(src, filename) - else: - if type_comments: - # Typed-ast is guaranteed to be used here and automatically tracks type - # comments separately. - return ast3.parse(src, filename, feature_version=version[1]) - else: - return ast.parse(src, filename) + return ast.parse( + src, filename, feature_version=version, type_comments=type_comments + ) -def parse_ast(src: str) -> Union[ast.AST, ast3.AST]: +def parse_ast(src: str) -> ast.AST: # TODO: support Python 4+ ;) versions = [(3, minor) for minor in range(3, sys.version_info[1] + 1)] @@ -193,9 +151,6 @@ def parse_ast(src: str) -> Union[ast.AST, ast3.AST]: raise SyntaxError(first_error) -ast3_AST: Final[Type[ast3.AST]] = ast3.AST - - def _normalize(lineend: str, value: str) -> str: # To normalize, we strip any leading and trailing space from # each line... @@ -206,23 +161,25 @@ def _normalize(lineend: str, value: str) -> str: return normalized.strip() -def stringify_ast(node: Union[ast.AST, ast3.AST], depth: int = 0) -> Iterator[str]: +def stringify_ast(node: ast.AST, depth: int = 0) -> Iterator[str]: """Simple visitor generating strings to compare ASTs by content.""" - node = fixup_ast_constants(node) + if ( + isinstance(node, ast.Constant) + and isinstance(node.value, str) + and node.kind == "u" + ): + # It's a quirk of history that we strip the u prefix over here. We used to + # rewrite the AST nodes for Python version compatibility and we never copied + # over the kind + node.kind = None yield f"{' ' * depth}{node.__class__.__name__}(" - type_ignore_classes: Tuple[Type[Any], ...] for field in sorted(node._fields): # noqa: F402 - # TypeIgnore will not be present using pypy < 3.8, so need for this - if not (_IS_PYPY and sys.version_info < (3, 8)): - # TypeIgnore has only one field 'lineno' which breaks this comparison - type_ignore_classes = (ast3.TypeIgnore,) - if sys.version_info >= (3, 8): - type_ignore_classes += (ast.TypeIgnore,) - if isinstance(node, type_ignore_classes): - break + # TypeIgnore has only one field 'lineno' which breaks this comparison + if isinstance(node, ast.TypeIgnore): + break try: value: object = getattr(node, field) @@ -237,22 +194,16 @@ def stringify_ast(node: Union[ast.AST, ast3.AST], depth: int = 0) -> Iterator[st # parentheses and they change the AST. if ( field == "targets" - and isinstance(node, (ast.Delete, ast3.Delete)) - and isinstance(item, (ast.Tuple, ast3.Tuple)) + and isinstance(node, ast.Delete) + and isinstance(item, ast.Tuple) ): for elt in item.elts: yield from stringify_ast(elt, depth + 2) - elif isinstance(item, (ast.AST, ast3.AST)): + elif isinstance(item, ast.AST): yield from stringify_ast(item, depth + 2) - # Note that we are referencing the typed-ast ASTs via global variables and not - # direct module attribute accesses because that breaks mypyc. It's probably - # something to do with the ast3 variables being marked as Any leading - # mypy to think this branch is always taken, leaving the rest of the code - # unanalyzed. Tighting up the types for the typed-ast AST types avoids the - # mypyc crash. - elif isinstance(value, (ast.AST, ast3_AST)): + elif isinstance(value, ast.AST): yield from stringify_ast(value, depth + 2) else: @@ -271,17 +222,3 @@ def stringify_ast(node: Union[ast.AST, ast3.AST], depth: int = 0) -> Iterator[st yield f"{' ' * (depth+2)}{normalized!r}, # {value.__class__.__name__}" yield f"{' ' * depth}) # /{node.__class__.__name__}" - - -def fixup_ast_constants(node: Union[ast.AST, ast3.AST]) -> Union[ast.AST, ast3.AST]: - """Map ast nodes deprecated in 3.8 to Constant.""" - if isinstance(node, (ast.Str, ast3.Str, ast.Bytes, ast3.Bytes)): - return ast.Constant(value=node.s) - - if isinstance(node, (ast.Num, ast3.Num)): - return ast.Constant(value=node.n) - - if isinstance(node, (ast.NameConstant, ast3.NameConstant)): - return ast.Constant(value=node.value) - - return node diff --git a/src/black/strings.py b/src/black/strings.py index ac18aef51ed..0d30f09ed11 100644 --- a/src/black/strings.py +++ b/src/black/strings.py @@ -5,16 +5,10 @@ import re import sys from functools import lru_cache -from typing import List, Match, Pattern - -from blib2to3.pytree import Leaf - -if sys.version_info < (3, 8): - from typing_extensions import Final -else: - from typing import Final +from typing import Final, List, Match, Pattern from black._width_table import WIDTH_TABLE +from blib2to3.pytree import Leaf STRING_PREFIX_CHARS: Final = "furbFURB" # All possible string prefix characters. STRING_PREFIX_RE: Final = re.compile( diff --git a/src/black/trans.py b/src/black/trans.py index 4d40cb4bdf6..daed26427d7 100644 --- a/src/black/trans.py +++ b/src/black/trans.py @@ -2,7 +2,6 @@ String transformers that can split and merge strings. """ import re -import sys from abc import ABC, abstractmethod from collections import defaultdict from dataclasses import dataclass @@ -12,9 +11,11 @@ ClassVar, Collection, Dict, + Final, Iterable, Iterator, List, + Literal, Optional, Sequence, Set, @@ -23,11 +24,6 @@ Union, ) -if sys.version_info < (3, 8): - from typing_extensions import Final, Literal -else: - from typing import Literal, Final - from mypy_extensions import trait from black.comments import contains_pragma_comment diff --git a/src/blib2to3/pgen2/token.py b/src/blib2to3/pgen2/token.py index 1e0dec9c714..c939531d7c8 100644 --- a/src/blib2to3/pgen2/token.py +++ b/src/blib2to3/pgen2/token.py @@ -3,10 +3,7 @@ import sys from typing import Dict -if sys.version_info < (3, 8): - from typing_extensions import Final -else: - from typing import Final +from typing import Final # Taken from Python (r53757) and modified to include some tokens # originally monkeypatched in by pgen2.tokenize diff --git a/src/blib2to3/pgen2/tokenize.py b/src/blib2to3/pgen2/tokenize.py index 2d0cc4324ce..a5e89188d87 100644 --- a/src/blib2to3/pgen2/tokenize.py +++ b/src/blib2to3/pgen2/tokenize.py @@ -42,10 +42,7 @@ cast, ) -if sys.version_info >= (3, 8): - from typing import Final -else: - from typing_extensions import Final +from typing import Final from blib2to3.pgen2.token import * from blib2to3.pgen2.grammar import Grammar From 4130c65578b9ac2a42b3b18f4f38917607db3500 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 9 Jul 2023 08:14:38 -0700 Subject: [PATCH 044/279] Fix CI for Click typing issue (#3770) https://github.com/pallets/click/issues/2558 --- .github/workflows/diff_shades.yml | 4 ++-- .pre-commit-config.yaml | 2 +- pyproject.toml | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/diff_shades.yml b/.github/workflows/diff_shades.yml index a126756f102..d685ef9456d 100644 --- a/.github/workflows/diff_shades.yml +++ b/.github/workflows/diff_shades.yml @@ -26,7 +26,7 @@ jobs: - name: Install diff-shades and support dependencies run: | - python -m pip install click packaging urllib3 + python -m pip install 'click==8.1.3' packaging urllib3 python -m pip install https://github.com/ichard26/diff-shades/archive/stable.zip - name: Calculate run configuration & metadata @@ -64,7 +64,7 @@ jobs: - name: Install diff-shades and support dependencies run: | python -m pip install https://github.com/ichard26/diff-shades/archive/stable.zip - python -m pip install click packaging urllib3 + python -m pip install 'click==8.1.3' packaging urllib3 # After checking out old revisions, this might not exist so we'll use a copy. cat scripts/diff_shades_gha_helper.py > helper.py git config user.name "diff-shades-gha" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a69fb645238..bb647763e70 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,7 +47,7 @@ repos: - types-PyYAML - tomli >= 0.2.6, < 2.0.0 - types-typed-ast >= 1.4.1 - - click >= 8.1.0 + - click >= 8.1.0, != 8.1.4 - packaging >= 22.0 - platformdirs >= 2.1.0 - pytest diff --git a/pyproject.toml b/pyproject.toml index 2d8da88f6c6..175f7851dee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,6 +120,7 @@ enable-by-default = false dependencies = [ "hatch-mypyc>=0.16.0", "mypy==1.3", + "click==8.1.3", # avoid https://github.com/pallets/click/issues/2558 ] require-runtime-dependencies = true exclude = [ From 114e8357e65384e17baaa3c31aa528371e15679b Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 9 Jul 2023 13:29:47 -0700 Subject: [PATCH 045/279] Remove click patch (#3768) Apparently this was only needed on Python 3.6. We've now dropped support for 3.6 and 3.7. It's also not needed on new enough click. --- CHANGES.md | 1 + .../reference/reference_functions.rst | 2 -- src/black/__init__.py | 35 ------------------- src/blackd/__init__.py | 1 - tests/test_black.py | 24 ------------- 5 files changed, 1 insertion(+), 62 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 24ca54a82ac..1b0475fc7b9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -43,6 +43,7 @@ - Upgrade mypyc from 0.991 to 1.3 (#3697) +- Remove patching of Click that mitigated errors on Python 3.6 with `LANG=C` (#3768) ### Parser diff --git a/docs/contributing/reference/reference_functions.rst b/docs/contributing/reference/reference_functions.rst index 3bda5de1774..09517f73961 100644 --- a/docs/contributing/reference/reference_functions.rst +++ b/docs/contributing/reference/reference_functions.rst @@ -165,8 +165,6 @@ Utilities .. autofunction:: black.linegen.normalize_invisible_parens -.. autofunction:: black.patch_click - .. autofunction:: black.nodes.preceding_leaf .. autofunction:: black.re_compile_maybe_verbose diff --git a/src/black/__init__.py b/src/black/__init__.py index b6611bef84b..301c18f7338 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1410,40 +1410,6 @@ def nullcontext() -> Iterator[None]: yield -def patch_click() -> None: - """Make Click not crash on Python 3.6 with LANG=C. - - On certain misconfigured environments, Python 3 selects the ASCII encoding as the - default which restricts paths that it can access during the lifetime of the - application. Click refuses to work in this scenario by raising a RuntimeError. - - In case of Black the likelihood that non-ASCII characters are going to be used in - file paths is minimal since it's Python source code. Moreover, this crash was - spurious on Python 3.7 thanks to PEP 538 and PEP 540. - """ - modules: List[Any] = [] - try: - from click import core - except ImportError: - pass - else: - modules.append(core) - try: - # Removed in Click 8.1.0 and newer; we keep this around for users who have - # older versions installed. - from click import _unicodefun # type: ignore - except ImportError: - pass - else: - modules.append(_unicodefun) - - for module in modules: - if hasattr(module, "_verify_python3_env"): - module._verify_python3_env = lambda: None - if hasattr(module, "_verify_python_env"): - module._verify_python_env = lambda: None - - def patched_main() -> None: # PyInstaller patches multiprocessing to need freeze_support() even in non-Windows # environments so just assume we always need to call it if frozen. @@ -1452,7 +1418,6 @@ def patched_main() -> None: freeze_support() - patch_click() main() diff --git a/src/blackd/__init__.py b/src/blackd/__init__.py index c1b69feed63..4f2d87d0fca 100644 --- a/src/blackd/__init__.py +++ b/src/blackd/__init__.py @@ -225,7 +225,6 @@ def parse_python_variant_header(value: str) -> Tuple[bool, Set[black.TargetVersi def patched_main() -> None: maybe_install_uvloop() freeze_support() - black.patch_click() main() diff --git a/tests/test_black.py b/tests/test_black.py index dee6ead50d0..dd21d0a7ae6 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1455,30 +1455,6 @@ def test_assert_equivalent_different_asts(self) -> None: with self.assertRaises(AssertionError): black.assert_equivalent("{}", "None") - def test_shhh_click(self) -> None: - try: - from click import _unicodefun # type: ignore - except ImportError: - self.skipTest("Incompatible Click version") - - if not hasattr(_unicodefun, "_verify_python_env"): - self.skipTest("Incompatible Click version") - - # First, let's see if Click is crashing with a preferred ASCII charset. - with patch("locale.getpreferredencoding") as gpe: - gpe.return_value = "ASCII" - with self.assertRaises(RuntimeError): - _unicodefun._verify_python_env() - # Now, let's silence Click... - black.patch_click() - # ...and confirm it's silent. - with patch("locale.getpreferredencoding") as gpe: - gpe.return_value = "ASCII" - try: - _unicodefun._verify_python_env() - except RuntimeError as re: - self.fail(f"`patch_click()` failed, exception still raised: {re}") - def test_root_logger_not_used_directly(self) -> None: def fail(*args: Any, **kwargs: Any) -> None: self.fail("Record created with root logger") From 0b4d7d55f78913be9e0a3738681ef3aafd5d9a5a Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 9 Jul 2023 15:05:01 -0700 Subject: [PATCH 046/279] Run pyupgrade on blib2to3 and src (#3771) --- src/black/files.py | 6 +-- src/black/handle_ipynb_magics.py | 2 +- src/blib2to3/pgen2/conv.py | 6 +-- src/blib2to3/pgen2/driver.py | 29 +++++++------- src/blib2to3/pgen2/grammar.py | 6 +-- src/blib2to3/pgen2/literals.py | 8 ++-- src/blib2to3/pgen2/parse.py | 22 +++++------ src/blib2to3/pgen2/pgen.py | 35 ++++++++--------- src/blib2to3/pgen2/token.py | 3 +- src/blib2to3/pgen2/tokenize.py | 27 +++++++------ src/blib2to3/pygram.py | 3 +- src/blib2to3/pytree.py | 67 ++++++++++++++++---------------- 12 files changed, 102 insertions(+), 112 deletions(-) diff --git a/src/black/files.py b/src/black/files.py index 65b2d0a8402..4e2209e557d 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -42,7 +42,7 @@ import colorama # noqa: F401 -@lru_cache() +@lru_cache def find_project_root( srcs: Sequence[str], stdin_filename: Optional[str] = None ) -> Tuple[Path, str]: @@ -212,7 +212,7 @@ def strip_specifier_set(specifier_set: SpecifierSet) -> SpecifierSet: return SpecifierSet(",".join(str(s) for s in specifiers)) -@lru_cache() +@lru_cache def find_user_pyproject_toml() -> Path: r"""Return the path to the top-level user configuration for black. @@ -232,7 +232,7 @@ def find_user_pyproject_toml() -> Path: return user_config_path.resolve() -@lru_cache() +@lru_cache def get_gitignore(root: Path) -> PathSpec: """Return a PathSpec matching gitignore content if present.""" gitignore = root / ".gitignore" diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index 4324819beaf..2a2d62220e2 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -55,7 +55,7 @@ class Replacement: src: str -@lru_cache() +@lru_cache def jupyter_dependencies_are_installed(*, verbose: bool, quiet: bool) -> bool: try: # isort: off diff --git a/src/blib2to3/pgen2/conv.py b/src/blib2to3/pgen2/conv.py index fa9825e54d6..04eccfa1d4b 100644 --- a/src/blib2to3/pgen2/conv.py +++ b/src/blib2to3/pgen2/conv.py @@ -63,7 +63,7 @@ def parse_graminit_h(self, filename): try: f = open(filename) except OSError as err: - print("Can't open %s: %s" % (filename, err)) + print(f"Can't open {filename}: {err}") return False self.symbol2number = {} self.number2symbol = {} @@ -72,7 +72,7 @@ def parse_graminit_h(self, filename): lineno += 1 mo = re.match(r"^#define\s+(\w+)\s+(\d+)$", line) if not mo and line.strip(): - print("%s(%s): can't parse %s" % (filename, lineno, line.strip())) + print(f"{filename}({lineno}): can't parse {line.strip()}") else: symbol, number = mo.groups() number = int(number) @@ -113,7 +113,7 @@ def parse_graminit_c(self, filename): try: f = open(filename) except OSError as err: - print("Can't open %s: %s" % (filename, err)) + print(f"Can't open {filename}: {err}") return False # The code below essentially uses f's iterator-ness! lineno = 0 diff --git a/src/blib2to3/pgen2/driver.py b/src/blib2to3/pgen2/driver.py index 1741b33c510..bb73016a4c1 100644 --- a/src/blib2to3/pgen2/driver.py +++ b/src/blib2to3/pgen2/driver.py @@ -28,11 +28,8 @@ Iterable, List, Optional, - Text, Iterator, Tuple, - TypeVar, - Generic, Union, ) from contextlib import contextmanager @@ -116,7 +113,7 @@ def can_advance(self, to: int) -> bool: return True -class Driver(object): +class Driver: def __init__(self, grammar: Grammar, logger: Optional[Logger] = None) -> None: self.grammar = grammar if logger is None: @@ -189,30 +186,30 @@ def parse_tokens(self, tokens: Iterable[GoodTokenInfo], debug: bool = False) -> assert p.rootnode is not None return p.rootnode - def parse_stream_raw(self, stream: IO[Text], debug: bool = False) -> NL: + def parse_stream_raw(self, stream: IO[str], debug: bool = False) -> NL: """Parse a stream and return the syntax tree.""" tokens = tokenize.generate_tokens(stream.readline, grammar=self.grammar) return self.parse_tokens(tokens, debug) - def parse_stream(self, stream: IO[Text], debug: bool = False) -> NL: + def parse_stream(self, stream: IO[str], debug: bool = False) -> NL: """Parse a stream and return the syntax tree.""" return self.parse_stream_raw(stream, debug) def parse_file( - self, filename: Path, encoding: Optional[Text] = None, debug: bool = False + self, filename: Path, encoding: Optional[str] = None, debug: bool = False ) -> NL: """Parse a file and return the syntax tree.""" - with io.open(filename, "r", encoding=encoding) as stream: + with open(filename, encoding=encoding) as stream: return self.parse_stream(stream, debug) - def parse_string(self, text: Text, debug: bool = False) -> NL: + def parse_string(self, text: str, debug: bool = False) -> NL: """Parse a string and return the syntax tree.""" tokens = tokenize.generate_tokens( io.StringIO(text).readline, grammar=self.grammar ) return self.parse_tokens(tokens, debug) - def _partially_consume_prefix(self, prefix: Text, column: int) -> Tuple[Text, Text]: + def _partially_consume_prefix(self, prefix: str, column: int) -> Tuple[str, str]: lines: List[str] = [] current_line = "" current_column = 0 @@ -240,7 +237,7 @@ def _partially_consume_prefix(self, prefix: Text, column: int) -> Tuple[Text, Te return "".join(lines), current_line -def _generate_pickle_name(gt: Path, cache_dir: Optional[Path] = None) -> Text: +def _generate_pickle_name(gt: Path, cache_dir: Optional[Path] = None) -> str: head, tail = os.path.splitext(gt) if tail == ".txt": tail = "" @@ -252,8 +249,8 @@ def _generate_pickle_name(gt: Path, cache_dir: Optional[Path] = None) -> Text: def load_grammar( - gt: Text = "Grammar.txt", - gp: Optional[Text] = None, + gt: str = "Grammar.txt", + gp: Optional[str] = None, save: bool = True, force: bool = False, logger: Optional[Logger] = None, @@ -276,7 +273,7 @@ def load_grammar( return g -def _newer(a: Text, b: Text) -> bool: +def _newer(a: str, b: str) -> bool: """Inquire whether file a was written since file b.""" if not os.path.exists(a): return False @@ -286,7 +283,7 @@ def _newer(a: Text, b: Text) -> bool: def load_packaged_grammar( - package: str, grammar_source: Text, cache_dir: Optional[Path] = None + package: str, grammar_source: str, cache_dir: Optional[Path] = None ) -> grammar.Grammar: """Normally, loads a pickled grammar by doing pkgutil.get_data(package, pickled_grammar) @@ -309,7 +306,7 @@ def load_packaged_grammar( return g -def main(*args: Text) -> bool: +def main(*args: str) -> bool: """Main program, when run as a script: produce grammar pickle files. Calls load_grammar for each argument, a path to a grammar text file. diff --git a/src/blib2to3/pgen2/grammar.py b/src/blib2to3/pgen2/grammar.py index 337a64f1726..1f3fdc55b97 100644 --- a/src/blib2to3/pgen2/grammar.py +++ b/src/blib2to3/pgen2/grammar.py @@ -16,19 +16,19 @@ import os import pickle import tempfile -from typing import Any, Dict, List, Optional, Text, Tuple, TypeVar, Union +from typing import Any, Dict, List, Optional, Tuple, TypeVar, Union # Local imports from . import token _P = TypeVar("_P", bound="Grammar") -Label = Tuple[int, Optional[Text]] +Label = Tuple[int, Optional[str]] DFA = List[List[Tuple[int, int]]] DFAS = Tuple[DFA, Dict[int, int]] Path = Union[str, "os.PathLike[str]"] -class Grammar(object): +class Grammar: """Pgen parsing tables conversion class. Once initialized, this class supplies the grammar tables for the diff --git a/src/blib2to3/pgen2/literals.py b/src/blib2to3/pgen2/literals.py index b5fe4285114..c67b91d0463 100644 --- a/src/blib2to3/pgen2/literals.py +++ b/src/blib2to3/pgen2/literals.py @@ -5,10 +5,10 @@ import re -from typing import Dict, Match, Text +from typing import Dict, Match -simple_escapes: Dict[Text, Text] = { +simple_escapes: Dict[str, str] = { "a": "\a", "b": "\b", "f": "\f", @@ -22,7 +22,7 @@ } -def escape(m: Match[Text]) -> Text: +def escape(m: Match[str]) -> str: all, tail = m.group(0, 1) assert all.startswith("\\") esc = simple_escapes.get(tail) @@ -44,7 +44,7 @@ def escape(m: Match[Text]) -> Text: return chr(i) -def evalString(s: Text) -> Text: +def evalString(s: str) -> str: assert s.startswith("'") or s.startswith('"'), repr(s[:1]) q = s[0] if s[:3] == q * 3: diff --git a/src/blib2to3/pgen2/parse.py b/src/blib2to3/pgen2/parse.py index c462f63ad2c..17bf118e9fc 100644 --- a/src/blib2to3/pgen2/parse.py +++ b/src/blib2to3/pgen2/parse.py @@ -9,7 +9,6 @@ how this parsing engine works. """ -import copy from contextlib import contextmanager # Local imports @@ -18,7 +17,6 @@ cast, Any, Optional, - Text, Union, Tuple, Dict, @@ -35,7 +33,7 @@ from blib2to3.pgen2.driver import TokenProxy -Results = Dict[Text, NL] +Results = Dict[str, NL] Convert = Callable[[Grammar, RawNode], Union[Node, Leaf]] DFA = List[List[Tuple[int, int]]] DFAS = Tuple[DFA, Dict[int, int]] @@ -100,7 +98,7 @@ def backtrack(self) -> Iterator[None]: finally: self.parser.is_backtracking = is_backtracking - def add_token(self, tok_type: int, tok_val: Text, raw: bool = False) -> None: + def add_token(self, tok_type: int, tok_val: str, raw: bool = False) -> None: func: Callable[..., Any] if raw: func = self.parser._addtoken @@ -114,7 +112,7 @@ def add_token(self, tok_type: int, tok_val: Text, raw: bool = False) -> None: args.insert(0, ilabel) func(*args) - def determine_route(self, value: Optional[Text] = None, force: bool = False) -> Optional[int]: + def determine_route(self, value: Optional[str] = None, force: bool = False) -> Optional[int]: alive_ilabels = self.ilabels if len(alive_ilabels) == 0: *_, most_successful_ilabel = self._dead_ilabels @@ -131,10 +129,10 @@ class ParseError(Exception): """Exception to signal the parser is stuck.""" def __init__( - self, msg: Text, type: Optional[int], value: Optional[Text], context: Context + self, msg: str, type: Optional[int], value: Optional[str], context: Context ) -> None: Exception.__init__( - self, "%s: type=%r, value=%r, context=%r" % (msg, type, value, context) + self, f"{msg}: type={type!r}, value={value!r}, context={context!r}" ) self.msg = msg self.type = type @@ -142,7 +140,7 @@ def __init__( self.context = context -class Parser(object): +class Parser: """Parser engine. The proper usage sequence is: @@ -236,7 +234,7 @@ def setup(self, proxy: "TokenProxy", start: Optional[int] = None) -> None: self.used_names: Set[str] = set() self.proxy = proxy - def addtoken(self, type: int, value: Text, context: Context) -> bool: + def addtoken(self, type: int, value: str, context: Context) -> bool: """Add a token; return True iff this is the end of the program.""" # Map from token to label ilabels = self.classify(type, value, context) @@ -284,7 +282,7 @@ def addtoken(self, type: int, value: Text, context: Context) -> bool: return self._addtoken(ilabel, type, value, context) - def _addtoken(self, ilabel: int, type: int, value: Text, context: Context) -> bool: + def _addtoken(self, ilabel: int, type: int, value: str, context: Context) -> bool: # Loop until the token is shifted; may raise exceptions while True: dfa, state, node = self.stack[-1] @@ -329,7 +327,7 @@ def _addtoken(self, ilabel: int, type: int, value: Text, context: Context) -> bo # No success finding a transition raise ParseError("bad input", type, value, context) - def classify(self, type: int, value: Text, context: Context) -> List[int]: + def classify(self, type: int, value: str, context: Context) -> List[int]: """Turn a token into a label. (Internal) Depending on whether the value is a soft-keyword or not, @@ -352,7 +350,7 @@ def classify(self, type: int, value: Text, context: Context) -> List[int]: raise ParseError("bad token", type, value, context) return [ilabel] - def shift(self, type: int, value: Text, newstate: int, context: Context) -> None: + def shift(self, type: int, value: str, newstate: int, context: Context) -> None: """Shift a token. (Internal)""" if self.is_backtracking: dfa, state, _ = self.stack[-1] diff --git a/src/blib2to3/pgen2/pgen.py b/src/blib2to3/pgen2/pgen.py index b5ebc7b3e42..046efd09338 100644 --- a/src/blib2to3/pgen2/pgen.py +++ b/src/blib2to3/pgen2/pgen.py @@ -11,7 +11,6 @@ Iterator, List, Optional, - Text, Tuple, Union, Sequence, @@ -29,13 +28,13 @@ class PgenGrammar(grammar.Grammar): pass -class ParserGenerator(object): +class ParserGenerator: filename: Path - stream: IO[Text] + stream: IO[str] generator: Iterator[GoodTokenInfo] - first: Dict[Text, Optional[Dict[Text, int]]] + first: Dict[str, Optional[Dict[str, int]]] - def __init__(self, filename: Path, stream: Optional[IO[Text]] = None) -> None: + def __init__(self, filename: Path, stream: Optional[IO[str]] = None) -> None: close_stream = None if stream is None: stream = open(filename, encoding="utf-8") @@ -75,7 +74,7 @@ def make_grammar(self) -> PgenGrammar: c.start = c.symbol2number[self.startsymbol] return c - def make_first(self, c: PgenGrammar, name: Text) -> Dict[int, int]: + def make_first(self, c: PgenGrammar, name: str) -> Dict[int, int]: rawfirst = self.first[name] assert rawfirst is not None first = {} @@ -85,7 +84,7 @@ def make_first(self, c: PgenGrammar, name: Text) -> Dict[int, int]: first[ilabel] = 1 return first - def make_label(self, c: PgenGrammar, label: Text) -> int: + def make_label(self, c: PgenGrammar, label: str) -> int: # XXX Maybe this should be a method on a subclass of converter? ilabel = len(c.labels) if label[0].isalpha(): @@ -144,7 +143,7 @@ def addfirstsets(self) -> None: self.calcfirst(name) # print name, self.first[name].keys() - def calcfirst(self, name: Text) -> None: + def calcfirst(self, name: str) -> None: dfa = self.dfas[name] self.first[name] = None # dummy to detect left recursion state = dfa[0] @@ -176,7 +175,7 @@ def calcfirst(self, name: Text) -> None: inverse[symbol] = label self.first[name] = totalset - def parse(self) -> Tuple[Dict[Text, List["DFAState"]], Text]: + def parse(self) -> Tuple[Dict[str, List["DFAState"]], str]: dfas = {} startsymbol: Optional[str] = None # MSTART: (NEWLINE | RULE)* ENDMARKER @@ -240,7 +239,7 @@ def addclosure(state: NFAState, base: Dict[NFAState, int]) -> None: state.addarc(st, label) return states # List of DFAState instances; first one is start - def dump_nfa(self, name: Text, start: "NFAState", finish: "NFAState") -> None: + def dump_nfa(self, name: str, start: "NFAState", finish: "NFAState") -> None: print("Dump of NFA for", name) todo = [start] for i, state in enumerate(todo): @@ -256,7 +255,7 @@ def dump_nfa(self, name: Text, start: "NFAState", finish: "NFAState") -> None: else: print(" %s -> %d" % (label, j)) - def dump_dfa(self, name: Text, dfa: Sequence["DFAState"]) -> None: + def dump_dfa(self, name: str, dfa: Sequence["DFAState"]) -> None: print("Dump of DFA for", name) for i, state in enumerate(dfa): print(" State", i, state.isfinal and "(final)" or "") @@ -349,7 +348,7 @@ def parse_atom(self) -> Tuple["NFAState", "NFAState"]: ) assert False - def expect(self, type: int, value: Optional[Any] = None) -> Text: + def expect(self, type: int, value: Optional[Any] = None) -> str: if self.type != type or (value is not None and self.value != value): self.raise_error( "expected %s/%s, got %s/%s", type, value, self.type, self.value @@ -374,22 +373,22 @@ def raise_error(self, msg: str, *args: Any) -> NoReturn: raise SyntaxError(msg, (self.filename, self.end[0], self.end[1], self.line)) -class NFAState(object): - arcs: List[Tuple[Optional[Text], "NFAState"]] +class NFAState: + arcs: List[Tuple[Optional[str], "NFAState"]] def __init__(self) -> None: self.arcs = [] # list of (label, NFAState) pairs - def addarc(self, next: "NFAState", label: Optional[Text] = None) -> None: + def addarc(self, next: "NFAState", label: Optional[str] = None) -> None: assert label is None or isinstance(label, str) assert isinstance(next, NFAState) self.arcs.append((label, next)) -class DFAState(object): +class DFAState: nfaset: Dict[NFAState, Any] isfinal: bool - arcs: Dict[Text, "DFAState"] + arcs: Dict[str, "DFAState"] def __init__(self, nfaset: Dict[NFAState, Any], final: NFAState) -> None: assert isinstance(nfaset, dict) @@ -399,7 +398,7 @@ def __init__(self, nfaset: Dict[NFAState, Any], final: NFAState) -> None: self.isfinal = final in nfaset self.arcs = {} # map from label to DFAState - def addarc(self, next: "DFAState", label: Text) -> None: + def addarc(self, next: "DFAState", label: str) -> None: assert isinstance(label, str) assert label not in self.arcs assert isinstance(next, DFAState) diff --git a/src/blib2to3/pgen2/token.py b/src/blib2to3/pgen2/token.py index c939531d7c8..117cc09d4ce 100644 --- a/src/blib2to3/pgen2/token.py +++ b/src/blib2to3/pgen2/token.py @@ -1,6 +1,5 @@ """Token constants (from "token.h").""" -import sys from typing import Dict from typing import Final @@ -75,7 +74,7 @@ tok_name: Final[Dict[int, str]] = {} for _name, _value in list(globals().items()): - if type(_value) is type(0): + if type(_value) is int: tok_name[_value] = _name diff --git a/src/blib2to3/pgen2/tokenize.py b/src/blib2to3/pgen2/tokenize.py index a5e89188d87..1dea89d7bb8 100644 --- a/src/blib2to3/pgen2/tokenize.py +++ b/src/blib2to3/pgen2/tokenize.py @@ -35,7 +35,6 @@ List, Optional, Set, - Text, Tuple, Pattern, Union, @@ -77,7 +76,7 @@ def maybe(*choices: str) -> str: def _combinations(*l: str) -> Set[str]: - return set(x + y for x in l for y in l + ("",) if x.casefold() != y.casefold()) + return {x + y for x in l for y in l + ("",) if x.casefold() != y.casefold()} Whitespace = r"[ \f\t]*" @@ -189,7 +188,7 @@ class StopTokenizing(Exception): def printtoken( - type: int, token: Text, srow_col: Coord, erow_col: Coord, line: Text + type: int, token: str, srow_col: Coord, erow_col: Coord, line: str ) -> None: # for testing (srow, scol) = srow_col (erow, ecol) = erow_col @@ -198,10 +197,10 @@ def printtoken( ) -TokenEater = Callable[[int, Text, Coord, Coord, Text], None] +TokenEater = Callable[[int, str, Coord, Coord, str], None] -def tokenize(readline: Callable[[], Text], tokeneater: TokenEater = printtoken) -> None: +def tokenize(readline: Callable[[], str], tokeneater: TokenEater = printtoken) -> None: """ The tokenize() function accepts two parameters: one representing the input stream, and one providing an output mechanism for tokenize(). @@ -221,17 +220,17 @@ def tokenize(readline: Callable[[], Text], tokeneater: TokenEater = printtoken) # backwards compatible interface -def tokenize_loop(readline: Callable[[], Text], tokeneater: TokenEater) -> None: +def tokenize_loop(readline: Callable[[], str], tokeneater: TokenEater) -> None: for token_info in generate_tokens(readline): tokeneater(*token_info) -GoodTokenInfo = Tuple[int, Text, Coord, Coord, Text] +GoodTokenInfo = Tuple[int, str, Coord, Coord, str] TokenInfo = Union[Tuple[int, str], GoodTokenInfo] class Untokenizer: - tokens: List[Text] + tokens: List[str] prev_row: int prev_col: int @@ -247,13 +246,13 @@ def add_whitespace(self, start: Coord) -> None: if col_offset: self.tokens.append(" " * col_offset) - def untokenize(self, iterable: Iterable[TokenInfo]) -> Text: + def untokenize(self, iterable: Iterable[TokenInfo]) -> str: for t in iterable: if len(t) == 2: self.compat(cast(Tuple[int, str], t), iterable) break tok_type, token, start, end, line = cast( - Tuple[int, Text, Coord, Coord, Text], t + Tuple[int, str, Coord, Coord, str], t ) self.add_whitespace(start) self.tokens.append(token) @@ -263,7 +262,7 @@ def untokenize(self, iterable: Iterable[TokenInfo]) -> Text: self.prev_col = 0 return "".join(self.tokens) - def compat(self, token: Tuple[int, Text], iterable: Iterable[TokenInfo]) -> None: + def compat(self, token: Tuple[int, str], iterable: Iterable[TokenInfo]) -> None: startline = False indents = [] toks_append = self.tokens.append @@ -335,7 +334,7 @@ def read_or_stop() -> bytes: try: return readline() except StopIteration: - return bytes() + return b'' def find_cookie(line: bytes) -> Optional[str]: try: @@ -384,7 +383,7 @@ def find_cookie(line: bytes) -> Optional[str]: return default, [first, second] -def untokenize(iterable: Iterable[TokenInfo]) -> Text: +def untokenize(iterable: Iterable[TokenInfo]) -> str: """Transform tokens back into Python source code. Each element returned by the iterable must be a token sequence @@ -407,7 +406,7 @@ def untokenize(iterable: Iterable[TokenInfo]) -> Text: def generate_tokens( - readline: Callable[[], Text], grammar: Optional[Grammar] = None + readline: Callable[[], str], grammar: Optional[Grammar] = None ) -> Iterator[GoodTokenInfo]: """ The generate_tokens() generator requires one argument, readline, which diff --git a/src/blib2to3/pygram.py b/src/blib2to3/pygram.py index 15702e4059e..1b4832362bf 100644 --- a/src/blib2to3/pygram.py +++ b/src/blib2to3/pygram.py @@ -9,7 +9,6 @@ from typing import Union # Local imports -from .pgen2 import token from .pgen2 import driver from .pgen2.grammar import Grammar @@ -21,7 +20,7 @@ # "PatternGrammar.txt") -class Symbols(object): +class Symbols: def __init__(self, grammar: Grammar) -> None: """Initializer. diff --git a/src/blib2to3/pytree.py b/src/blib2to3/pytree.py index ea60c894e20..156322cab7e 100644 --- a/src/blib2to3/pytree.py +++ b/src/blib2to3/pytree.py @@ -18,7 +18,6 @@ Iterator, List, Optional, - Text, Tuple, TypeVar, Union, @@ -34,10 +33,10 @@ HUGE: int = 0x7FFFFFFF # maximum repeat count, default max -_type_reprs: Dict[int, Union[Text, int]] = {} +_type_reprs: Dict[int, Union[str, int]] = {} -def type_repr(type_num: int) -> Union[Text, int]: +def type_repr(type_num: int) -> Union[str, int]: global _type_reprs if not _type_reprs: from .pygram import python_symbols @@ -54,11 +53,11 @@ def type_repr(type_num: int) -> Union[Text, int]: _P = TypeVar("_P", bound="Base") NL = Union["Node", "Leaf"] -Context = Tuple[Text, Tuple[int, int]] -RawNode = Tuple[int, Optional[Text], Optional[Context], Optional[List[NL]]] +Context = Tuple[str, Tuple[int, int]] +RawNode = Tuple[int, Optional[str], Optional[Context], Optional[List[NL]]] -class Base(object): +class Base: """ Abstract base class for Node and Leaf. @@ -92,7 +91,7 @@ def __eq__(self, other: Any) -> bool: return self._eq(other) @property - def prefix(self) -> Text: + def prefix(self) -> str: raise NotImplementedError def _eq(self: _P, other: _P) -> bool: @@ -225,7 +224,7 @@ def depth(self) -> int: return 0 return 1 + self.parent.depth() - def get_suffix(self) -> Text: + def get_suffix(self) -> str: """ Return the string immediately following the invocant node. This is effectively equivalent to node.next_sibling.prefix @@ -242,14 +241,14 @@ class Node(Base): """Concrete implementation for interior nodes.""" fixers_applied: Optional[List[Any]] - used_names: Optional[Set[Text]] + used_names: Optional[Set[str]] def __init__( self, type: int, children: List[NL], context: Optional[Any] = None, - prefix: Optional[Text] = None, + prefix: Optional[str] = None, fixers_applied: Optional[List[Any]] = None, ) -> None: """ @@ -274,16 +273,16 @@ def __init__( else: self.fixers_applied = None - def __repr__(self) -> Text: + def __repr__(self) -> str: """Return a canonical string representation.""" assert self.type is not None - return "%s(%s, %r)" % ( + return "{}({}, {!r})".format( self.__class__.__name__, type_repr(self.type), self.children, ) - def __str__(self) -> Text: + def __str__(self) -> str: """ Return a pretty string representation. @@ -317,7 +316,7 @@ def pre_order(self) -> Iterator[NL]: yield from child.pre_order() @property - def prefix(self) -> Text: + def prefix(self) -> str: """ The whitespace and comments preceding this node in the input. """ @@ -326,7 +325,7 @@ def prefix(self) -> Text: return self.children[0].prefix @prefix.setter - def prefix(self, prefix: Text) -> None: + def prefix(self, prefix: str) -> None: if self.children: self.children[0].prefix = prefix @@ -383,12 +382,12 @@ class Leaf(Base): """Concrete implementation for leaf nodes.""" # Default values for instance variables - value: Text + value: str fixers_applied: List[Any] bracket_depth: int # Changed later in brackets.py opening_bracket: Optional["Leaf"] = None - used_names: Optional[Set[Text]] + used_names: Optional[Set[str]] _prefix = "" # Whitespace and comments preceding this token in the input lineno: int = 0 # Line where this token starts in the input column: int = 0 # Column where this token starts in the input @@ -400,9 +399,9 @@ class Leaf(Base): def __init__( self, type: int, - value: Text, + value: str, context: Optional[Context] = None, - prefix: Optional[Text] = None, + prefix: Optional[str] = None, fixers_applied: List[Any] = [], opening_bracket: Optional["Leaf"] = None, fmt_pass_converted_first_leaf: Optional["Leaf"] = None, @@ -431,13 +430,13 @@ def __repr__(self) -> str: from .pgen2.token import tok_name assert self.type is not None - return "%s(%s, %r)" % ( + return "{}({}, {!r})".format( self.__class__.__name__, tok_name.get(self.type, self.type), self.value, ) - def __str__(self) -> Text: + def __str__(self) -> str: """ Return a pretty string representation. @@ -471,14 +470,14 @@ def pre_order(self) -> Iterator["Leaf"]: yield self @property - def prefix(self) -> Text: + def prefix(self) -> str: """ The whitespace and comments preceding this token in the input. """ return self._prefix @prefix.setter - def prefix(self, prefix: Text) -> None: + def prefix(self, prefix: str) -> None: self.changed() self._prefix = prefix @@ -503,10 +502,10 @@ def convert(gr: Grammar, raw_node: RawNode) -> NL: return Leaf(type, value or "", context=context) -_Results = Dict[Text, NL] +_Results = Dict[str, NL] -class BasePattern(object): +class BasePattern: """ A pattern is a tree matching pattern. @@ -526,19 +525,19 @@ class BasePattern(object): type: Optional[int] type = None # Node type (token if < 256, symbol if >= 256) content: Any = None # Optional content matching pattern - name: Optional[Text] = None # Optional name used to store match in results dict + name: Optional[str] = None # Optional name used to store match in results dict def __new__(cls, *args, **kwds): """Constructor that prevents BasePattern from being instantiated.""" assert cls is not BasePattern, "Cannot instantiate BasePattern" return object.__new__(cls) - def __repr__(self) -> Text: + def __repr__(self) -> str: assert self.type is not None args = [type_repr(self.type), self.content, self.name] while args and args[-1] is None: del args[-1] - return "%s(%s)" % (self.__class__.__name__, ", ".join(map(repr, args))) + return "{}({})".format(self.__class__.__name__, ", ".join(map(repr, args))) def _submatch(self, node, results=None) -> bool: raise NotImplementedError @@ -602,8 +601,8 @@ class LeafPattern(BasePattern): def __init__( self, type: Optional[int] = None, - content: Optional[Text] = None, - name: Optional[Text] = None, + content: Optional[str] = None, + name: Optional[str] = None, ) -> None: """ Initializer. Takes optional type, content, and name. @@ -653,8 +652,8 @@ class NodePattern(BasePattern): def __init__( self, type: Optional[int] = None, - content: Optional[Iterable[Text]] = None, - name: Optional[Text] = None, + content: Optional[Iterable[str]] = None, + name: Optional[str] = None, ) -> None: """ Initializer. Takes optional type, content, and name. @@ -734,10 +733,10 @@ class WildcardPattern(BasePattern): def __init__( self, - content: Optional[Text] = None, + content: Optional[str] = None, min: int = 0, max: int = HUGE, - name: Optional[Text] = None, + name: Optional[str] = None, ) -> None: """ Initializer. From f3b50e466969f9142393ec32a4b2a383ffbe5f23 Mon Sep 17 00:00:00 2001 From: Kenneth Schackart Date: Sun, 9 Jul 2023 15:07:21 -0700 Subject: [PATCH 047/279] Add CITATION.cff file (#3723) --- CHANGES.md | 3 +++ CITATION.cff | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 CITATION.cff diff --git a/CHANGES.md b/CHANGES.md index 1b0475fc7b9..15027afbf0b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -84,6 +84,9 @@ ### Documentation +- Add a CITATION.cff file to the root of the repository, containing metadata on how to + cite this software (#3723) + diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 00000000000..ddf64f616ff --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,22 @@ +cff-version: 1.2.0 +title: "Black: The uncompromising Python code formatter" +message: >- + If you use this software, please cite it using the metadata from this file. +type: software +authors: + - family-names: Langa + given-names: Łukasz + - name: "contributors to Black" +repository-code: "https://github.com/psf/black" +url: "https://black.readthedocs.io/en/stable/" +abstract: >- + Black is the uncompromising Python code formatter. By using it, you agree to cede + control over minutiae ofhand-formatting. In return, Black gives you speed, + determinism, and freedom from pycodestyle nagging about formatting. You will save time + and mental energy for more important matters. + + Blackened code looks the same regardless of the project you're reading. Formatting + becomes transparent after a while and you can focus on the content instead. + + Black makes code review faster by producing the smallest diffs possible. +license: MIT From 2593af2c5d211b58e28f7c1472f1f67e6783216a Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 9 Jul 2023 15:24:01 -0700 Subject: [PATCH 048/279] Improve performance by skipping unnecessary normalisation (#3751) This speeds up black by about 40% when the cache is full --- CHANGES.md | 1 + src/black/files.py | 23 +++++++++++++++++------ tests/test_black.py | 4 +++- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 15027afbf0b..93d8ee1921a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -55,6 +55,7 @@ +- Speed up _Black_ significantly when the cache is full (#3751) - Avoid importing `IPython` in a case where we wouldn't need it (#3748) ### Output diff --git a/src/black/files.py b/src/black/files.py index 4e2209e557d..ef6895ee3af 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -276,15 +276,24 @@ def normalize_path_maybe_ignore( return root_relative_path -def path_is_ignored( - path: Path, gitignore_dict: Dict[Path, PathSpec], report: Report +def _path_is_ignored( + root_relative_path: str, + root: Path, + gitignore_dict: Dict[Path, PathSpec], + report: Report, ) -> bool: + path = root / root_relative_path + # Note that this logic is sensitive to the ordering of gitignore_dict. Callers must + # ensure that gitignore_dict is ordered from least specific to most specific. for gitignore_path, pattern in gitignore_dict.items(): - relative_path = normalize_path_maybe_ignore(path, gitignore_path, report) - if relative_path is None: + try: + relative_path = path.relative_to(gitignore_path).as_posix() + except ValueError: break if pattern.match_file(relative_path): - report.path_ignored(path, "matches a .gitignore file content") + report.path_ignored( + path.relative_to(root), "matches a .gitignore file content" + ) return True return False @@ -326,7 +335,9 @@ def gen_python_files( continue # First ignore files matching .gitignore, if passed - if gitignore_dict and path_is_ignored(child, gitignore_dict, report): + if gitignore_dict and _path_is_ignored( + normalized_path, root, gitignore_dict, report + ): continue # Then ignore with `--exclude` `--extend-exclude` and `--force-exclude` options. diff --git a/tests/test_black.py b/tests/test_black.py index dd21d0a7ae6..3b3ab721c5f 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -508,6 +508,8 @@ def _mocked_calls() -> bool: "pathlib.Path.cwd", return_value=working_directory ), patch("pathlib.Path.is_dir", side_effect=mock_n_calls([True])): ctx = FakeContext() + # Note that the root folder (project_root) isn't the folder + # named "root" (aka working_directory) ctx.obj["root"] = project_root report = MagicMock(verbose=True) black.get_sources( @@ -527,7 +529,7 @@ def _mocked_calls() -> bool: for _, mock_args, _ in report.path_ignored.mock_calls ), "A symbolic link was reported." report.path_ignored.assert_called_once_with( - Path("child", "b.py"), "matches a .gitignore file content" + Path("root", "child", "b.py"), "matches a .gitignore file content" ) def test_report_verbose(self) -> None: From 257d392217974a76231e306133288748c7b70786 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 9 Jul 2023 15:52:41 -0700 Subject: [PATCH 049/279] Fix removed comments in stub files (#3745) --- CHANGES.md | 2 ++ src/black/nodes.py | 11 ++++++- tests/data/simple_cases/ignore_pyi.py | 41 +++++++++++++++++++++++++++ tests/test_format.py | 5 ++-- 4 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 tests/data/simple_cases/ignore_pyi.py diff --git a/CHANGES.md b/CHANGES.md index 93d8ee1921a..bb304296d63 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,6 +15,8 @@ - Fix a bug where an illegal trailing comma was added to return type annotations using PEP 604 unions (#3735) +- Fix several bugs and crashes where comments in stub files were removed or mishandled + under some circumstances. (#3745) - Fix a bug where multi-line open parenthesis magic comment like `type: ignore` were not correctly parsed (#3740) diff --git a/src/black/nodes.py b/src/black/nodes.py index ef42278d83f..45423b2596b 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -714,6 +714,11 @@ def is_multiline_string(leaf: Leaf) -> bool: def is_stub_suite(node: Node) -> bool: """Return True if `node` is a suite with a stub body.""" + + # If there is a comment, we want to keep it. + if node.prefix.strip(): + return False + if ( len(node.children) != 4 or node.children[0].type != token.NEWLINE @@ -722,6 +727,9 @@ def is_stub_suite(node: Node) -> bool: ): return False + if node.children[3].prefix.strip(): + return False + return is_stub_body(node.children[2]) @@ -735,7 +743,8 @@ def is_stub_body(node: LN) -> bool: child = node.children[0] return ( - child.type == syms.atom + not child.prefix.strip() + and child.type == syms.atom and len(child.children) == 3 and all(leaf == Leaf(token.DOT, ".") for leaf in child.children) ) diff --git a/tests/data/simple_cases/ignore_pyi.py b/tests/data/simple_cases/ignore_pyi.py new file mode 100644 index 00000000000..3ef61079bfe --- /dev/null +++ b/tests/data/simple_cases/ignore_pyi.py @@ -0,0 +1,41 @@ +def f(): # type: ignore + ... + +class x: # some comment + ... + +class y: + ... # comment + +# whitespace doesn't matter (note the next line has a trailing space and tab) +class z: + ... + +def g(): + # hi + ... + +def h(): + ... + # bye + +# output + +def f(): # type: ignore + ... + +class x: # some comment + ... + +class y: ... # comment + +# whitespace doesn't matter (note the next line has a trailing space and tab) +class z: ... + +def g(): + # hi + ... + +def h(): + ... + # bye diff --git a/tests/test_format.py b/tests/test_format.py index 8e0ada99cba..fb4d8eb4346 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -33,9 +33,10 @@ def check_file( @pytest.mark.parametrize("filename", all_data_cases("simple_cases")) def test_simple_format(filename: str) -> None: magic_trailing_comma = filename != "skip_magic_trailing_comma" - check_file( - "simple_cases", filename, black.Mode(magic_trailing_comma=magic_trailing_comma) + mode = black.Mode( + magic_trailing_comma=magic_trailing_comma, is_pyi=filename.endswith("_pyi") ) + check_file("simple_cases", filename, mode) @pytest.mark.parametrize("filename", all_data_cases("preview")) From b8e2ec728cc09d0f00829a9cffcb54da3efa5760 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 9 Jul 2023 16:28:26 -0700 Subject: [PATCH 050/279] Fix crash on type comment with trailing space (#3773) --- CHANGES.md | 2 ++ src/black/parsing.py | 9 ++++++--- tests/data/simple_cases/comments2.py | 5 +++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index bb304296d63..c7389ce57c6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,6 +19,8 @@ under some circumstances. (#3745) - Fix a bug where multi-line open parenthesis magic comment like `type: ignore` were not correctly parsed (#3740) +- Fix error in AST validation when Black removes trailing whitespace in a type comment + (#3773) ### Preview style diff --git a/src/black/parsing.py b/src/black/parsing.py index 455c5eed968..e98e019cac6 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -208,15 +208,18 @@ def stringify_ast(node: ast.AST, depth: int = 0) -> Iterator[str]: else: normalized: object - # Constant strings may be indented across newlines, if they are - # docstrings; fold spaces after newlines when comparing. Similarly, - # trailing and leading space may be removed. if ( isinstance(node, ast.Constant) and field == "value" and isinstance(value, str) ): + # Constant strings may be indented across newlines, if they are + # docstrings; fold spaces after newlines when comparing. Similarly, + # trailing and leading space may be removed. normalized = _normalize("\n", value) + elif field == "type_comment" and isinstance(value, str): + # Trailing whitespace in type comments is removed. + normalized = value.rstrip() else: normalized = value yield f"{' ' * (depth+2)}{normalized!r}, # {value.__class__.__name__}" diff --git a/tests/data/simple_cases/comments2.py b/tests/data/simple_cases/comments2.py index 37e185abf4f..1487dc4b6e2 100644 --- a/tests/data/simple_cases/comments2.py +++ b/tests/data/simple_cases/comments2.py @@ -154,6 +154,9 @@ def _init_host(self, parsed) -> None: not parsed.hostname.strip()): pass + +a = "type comment with trailing space" # type: str + ####################### ### SECTION COMMENT ### ####################### @@ -332,6 +335,8 @@ def _init_host(self, parsed) -> None: pass +a = "type comment with trailing space" # type: str + ####################### ### SECTION COMMENT ### ####################### From ad3724b7ffc01d8152c97fe7f4dcf35220f21a8e Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 9 Jul 2023 17:04:50 -0700 Subject: [PATCH 051/279] Upgrade to latest mypy (#3775) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bb647763e70..c2f4b1684e6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -38,7 +38,7 @@ repos: - flake8-simplify - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.991 + rev: v1.4.1 hooks: - id: mypy exclude: ^docs/conf.py From 138769aa27d6bd86507a0cd98d9a5bf8f63a8e99 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 10 Jul 2023 08:37:12 -0700 Subject: [PATCH 052/279] Disable coverage on pypy tests (#3777) The pypy tests are reeeeaaally slow. Maybe this will help. --- .github/workflows/test.yml | 4 +++- tox.ini | 8 ++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 92d7d411510..4bf687435b4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,7 +58,9 @@ jobs: - name: Upload coverage to Coveralls # Upload coverage if we are on the main repository and # we're running on Linux (this action only supports Linux) - if: github.repository == 'psf/black' && matrix.os == 'ubuntu-latest' + if: + github.repository == 'psf/black' && matrix.os == 'ubuntu-latest' && + !startsWith(matrix.python-version, 'pypy') uses: AndreMiras/coveralls-python-action@v20201129 with: github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/tox.ini b/tox.ini index f8e1a785331..d34dbbc71db 100644 --- a/tox.ini +++ b/tox.ini @@ -39,19 +39,15 @@ deps = ; remove this when pypy releases the bugfix commands = pip install -e .[d] - coverage erase pytest tests \ --run-optional no_jupyter \ !ci: --numprocesses auto \ - ci: --numprocesses 1 \ - --cov {posargs} + ci: --numprocesses 1 pip install -e .[jupyter] pytest tests --run-optional jupyter \ -m jupyter \ !ci: --numprocesses auto \ - ci: --numprocesses 1 \ - --cov --cov-append {posargs} - coverage report + ci: --numprocesses 1 [testenv:{,ci-}311] setenv = From 38723bb7787d50f8751fad2eaa48b52c9e94c18d Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 10 Jul 2023 11:49:40 -0700 Subject: [PATCH 053/279] Unpin pytest-xdist (#3772) --- test_requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_requirements.txt b/test_requirements.txt index ef61a1210ee..a3d262bc53d 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,6 +1,6 @@ coverage >= 5.3 pre-commit pytest >= 6.1.1 -pytest-xdist >= 2.2.1, < 3.0.2 -pytest-cov >= 2.11.1 +pytest-xdist >= 3.0.2 +pytest-cov >= 4.1.0 tox From 193ee766ca496871f93621d6b58d57a6564ff81b Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 10 Jul 2023 17:09:47 -0700 Subject: [PATCH 054/279] Prepare release 23.7.0 (#3776) --- CHANGES.md | 89 +++++++++++++-------- docs/integrations/source_version_control.md | 4 +- docs/usage_and_configuration/the_basics.md | 6 +- 3 files changed, 60 insertions(+), 39 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c7389ce57c6..c61ee698c5d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,97 +6,118 @@ +### Stable style + + + +### Preview style + + + +### Configuration + + + +### Packaging + + + +### Parser + + + +### Performance + + + +### Output + + + +### _Blackd_ + + + +### Integrations + + + +### Documentation + + + +## 23.7.0 + +### Highlights + - Runtime support for Python 3.7 has been removed. Formatting 3.7 code will still be supported until further notice (#3765) ### Stable style - - - Fix a bug where an illegal trailing comma was added to return type annotations using PEP 604 unions (#3735) - Fix several bugs and crashes where comments in stub files were removed or mishandled - under some circumstances. (#3745) -- Fix a bug where multi-line open parenthesis magic comment like `type: ignore` were not - correctly parsed (#3740) -- Fix error in AST validation when Black removes trailing whitespace in a type comment + under some circumstances (#3745) +- Fix a crash with multi-line magic comments like `type: ignore` within parentheses + (#3740) +- Fix error in AST validation when _Black_ removes trailing whitespace in a type comment (#3773) ### Preview style - - - Implicitly concatenated strings used as function args are no longer wrapped inside parentheses (#3640) - Remove blank lines between a class definition and its docstring (#3692) ### Configuration - - -- The `--workers` argument to Black can now be specified via the `BLACK_NUM_WORKERS` +- The `--workers` argument to _Black_ can now be specified via the `BLACK_NUM_WORKERS` environment variable (#3743) - `.pytest_cache`, `.ruff_cache` and `.vscode` are now excluded by default (#3691) -- Fix black not honouring `pyproject.toml` settings when running `--stdin-filename` and - the `pyproject.toml` found isn't in the current working directory (#3719) -- Black will now error if `exclude` and `extend-exclude` have invalid data types in +- Fix _Black_ not honouring `pyproject.toml` settings when running `--stdin-filename` + and the `pyproject.toml` found isn't in the current working directory (#3719) +- _Black_ will now error if `exclude` and `extend-exclude` have invalid data types in `pyproject.toml`, instead of silently doing the wrong thing (#3764) ### Packaging - - - Upgrade mypyc from 0.991 to 1.3 (#3697) - Remove patching of Click that mitigated errors on Python 3.6 with `LANG=C` (#3768) ### Parser - - - Add support for the new PEP 695 syntax in Python 3.12 (#3703) ### Performance - - - Speed up _Black_ significantly when the cache is full (#3751) - Avoid importing `IPython` in a case where we wouldn't need it (#3748) ### Output - - - Use aware UTC datetimes internally, avoids deprecation warning on Python 3.12 (#3728) - Change verbose logging to exactly mirror _Black_'s logic for source discovery (#3749) ### _Blackd_ - - - The `blackd` argument parser now shows the default values for options in their help text (#3712) ### Integrations - - - Black is now tested with [`PYTHONWARNDEFAULTENCODING = 1`](https://docs.python.org/3/library/io.html#io-encoding-warning) (#3763) - Update GitHub Action to display black output in the job summary (#3688) -- Deprecated `set-output` command in CI test to keep up to date with GitHub's - deprecation announcement (#3757) ### Documentation - Add a CITATION.cff file to the root of the repository, containing metadata on how to cite this software (#3723) - - - -- Updated the _classes_ and _exceptions_ documentation in Developer reference to match - the latest ccode base. (#3755) +- Update the _classes_ and _exceptions_ documentation in Developer reference to match + the latest code base (#3755) ## 23.3.0 diff --git a/docs/integrations/source_version_control.md b/docs/integrations/source_version_control.md index 8b8fd658e0e..a9d33d2d853 100644 --- a/docs/integrations/source_version_control.md +++ b/docs/integrations/source_version_control.md @@ -7,7 +7,7 @@ Use [pre-commit](https://pre-commit.com/). Once you ```yaml repos: - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 23.7.0 hooks: - id: black # It is recommended to specify the latest version of Python @@ -31,7 +31,7 @@ include Jupyter Notebooks. To use this hook, simply replace the hook's `id: blac ```yaml repos: - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 23.7.0 hooks: - id: black-jupyter # It is recommended to specify the latest version of Python diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 2a461487210..f5862edccaa 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -193,8 +193,8 @@ configuration file for consistent results across environments. ```console $ black --version -black, 23.3.0 (compiled: yes) -$ black --required-version 23.3.0 -c "format = 'this'" +black, 23.7.0 (compiled: yes) +$ black --required-version 23.7.0 -c "format = 'this'" format = "this" $ black --required-version 31.5b2 -c "still = 'beta?!'" Oh no! 💥 💔 💥 The required version does not match the running version! @@ -285,7 +285,7 @@ You can check the version of _Black_ you have installed using the `--version` fl ```console $ black --version -black, 23.3.0 +black, 23.7.0 ``` #### `--config` From a062d5c9854205c4ae3f4cbe8859ed59bcd6259c Mon Sep 17 00:00:00 2001 From: skykasko <88055150+skykasko@users.noreply.github.com> Date: Mon, 10 Jul 2023 22:38:01 -0400 Subject: [PATCH 055/279] Fix typo in CITATION.cff (#3779) Fix tiny typo in CITATION.cff --- CITATION.cff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CITATION.cff b/CITATION.cff index ddf64f616ff..7ff0e3ca9bc 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -11,7 +11,7 @@ repository-code: "https://github.com/psf/black" url: "https://black.readthedocs.io/en/stable/" abstract: >- Black is the uncompromising Python code formatter. By using it, you agree to cede - control over minutiae ofhand-formatting. In return, Black gives you speed, + control over minutiae of hand-formatting. In return, Black gives you speed, determinism, and freedom from pycodestyle nagging about formatting. You will save time and mental energy for more important matters. From 027afda403d5da7b0ea2a1bf40788ad4c3eb510e Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 11 Jul 2023 15:21:15 +0100 Subject: [PATCH 056/279] Remove Python 3.7 from classifiers (#3784) Follow-up on https://github.com/psf/black/pull/3765 --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 175f7851dee..aaac42b44b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,6 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", From f4490acfd7bf466ae30d7573d85107b46d4686b3 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 11 Jul 2023 15:21:36 +0100 Subject: [PATCH 057/279] Remove unneeded mypy dependencies (#3783) --- .pre-commit-config.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c2f4b1684e6..3561df4f904 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,10 +43,8 @@ repos: - id: mypy exclude: ^docs/conf.py additional_dependencies: - - types-dataclasses >= 0.1.3 - types-PyYAML - tomli >= 0.2.6, < 2.0.0 - - types-typed-ast >= 1.4.1 - click >= 8.1.0, != 8.1.4 - packaging >= 22.0 - platformdirs >= 2.1.0 From 8d2110320bef37534997af47edff243d1ec2720b Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Tue, 11 Jul 2023 07:35:41 -0700 Subject: [PATCH 058/279] Fix lint in test_ipynb (#3781) Unblocks #3780 --- tests/test_ipynb.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index 91e7901125b..a74f8ad5690 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -77,7 +77,7 @@ def test_trailing_semicolon_noop() -> None: [ pytest.param(JUPYTER_MODE, id="default mode"), pytest.param( - replace(JUPYTER_MODE, python_cell_magics={"cust1", "cust1"}), + replace(JUPYTER_MODE, python_cell_magics={"cust1", "cust2"}), id="custom cell magics mode", ), ], @@ -100,7 +100,7 @@ def test_cell_magic_noop() -> None: [ pytest.param(JUPYTER_MODE, id="default mode"), pytest.param( - replace(JUPYTER_MODE, python_cell_magics={"cust1", "cust1"}), + replace(JUPYTER_MODE, python_cell_magics={"cust1", "cust2"}), id="custom cell magics mode", ), ], @@ -183,7 +183,7 @@ def test_cell_magic_with_magic() -> None: id="No change when cell magic not registered", ), pytest.param( - replace(JUPYTER_MODE, python_cell_magics={"cust1", "cust1"}), + replace(JUPYTER_MODE, python_cell_magics={"cust1", "cust2"}), "%%custom_python_magic -n1 -n2\nx=2", pytest.raises(NothingChanged), id="No change when other cell magics registered", From 37895f8e50486a0fa581f8fb039e536dc6d0d0e4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 11 Jul 2023 07:50:52 -0700 Subject: [PATCH 059/279] [pre-commit.ci] pre-commit autoupdate (#3780) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pycqa/flake8: 4.0.1 → 6.0.0](https://github.com/pycqa/flake8/compare/4.0.1...6.0.0) - [github.com/pre-commit/mirrors-prettier: v2.7.1 → v3.0.0](https://github.com/pre-commit/mirrors-prettier/compare/v2.7.1...v3.0.0) - [github.com/pre-commit/pre-commit-hooks: v4.3.0 → v4.4.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.3.0...v4.4.0) --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3561df4f904..70b4cd82532 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: - id: isort - repo: https://github.com/pycqa/flake8 - rev: 4.0.1 + rev: 6.0.0 hooks: - id: flake8 additional_dependencies: @@ -52,13 +52,13 @@ repos: - hypothesis - repo: https://github.com/pre-commit/mirrors-prettier - rev: v2.7.1 + rev: v3.0.0 hooks: - id: prettier exclude: \.github/workflows/diff_shades\.yml - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.4.0 hooks: - id: end-of-file-fixer - id: trailing-whitespace From 6123b4ac2696116090fee3da77e9be66417980dd Mon Sep 17 00:00:00 2001 From: rax <133822160+kotnen@users.noreply.github.com> Date: Tue, 11 Jul 2023 14:16:43 -0500 Subject: [PATCH 060/279] Document shebang comment behaviour (#3787) --- docs/the_black_code_style/current_style.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/the_black_code_style/current_style.md b/docs/the_black_code_style/current_style.md index 0fb59fe5aae..e1a8078bf2c 100644 --- a/docs/the_black_code_style/current_style.md +++ b/docs/the_black_code_style/current_style.md @@ -254,11 +254,12 @@ required due to an inner function starting immediately after. _Black_ does not format comment contents, but it enforces two spaces between code and a comment on the same line, and a space before the comment text begins. Some types of -comments that require specific spacing rules are respected: doc comments (`#: comment`), -section comments with long runs of hashes, and Spyder cells. Non-breaking spaces after -hashes are also preserved. Comments may sometimes be moved because of formatting -changes, which can break tools that assign special meaning to them. See -[AST before and after formatting](#ast-before-and-after-formatting) for more discussion. +comments that require specific spacing rules are respected: shebangs (`#! comment`), doc +comments (`#: comment`), section comments with long runs of hashes, and Spyder cells. +Non-breaking spaces after hashes are also preserved. Comments may sometimes be moved +because of formatting changes, which can break tools that assign special meaning to +them. See [AST before and after formatting](#ast-before-and-after-formatting) for more +discussion. ### Trailing commas From 0e26ada66d16d2aea97bda5f907bb0b20b0985e7 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 16 Jul 2023 17:35:19 -0700 Subject: [PATCH 061/279] Continue to avoid Click typing issue (#3791) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 70b4cd82532..89c0de39c86 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -45,7 +45,7 @@ repos: additional_dependencies: - types-PyYAML - tomli >= 0.2.6, < 2.0.0 - - click >= 8.1.0, != 8.1.4 + - click >= 8.1.0, != 8.1.4, != 8.1.5 - packaging >= 22.0 - platformdirs >= 2.1.0 - pytest From 92e0f5b96500459b232a927fb26b0c990800b586 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Robert?= Date: Mon, 17 Jul 2023 03:09:26 +0200 Subject: [PATCH 062/279] Avoid importing `IPython` if notebook cells do not contain magics (#3782) Co-authored-by: hauntsaninja Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- CHANGES.md | 2 ++ src/black/__init__.py | 2 +- src/black/files.py | 2 +- src/black/handle_ipynb_magics.py | 31 ++++++++++++------------------- tests/test_ipynb.py | 12 ++++-------- 5 files changed, 20 insertions(+), 29 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c61ee698c5d..709c767b329 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -30,6 +30,8 @@ +- Avoid importing `IPython` if notebook cells do not contain magics (#3782) + ### Output diff --git a/src/black/__init__.py b/src/black/__init__.py index 301c18f7338..923a51867b5 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -668,7 +668,7 @@ def get_sources( p = Path(f"{STDIN_PLACEHOLDER}{str(p)}") if p.suffix == ".ipynb" and not jupyter_dependencies_are_installed( - verbose=verbose, quiet=quiet + warn=verbose or not quiet ): continue diff --git a/src/black/files.py b/src/black/files.py index ef6895ee3af..368e4170d47 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -384,7 +384,7 @@ def gen_python_files( elif child.is_file(): if child.suffix == ".ipynb" and not jupyter_dependencies_are_installed( - verbose=verbose, quiet=quiet + warn=verbose or not quiet ): continue include_match = include.search(normalized_path) if include else True diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index 2a2d62220e2..55ef2267df8 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -6,6 +6,7 @@ import secrets import sys from functools import lru_cache +from importlib.util import find_spec from typing import Dict, List, Optional, Tuple if sys.version_info >= (3, 10): @@ -56,25 +57,17 @@ class Replacement: @lru_cache -def jupyter_dependencies_are_installed(*, verbose: bool, quiet: bool) -> bool: - try: - # isort: off - # tokenize_rt is less commonly installed than IPython - # and IPython is expensive to import - import tokenize_rt # noqa:F401 - import IPython # noqa:F401 - - # isort: on - except ModuleNotFoundError: - if verbose or not quiet: - msg = ( - "Skipping .ipynb files as Jupyter dependencies are not installed.\n" - 'You can fix this by running ``pip install "black[jupyter]"``' - ) - out(msg) - return False - else: - return True +def jupyter_dependencies_are_installed(*, warn: bool) -> bool: + installed = ( + find_spec("tokenize_rt") is not None and find_spec("IPython") is not None + ) + if not installed and warn: + msg = ( + "Skipping .ipynb files as Jupyter dependencies are not installed.\n" + 'You can fix this by running ``pip install "black[jupyter]"``' + ) + out(msg) + return installed def remove_trailing_semicolon(src: str) -> Tuple[str, bool]: diff --git a/tests/test_ipynb.py b/tests/test_ipynb.py index a74f8ad5690..59897190304 100644 --- a/tests/test_ipynb.py +++ b/tests/test_ipynb.py @@ -440,17 +440,13 @@ def test_cache_isnt_written_if_no_jupyter_deps_single( nb = get_case_path("jupyter", "notebook_trailing_newline.ipynb") tmp_nb = tmp_path / "notebook.ipynb" tmp_nb.write_bytes(nb.read_bytes()) - monkeypatch.setattr( - "black.jupyter_dependencies_are_installed", lambda verbose, quiet: False - ) + monkeypatch.setattr("black.jupyter_dependencies_are_installed", lambda warn: False) result = runner.invoke( main, [str(tmp_path / "notebook.ipynb"), f"--config={EMPTY_CONFIG}"] ) assert "No Python files are present to be formatted. Nothing to do" in result.output jupyter_dependencies_are_installed.cache_clear() - monkeypatch.setattr( - "black.jupyter_dependencies_are_installed", lambda verbose, quiet: True - ) + monkeypatch.setattr("black.jupyter_dependencies_are_installed", lambda warn: True) result = runner.invoke( main, [str(tmp_path / "notebook.ipynb"), f"--config={EMPTY_CONFIG}"] ) @@ -466,13 +462,13 @@ def test_cache_isnt_written_if_no_jupyter_deps_dir( tmp_nb = tmp_path / "notebook.ipynb" tmp_nb.write_bytes(nb.read_bytes()) monkeypatch.setattr( - "black.files.jupyter_dependencies_are_installed", lambda verbose, quiet: False + "black.files.jupyter_dependencies_are_installed", lambda warn: False ) result = runner.invoke(main, [str(tmp_path), f"--config={EMPTY_CONFIG}"]) assert "No Python files are present to be formatted. Nothing to do" in result.output jupyter_dependencies_are_installed.cache_clear() monkeypatch.setattr( - "black.files.jupyter_dependencies_are_installed", lambda verbose, quiet: True + "black.files.jupyter_dependencies_are_installed", lambda warn: True ) result = runner.invoke(main, [str(tmp_path), f"--config={EMPTY_CONFIG}"]) assert "reformatted" in result.output From 8d80aecd50ea55a817807ae2d5174ccedaf12ecb Mon Sep 17 00:00:00 2001 From: Richard Si Date: Sun, 16 Jul 2023 21:16:12 -0400 Subject: [PATCH 063/279] Maintainers += Shantanu Jain (hauntsaninja) (#3792) --- AUTHORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.md b/AUTHORS.md index ab3f30b8821..e0511bb9b7c 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -13,6 +13,7 @@ Maintained with: - [Richard Si](mailto:sichard26@gmail.com) - [Felix Hildén](mailto:felix.hilden@gmail.com) - [Batuhan Taskaya](mailto:batuhan@python.org) +- [Shantanu Jain](mailto:hauntsaninja@gmail.com) Multiple contributions by: From c1e30d97fe39e4c1b1967571b7e3854547239bf6 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 16 Jul 2023 21:33:58 -0700 Subject: [PATCH 064/279] Fix most blib2to3 lint (#3794) --- .pre-commit-config.yaml | 3 ++- pyproject.toml | 5 ++--- src/blib2to3/README | 13 +++++++------ src/blib2to3/pgen2/driver.py | 23 +++++++---------------- src/blib2to3/pgen2/literals.py | 2 -- src/blib2to3/pgen2/parse.py | 27 +++++++++++++++------------ src/blib2to3/pgen2/pgen.py | 25 +++++++++++-------------- src/blib2to3/pgen2/token.py | 4 +--- src/blib2to3/pgen2/tokenize.py | 29 ++++++++++++++++++++--------- src/blib2to3/pygram.py | 2 -- src/blib2to3/pytree.py | 11 +++-------- 11 files changed, 68 insertions(+), 76 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 89c0de39c86..0d68b81ccd7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ # Note: don't use this config for your own repositories. Instead, see # "Version control integration" in docs/integrations/source_version_control.md -exclude: ^(src/blib2to3/|profiling/|tests/data/) +exclude: ^(profiling/|tests/data/) repos: - repo: local hooks: @@ -36,6 +36,7 @@ repos: - flake8-bugbear - flake8-comprehensions - flake8-simplify + exclude: ^src/blib2to3/ - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.4.1 diff --git a/pyproject.toml b/pyproject.toml index aaac42b44b9..d29b768c289 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,8 +12,7 @@ include = '\.pyi?$' extend-exclude = ''' /( # The following are specific to Black, you probably don't want those. - | blib2to3 - | tests/data + tests/data | profiling )/ ''' @@ -183,7 +182,7 @@ atomic = true profile = "black" line_length = 88 skip_gitignore = true -skip_glob = ["src/blib2to3", "tests/data", "profiling"] +skip_glob = ["tests/data", "profiling"] known_first_party = ["black", "blib2to3", "blackd", "_black_version"] [tool.pytest.ini_options] diff --git a/src/blib2to3/README b/src/blib2to3/README index 0d3c607c9c7..38b04158ddb 100644 --- a/src/blib2to3/README +++ b/src/blib2to3/README @@ -1,18 +1,19 @@ -A subset of lib2to3 taken from Python 3.7.0b2. -Commit hash: 9c17e3a1987004b8bcfbe423953aad84493a7984 +A subset of lib2to3 taken from Python 3.7.0b2. Commit hash: +9c17e3a1987004b8bcfbe423953aad84493a7984 Reasons for forking: + - consistent handling of f-strings for users of Python < 3.6.2 -- backport of BPO-33064 that fixes parsing files with trailing commas after - *args and **kwargs -- backport of GH-6143 that restores the ability to reformat legacy usage of - `async` +- backport of BPO-33064 that fixes parsing files with trailing commas after \*args and + \*\*kwargs +- backport of GH-6143 that restores the ability to reformat legacy usage of `async` - support all types of string literals - better ability to debug (better reprs) - INDENT and DEDENT don't hold whitespace and comment prefixes - ability to Cythonize Change Log: + - Changes default logger used by Driver - Backported the following upstream parser changes: - "bpo-42381: Allow walrus in set literals and set comprehensions (GH-23332)" diff --git a/src/blib2to3/pgen2/driver.py b/src/blib2to3/pgen2/driver.py index bb73016a4c1..e629843f8b9 100644 --- a/src/blib2to3/pgen2/driver.py +++ b/src/blib2to3/pgen2/driver.py @@ -17,30 +17,21 @@ # Python imports import io -import os import logging +import os import pkgutil import sys -from typing import ( - Any, - cast, - IO, - Iterable, - List, - Optional, - Iterator, - Tuple, - Union, -) from contextlib import contextmanager from dataclasses import dataclass, field - -# Pgen imports -from . import grammar, parse, token, tokenize, pgen from logging import Logger -from blib2to3.pytree import NL +from typing import IO, Any, Iterable, Iterator, List, Optional, Tuple, Union, cast + from blib2to3.pgen2.grammar import Grammar from blib2to3.pgen2.tokenize import GoodTokenInfo +from blib2to3.pytree import NL + +# Pgen imports +from . import grammar, parse, pgen, token, tokenize Path = Union[str, "os.PathLike[str]"] diff --git a/src/blib2to3/pgen2/literals.py b/src/blib2to3/pgen2/literals.py index c67b91d0463..53c0b8ac2bb 100644 --- a/src/blib2to3/pgen2/literals.py +++ b/src/blib2to3/pgen2/literals.py @@ -4,10 +4,8 @@ """Safely evaluate Python string literals without using eval().""" import re - from typing import Dict, Match - simple_escapes: Dict[str, str] = { "a": "\a", "b": "\b", diff --git a/src/blib2to3/pgen2/parse.py b/src/blib2to3/pgen2/parse.py index 17bf118e9fc..299cc24a15f 100644 --- a/src/blib2to3/pgen2/parse.py +++ b/src/blib2to3/pgen2/parse.py @@ -10,24 +10,25 @@ """ from contextlib import contextmanager - -# Local imports -from . import grammar, token, tokenize from typing import ( - cast, + TYPE_CHECKING, Any, - Optional, - Union, - Tuple, + Callable, Dict, - List, Iterator, - Callable, + List, + Optional, Set, - TYPE_CHECKING, + Tuple, + Union, + cast, ) + from blib2to3.pgen2.grammar import Grammar -from blib2to3.pytree import convert, NL, Context, RawNode, Leaf, Node +from blib2to3.pytree import NL, Context, Leaf, Node, RawNode, convert + +# Local imports +from . import grammar, token, tokenize if TYPE_CHECKING: from blib2to3.pgen2.driver import TokenProxy @@ -112,7 +113,9 @@ def add_token(self, tok_type: int, tok_val: str, raw: bool = False) -> None: args.insert(0, ilabel) func(*args) - def determine_route(self, value: Optional[str] = None, force: bool = False) -> Optional[int]: + def determine_route( + self, value: Optional[str] = None, force: bool = False + ) -> Optional[int]: alive_ilabels = self.ilabels if len(alive_ilabels) == 0: *_, most_successful_ilabel = self._dead_ilabels diff --git a/src/blib2to3/pgen2/pgen.py b/src/blib2to3/pgen2/pgen.py index 046efd09338..3ece9bb41ed 100644 --- a/src/blib2to3/pgen2/pgen.py +++ b/src/blib2to3/pgen2/pgen.py @@ -1,25 +1,22 @@ # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. # Licensed to PSF under a Contributor Agreement. -# Pgen imports -from . import grammar, token, tokenize - +import os from typing import ( + IO, Any, Dict, - IO, Iterator, List, + NoReturn, Optional, + Sequence, Tuple, Union, - Sequence, - NoReturn, ) -from blib2to3.pgen2 import grammar -from blib2to3.pgen2.tokenize import GoodTokenInfo -import os +from blib2to3.pgen2 import grammar, token, tokenize +from blib2to3.pgen2.tokenize import GoodTokenInfo Path = Union[str, "os.PathLike[str]"] @@ -149,7 +146,7 @@ def calcfirst(self, name: str) -> None: state = dfa[0] totalset: Dict[str, int] = {} overlapcheck = {} - for label, next in state.arcs.items(): + for label in state.arcs: if label in self.dfas: if label in self.first: fset = self.first[label] @@ -190,9 +187,9 @@ def parse(self) -> Tuple[Dict[str, List["DFAState"]], str]: # self.dump_nfa(name, a, z) dfa = self.make_dfa(a, z) # self.dump_dfa(name, dfa) - oldlen = len(dfa) + # oldlen = len(dfa) self.simplify_dfa(dfa) - newlen = len(dfa) + # newlen = len(dfa) dfas[name] = dfa # print name, oldlen, newlen if startsymbol is None: @@ -346,7 +343,7 @@ def parse_atom(self) -> Tuple["NFAState", "NFAState"]: self.raise_error( "expected (...) or NAME or STRING, got %s/%s", self.type, self.value ) - assert False + raise AssertionError def expect(self, type: int, value: Optional[Any] = None) -> str: if self.type != type or (value is not None and self.value != value): @@ -368,7 +365,7 @@ def raise_error(self, msg: str, *args: Any) -> NoReturn: if args: try: msg = msg % args - except: + except Exception: msg = " ".join([msg] + list(map(str, args))) raise SyntaxError(msg, (self.filename, self.end[0], self.end[1], self.line)) diff --git a/src/blib2to3/pgen2/token.py b/src/blib2to3/pgen2/token.py index 117cc09d4ce..ed2fc4e85fc 100644 --- a/src/blib2to3/pgen2/token.py +++ b/src/blib2to3/pgen2/token.py @@ -1,8 +1,6 @@ """Token constants (from "token.h").""" -from typing import Dict - -from typing import Final +from typing import Dict, Final # Taken from Python (r53757) and modified to include some tokens # originally monkeypatched in by pgen2.tokenize diff --git a/src/blib2to3/pgen2/tokenize.py b/src/blib2to3/pgen2/tokenize.py index 1dea89d7bb8..d0607f4b1e1 100644 --- a/src/blib2to3/pgen2/tokenize.py +++ b/src/blib2to3/pgen2/tokenize.py @@ -30,28 +30,41 @@ import sys from typing import ( Callable, + Final, Iterable, Iterator, List, Optional, + Pattern, Set, Tuple, - Pattern, Union, cast, ) -from typing import Final - -from blib2to3.pgen2.token import * from blib2to3.pgen2.grammar import Grammar +from blib2to3.pgen2.token import ( + ASYNC, + AWAIT, + COMMENT, + DEDENT, + ENDMARKER, + ERRORTOKEN, + INDENT, + NAME, + NEWLINE, + NL, + NUMBER, + OP, + STRING, + tok_name, +) __author__ = "Ka-Ping Yee " __credits__ = "GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro" import re from codecs import BOM_UTF8, lookup -from blib2to3.pgen2.token import * from . import token @@ -334,7 +347,7 @@ def read_or_stop() -> bytes: try: return readline() except StopIteration: - return b'' + return b"" def find_cookie(line: bytes) -> Optional[str]: try: @@ -676,14 +689,12 @@ def generate_tokens( yield stashed stashed = None - for indent in indents[1:]: # pop remaining indent levels + for _indent in indents[1:]: # pop remaining indent levels yield (DEDENT, "", (lnum, 0), (lnum, 0), "") yield (ENDMARKER, "", (lnum, 0), (lnum, 0), "") if __name__ == "__main__": # testing - import sys - if len(sys.argv) > 1: tokenize(open(sys.argv[1]).readline) else: diff --git a/src/blib2to3/pygram.py b/src/blib2to3/pygram.py index 1b4832362bf..c30c630e816 100644 --- a/src/blib2to3/pygram.py +++ b/src/blib2to3/pygram.py @@ -5,12 +5,10 @@ # Python imports import os - from typing import Union # Local imports from .pgen2 import driver - from .pgen2.grammar import Grammar # Moved into initialize because mypyc can't handle __file__ (XXX bug) diff --git a/src/blib2to3/pytree.py b/src/blib2to3/pytree.py index 156322cab7e..2a0cd6d196a 100644 --- a/src/blib2to3/pytree.py +++ b/src/blib2to3/pytree.py @@ -15,15 +15,16 @@ from typing import ( Any, Dict, + Iterable, Iterator, List, Optional, + Set, Tuple, TypeVar, Union, - Set, - Iterable, ) + from blib2to3.pgen2.grammar import Grammar __author__ = "Guido van Rossum " @@ -58,7 +59,6 @@ def type_repr(type_num: int) -> Union[str, int]: class Base: - """ Abstract base class for Node and Leaf. @@ -237,7 +237,6 @@ def get_suffix(self) -> str: class Node(Base): - """Concrete implementation for interior nodes.""" fixers_applied: Optional[List[Any]] @@ -378,7 +377,6 @@ def update_sibling_maps(self) -> None: class Leaf(Base): - """Concrete implementation for leaf nodes.""" # Default values for instance variables @@ -506,7 +504,6 @@ def convert(gr: Grammar, raw_node: RawNode) -> NL: class BasePattern: - """ A pattern is a tree matching pattern. @@ -646,7 +643,6 @@ def _submatch(self, node, results=None): class NodePattern(BasePattern): - wildcards: bool = False def __init__( @@ -715,7 +711,6 @@ def _submatch(self, node, results=None) -> bool: class WildcardPattern(BasePattern): - """ A wildcard pattern can match zero or more nodes. From 068f6fb8fa2b52f647aec8696033e43f6b0db70b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 16 Jul 2023 23:59:36 -0700 Subject: [PATCH 065/279] Bump pypa/cibuildwheel from 2.13.1 to 2.14.1 (#3795) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pypi_upload.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index 06600fcbc45..291193efc7a 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -58,7 +58,7 @@ jobs: - uses: actions/checkout@v3 - name: Build wheels via cibuildwheel - uses: pypa/cibuildwheel@v2.13.1 + uses: pypa/cibuildwheel@v2.14.1 env: CIBW_ARCHS_MACOS: "${{ matrix.macos_arch }}" From 2f68ac850b5b5b8e955110112f841121b76effa4 Mon Sep 17 00:00:00 2001 From: Richard Si Date: Tue, 18 Jul 2023 10:51:16 -0400 Subject: [PATCH 066/279] Fix diff-shades comment missing newlines (#3799) Preserving newlines is done differently when writing to $GITHUB_OUTPUT over the deprecated :set-output: command. --- scripts/diff_shades_gha_helper.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/diff_shades_gha_helper.py b/scripts/diff_shades_gha_helper.py index 994fbe05045..7a58fbe9b28 100644 --- a/scripts/diff_shades_gha_helper.py +++ b/scripts/diff_shades_gha_helper.py @@ -21,6 +21,7 @@ import subprocess import sys import zipfile +from base64 import b64encode from io import BytesIO from pathlib import Path from typing import Any @@ -53,12 +54,16 @@ def set_output(name: str, value: str) -> None: else: print(f"[INFO]: setting '{name}' to [{len(value)} chars]") - # Originally the `set-output` workflow command was used here, now replaced - # by setting variables through the `GITHUB_OUTPUT` environment variable - # to stay up to date with GitHub's update. if "GITHUB_OUTPUT" in os.environ: + if "\n" in value: + # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings + delimiter = b64encode(os.urandom(16)).decode() + value = f"{delimiter}\n{value}\n{delimiter}" + command = f"{name}<<{value}" + else: + command = f"{name}={value}" with open(os.environ["GITHUB_OUTPUT"], "a") as f: - print(f"{name}={value}", file=f) + print(command, file=f) def http_get(url: str, *, is_json: bool = True, **kwargs: Any) -> Any: @@ -224,9 +229,7 @@ def comment_details(run_id: str) -> None: # while it's still in progress seems impossible). body = body.replace("$workflow-run-url", data["html_url"]) body = body.replace("$job-diff-url", diff_url) - # https://github.community/t/set-output-truncates-multiline-strings/16852/3 - escaped = body.replace("%", "%25").replace("\n", "%0A").replace("\r", "%0D") - set_output("comment-body", escaped) + set_output("comment-body", body) if __name__ == "__main__": From 0b301f80954a026693c4c22de89267ad8c85f9b6 Mon Sep 17 00:00:00 2001 From: rdrll <13176405+rdrll@users.noreply.github.com> Date: Tue, 18 Jul 2023 14:11:24 -0700 Subject: [PATCH 067/279] Improvements to contributing docs (#3753) --- docs/contributing/the_basics.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/contributing/the_basics.md b/docs/contributing/the_basics.md index 5fdcdd802bd..40d233257e3 100644 --- a/docs/contributing/the_basics.md +++ b/docs/contributing/the_basics.md @@ -12,7 +12,9 @@ example: ```console $ python3 -m venv .venv -$ source .venv/bin/activate +$ source .venv/bin/activate # activation for linux and mac +$ .venv\Scripts\activate # activation for windows + (.venv)$ pip install -r test_requirements.txt (.venv)$ pip install -e .[d] (.venv)$ pre-commit install @@ -30,6 +32,9 @@ the root of the black repo: # Optional Fuzz testing (.venv)$ tox -e fuzz + +# Format Black itself +(.venv)$ tox -e run_self ``` ### News / Changelog Requirement @@ -62,7 +67,7 @@ If you make changes to docs, you can test they still build locally too. ```console (.venv)$ pip install -r docs/requirements.txt -(.venv)$ pip install [-e] .[d] +(.venv)$ pip install -e .[d] (.venv)$ sphinx-build -a -b html -W docs/ docs/_build/ ``` From e7e8d6287b38db3f15bdf3d4ec6987d4490b8d14 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sat, 22 Jul 2023 08:49:51 -0700 Subject: [PATCH 068/279] Simplify empty line tracker (#3797) --- src/black/lines.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/black/lines.py b/src/black/lines.py index ea8fe520756..016a489310d 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -49,7 +49,7 @@ class Line: """Holds leaves and comments. Can be printed with `str(line)`.""" - mode: Mode + mode: Mode = field(repr=False) depth: int = 0 leaves: List[Leaf] = field(default_factory=list) # keys ordered like `leaves` @@ -579,16 +579,21 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: else: before = 0 depth = current_line.depth + + previous_def = None while self.previous_defs and self.previous_defs[-1].depth >= depth: + previous_def = self.previous_defs.pop() + + if previous_def is not None: + assert self.previous_line is not None if self.mode.is_pyi: - assert self.previous_line is not None if depth and not current_line.is_def and self.previous_line.is_def: # Empty lines between attributes and methods should be preserved. before = min(1, before) elif ( Preview.blank_line_after_nested_stub_class in self.mode - and self.previous_defs[-1].is_class - and not self.previous_defs[-1].is_stub_class + and previous_def.is_class + and not previous_def.is_stub_class ): before = 1 elif depth: @@ -600,7 +605,7 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: before = 1 elif ( not depth - and self.previous_defs[-1].depth + and previous_def.depth and current_line.leaves[-1].type == token.COLON and ( current_line.leaves[0].value @@ -617,7 +622,7 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: before = 1 else: before = 2 - self.previous_defs.pop() + if current_line.is_decorator or current_line.is_def or current_line.is_class: return self._maybe_empty_lines_for_class_or_def(current_line, before) From 13bd4fffae0b95b0a1f55d335dd55a1de7de3d10 Mon Sep 17 00:00:00 2001 From: mihazagar Date: Sat, 22 Jul 2023 20:12:37 +0200 Subject: [PATCH 069/279] Fixing pre-commit using pyyaml with broken version (#3804) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0d68b81ccd7..10e65316e82 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,7 +12,7 @@ repos: additional_dependencies: &version_check_dependencies [ commonmark==0.9.1, - pyyaml==5.4.1, + pyyaml==6.0.1, beautifulsoup4==4.9.3, ] From c3235e6da7259394cd0c00fe36c3e089fbae1e4f Mon Sep 17 00:00:00 2001 From: Pradeep Kumar Date: Sun, 23 Jul 2023 21:56:19 -0700 Subject: [PATCH 070/279] Fix unintentionally swapped words in index.md (#3809) Fix unintentionally swapped words in index.md I think the intent was to say "large changes in formatting", because it doesn't make sense to say "large formatting in changes". --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index 9d0db465022..49a44ecca5a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -21,7 +21,7 @@ Try it out now using the [Black Playground](https://black.vercel.app). *Black* is [successfully used](https://github.com/psf/black#used-by) by many projects, small and big. *Black* has a comprehensive test suite, with efficient parallel tests, our own auto formatting and parallel Continuous Integration runner. -Now that we have become stable, you should not expect large formatting to changes in +Now that we have become stable, you should not expect large changes to formatting in the future. Stylistic changes will mostly be responses to bug reports and support for new Python syntax. From d9d0a02d89207f712a40b6dabee708389208e558 Mon Sep 17 00:00:00 2001 From: "Yury V. Zaytsev" Date: Thu, 27 Jul 2023 16:12:38 +0200 Subject: [PATCH 071/279] Fix typo in `target-version` param wrongly used in plural (#3817) --- docs/usage_and_configuration/the_basics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index f5862edccaa..5efb50a9a12 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -63,7 +63,7 @@ $ black -t py37 -t py38 -t py39 -t py310 In a [configuration file](#configuration-via-a-file), you can write: ```toml -target-versions = ["py37", "py38", "py39", "py310"] +target-version = ["py37", "py38", "py39", "py310"] ``` _Black_ uses this option to decide what grammar to use to parse your code. In addition, From 133af572072bf7bc92c23a609773c2ea66e483b7 Mon Sep 17 00:00:00 2001 From: freddiewanah Date: Fri, 28 Jul 2023 02:51:28 +1000 Subject: [PATCH 072/279] Rewrite mostly useless assert in test_trans.py (#3810) This PR updates an assert statement that checks the bounds of a string-slicing operation. The updated assertion provides more accurate and informative error handling by specifically checking the relative values of the indices and the string length. The original assertion was essentially checking if Python's string slicing was behaving as expected. However, it wasn't providing any guarantees or useful information about the bounds i and j themselves. The updated assertion checks that the indices used for slicing are within the bounds of the string. It will throw an AssertionError if the indices are out of bounds or if i > j, providing a more specific and informative error. --- tests/test_trans.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_trans.py b/tests/test_trans.py index dce8a939677..784e852e12a 100644 --- a/tests/test_trans.py +++ b/tests/test_trans.py @@ -13,7 +13,7 @@ def check( # a glance than only spans assert len(spans) == len(expected_slices) for (i, j), slice in zip(spans, expected_slices): - assert len(string[i:j]) == j - i + assert 0 <= i <= j <= len(string) assert string[i:j] == slice assert spans == expected_spans From 1a972e3e11b144912155babdf48ff23d68059d57 Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Thu, 27 Jul 2023 17:50:51 -0400 Subject: [PATCH 073/279] Add Lyft to organizations using black (#3818) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b12ddfb1290..9d0b29af215 100644 --- a/README.md +++ b/README.md @@ -137,8 +137,8 @@ SQLAlchemy, Poetry, PyPA applications (Warehouse, Bandersnatch, Pipenv, virtuale pandas, Pillow, Twisted, LocalStack, every Datadog Agent Integration, Home Assistant, Zulip, Kedro, OpenOA, FLORIS, ORBIT, WOMBAT, and many more. -The following organizations use _Black_: Facebook, Dropbox, KeepTruckin, Mozilla, Quora, -Duolingo, QuantumBlack, Tesla, Archer Aviation. +The following organizations use _Black_: Facebook, Dropbox, KeepTruckin, Lyft, Mozilla, +Quora, Duolingo, QuantumBlack, Tesla, Archer Aviation. Are we missing anyone? Let us know. From 8a16b25fb1145e5b7de9c322e52167e8f6a59c79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Jul 2023 08:43:32 -0700 Subject: [PATCH 074/279] Bump furo from 2023.5.20 to 2023.7.26 in /docs (#3824) Bumps [furo](https://github.com/pradyunsg/furo) from 2023.5.20 to 2023.7.26. - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2023.05.20...2023.07.26) --- updated-dependencies: - dependency-name: furo dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index f1b47c69413..ff179f3805e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,4 +6,4 @@ Sphinx==6.1.3 docutils==0.19 sphinxcontrib-programoutput==0.17 sphinx_copybutton==0.5.2 -furo==2023.5.20 +furo==2023.7.26 From 1b028cc9d99c2c2e82f9b727742539173a92a373 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 1 Aug 2023 01:48:21 -0700 Subject: [PATCH 075/279] [pre-commit.ci] pre-commit autoupdate (#3825) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 10e65316e82..5430eef9180 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,7 +29,7 @@ repos: - id: isort - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 + rev: 6.1.0 hooks: - id: flake8 additional_dependencies: From 59e8936768889f583488df609c45302da8e88507 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 3 Aug 2023 18:46:08 -0700 Subject: [PATCH 076/279] Document pre-commit mirror (#3828) --- .pre-commit-hooks.yaml | 2 ++ CHANGES.md | 5 +++++ docs/integrations/source_version_control.md | 13 ++++++++----- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 137957045a6..a1ff41fded8 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -1,3 +1,5 @@ +# Note that we recommend using https://github.com/psf/black-pre-commit-mirror instead +# This will work about 2x as fast as using the hooks in this repository - id: black name: black description: "Black: The uncompromising Python code formatter" diff --git a/CHANGES.md b/CHANGES.md index 709c767b329..1de08792f75 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -44,6 +44,11 @@ +- Black now has an + [official pre-commit mirror](https://github.com/psf/black-pre-commit-mirror). Swapping + `https://github.com/psf/black` to `https://github.com/psf/black-pre-commit-mirror` in + your `.pre-commit-config.yaml` will make Black about 2x faster (#3828) + ### Documentation +- More concise formatting for dummy implementations (#3796) + ### Configuration diff --git a/src/black/linegen.py b/src/black/linegen.py index 5ef3bbd1705..507e860190f 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -281,7 +281,9 @@ def visit_match_case(self, node: Node) -> Iterator[Line]: def visit_suite(self, node: Node) -> Iterator[Line]: """Visit a suite.""" - if self.mode.is_pyi and is_stub_suite(node): + if ( + self.mode.is_pyi or Preview.dummy_implementations in self.mode + ) and is_stub_suite(node): yield from self.visit(node.children[2]) else: yield from self.visit_default(node) @@ -296,7 +298,9 @@ def visit_simple_stmt(self, node: Node) -> Iterator[Line]: is_suite_like = node.parent and node.parent.type in STATEMENT if is_suite_like: - if self.mode.is_pyi and is_stub_body(node): + if ( + self.mode.is_pyi or Preview.dummy_implementations in self.mode + ) and is_stub_body(node): yield from self.visit_default(node) else: yield from self.line(+1) @@ -305,7 +309,7 @@ def visit_simple_stmt(self, node: Node) -> Iterator[Line]: else: if ( - not self.mode.is_pyi + not (self.mode.is_pyi or Preview.dummy_implementations in self.mode) or not node.parent or not is_stub_suite(node.parent) ): diff --git a/src/black/lines.py b/src/black/lines.py index 016a489310d..0a307b45eff 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -165,6 +165,13 @@ def is_def(self) -> bool: and second_leaf.value == "def" ) + @property + def is_stub_def(self) -> bool: + """Is this line a function definition with a body consisting only of "..."?""" + return self.is_def and self.leaves[-4:] == [Leaf(token.COLON, ":")] + [ + Leaf(token.DOT, ".") for _ in range(3) + ] + @property def is_class_paren_empty(self) -> bool: """Is this a class with no base classes but using parentheses? @@ -578,6 +585,8 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: first_leaf.prefix = "" else: before = 0 + + user_had_newline = bool(before) depth = current_line.depth previous_def = None @@ -589,7 +598,7 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: if self.mode.is_pyi: if depth and not current_line.is_def and self.previous_line.is_def: # Empty lines between attributes and methods should be preserved. - before = min(1, before) + before = 1 if user_had_newline else 0 elif ( Preview.blank_line_after_nested_stub_class in self.mode and previous_def.is_class @@ -624,7 +633,9 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: before = 2 if current_line.is_decorator or current_line.is_def or current_line.is_class: - return self._maybe_empty_lines_for_class_or_def(current_line, before) + return self._maybe_empty_lines_for_class_or_def( + current_line, before, user_had_newline + ) if ( self.previous_line @@ -648,8 +659,8 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: return 0, 0 return before, 0 - def _maybe_empty_lines_for_class_or_def( - self, current_line: Line, before: int + def _maybe_empty_lines_for_class_or_def( # noqa: C901 + self, current_line: Line, before: int, user_had_newline: bool ) -> Tuple[int, int]: if not current_line.is_decorator: self.previous_defs.append(current_line) @@ -715,6 +726,14 @@ def _maybe_empty_lines_for_class_or_def( newlines = 0 else: newlines = 1 if current_line.depth else 2 + # If a user has left no space after a dummy implementation, don't insert + # new lines. This is useful for instance for @overload or Protocols. + if ( + Preview.dummy_implementations in self.mode + and self.previous_line.is_stub_def + and not user_had_newline + ): + newlines = 0 if comment_to_add_newlines is not None: previous_block = comment_to_add_newlines.previous_block if previous_block is not None: diff --git a/src/black/mode.py b/src/black/mode.py index 4d979afd84d..282c1669da7 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -182,6 +182,7 @@ class Preview(Enum): skip_magic_trailing_comma_in_subscript = auto() wrap_long_dict_values_in_parens = auto() wrap_multiple_context_managers_in_parens = auto() + dummy_implementations = auto() class Deprecated(UserWarning): diff --git a/tests/data/miscellaneous/force_py36.py b/tests/data/miscellaneous/force_py36.py index cad935e525a..4c9b70336e7 100644 --- a/tests/data/miscellaneous/force_py36.py +++ b/tests/data/miscellaneous/force_py36.py @@ -1,6 +1,6 @@ # The input source must not contain any Py36-specific syntax (e.g. argument type # annotations, trailing comma after *rest) or this test becomes invalid. -def long_function_name(argument_one, argument_two, argument_three, argument_four, argument_five, argument_six, *rest): ... +def long_function_name(argument_one, argument_two, argument_three, argument_four, argument_five, argument_six, *rest): pass # output # The input source must not contain any Py36-specific syntax (e.g. argument type # annotations, trailing comma after *rest) or this test becomes invalid. @@ -13,4 +13,4 @@ def long_function_name( argument_six, *rest, ): - ... + pass diff --git a/tests/data/preview/comments7.py b/tests/data/preview/comments7.py index ec2dc501d8e..8b1224017e5 100644 --- a/tests/data/preview/comments7.py +++ b/tests/data/preview/comments7.py @@ -278,8 +278,7 @@ class C: ) def test_fails_invalid_post_data( self, pyramid_config, db_request, post_data, message - ): - ... + ): ... square = Square(4) # type: Optional[Square] diff --git a/tests/data/preview/dummy_implementations.py b/tests/data/preview/dummy_implementations.py new file mode 100644 index 00000000000..e07c25ed129 --- /dev/null +++ b/tests/data/preview/dummy_implementations.py @@ -0,0 +1,99 @@ +from typing import NoReturn, Protocol, Union, overload + + +def dummy(a): ... +def other(b): ... + + +@overload +def a(arg: int) -> int: ... +@overload +def a(arg: str) -> str: ... +@overload +def a(arg: object) -> NoReturn: ... +def a(arg: Union[int, str, object]) -> Union[int, str]: + if not isinstance(arg, (int, str)): + raise TypeError + return arg + +class Proto(Protocol): + def foo(self, a: int) -> int: + ... + + def bar(self, b: str) -> str: ... + def baz(self, c: bytes) -> str: + ... + + +def dummy_two(): + ... +@dummy +def dummy_three(): + ... + +def dummy_four(): + ... + +@overload +def b(arg: int) -> int: ... + +@overload +def b(arg: str) -> str: ... +@overload +def b(arg: object) -> NoReturn: ... + +def b(arg: Union[int, str, object]) -> Union[int, str]: + if not isinstance(arg, (int, str)): + raise TypeError + return arg + +# output + +from typing import NoReturn, Protocol, Union, overload + + +def dummy(a): ... +def other(b): ... + + +@overload +def a(arg: int) -> int: ... +@overload +def a(arg: str) -> str: ... +@overload +def a(arg: object) -> NoReturn: ... +def a(arg: Union[int, str, object]) -> Union[int, str]: + if not isinstance(arg, (int, str)): + raise TypeError + return arg + + +class Proto(Protocol): + def foo(self, a: int) -> int: ... + + def bar(self, b: str) -> str: ... + def baz(self, c: bytes) -> str: ... + + +def dummy_two(): ... +@dummy +def dummy_three(): ... + + +def dummy_four(): ... + + +@overload +def b(arg: int) -> int: ... + + +@overload +def b(arg: str) -> str: ... +@overload +def b(arg: object) -> NoReturn: ... + + +def b(arg: Union[int, str, object]) -> Union[int, str]: + if not isinstance(arg, (int, str)): + raise TypeError + return arg From 77f19944f632c48765175cafad07dc76b3810911 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 8 Aug 2023 08:41:39 -0700 Subject: [PATCH 078/279] [pre-commit.ci] pre-commit autoupdate (#3833) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-prettier: v3.0.0 → v3.0.1](https://github.com/pre-commit/mirrors-prettier/compare/v3.0.0...v3.0.1) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5430eef9180..60a092f8b29 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -53,7 +53,7 @@ repos: - hypothesis - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.0 + rev: v3.0.1 hooks: - id: prettier exclude: \.github/workflows/diff_shades\.yml From c36e468794f9256d5e922c399240d49782ba04f1 Mon Sep 17 00:00:00 2001 From: Christian Proud Date: Wed, 9 Aug 2023 02:12:05 +0800 Subject: [PATCH 079/279] Remove ENV_PATH on Black action completion (#3759) --- CHANGES.md | 2 ++ action/main.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index d084498f404..8bf4188606c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -50,6 +50,8 @@ [official pre-commit mirror](https://github.com/psf/black-pre-commit-mirror). Swapping `https://github.com/psf/black` to `https://github.com/psf/black-pre-commit-mirror` in your `.pre-commit-config.yaml` will make Black about 2x faster (#3828) +- The `.black.env` folder specified by `ENV_PATH` will now be removed on the completion + of the GitHub Action. (#3759) ### Documentation diff --git a/action/main.py b/action/main.py index 1911cfd7a01..c0af3930dbb 100644 --- a/action/main.py +++ b/action/main.py @@ -1,5 +1,6 @@ import os import shlex +import shutil import sys from pathlib import Path from subprocess import PIPE, STDOUT, run @@ -73,5 +74,6 @@ stderr=STDOUT, encoding="utf-8", ) +shutil.rmtree(ENV_PATH, ignore_errors=True) print(proc.stdout) sys.exit(proc.returncode) From 66648c528a95553c1f822ece394ac98784baee47 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Aug 2023 00:30:56 -0700 Subject: [PATCH 080/279] Bump pypa/cibuildwheel from 2.14.1 to 2.15.0 (#3836) --- .github/workflows/pypi_upload.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index 291193efc7a..9be231dd305 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -58,7 +58,7 @@ jobs: - uses: actions/checkout@v3 - name: Build wheels via cibuildwheel - uses: pypa/cibuildwheel@v2.14.1 + uses: pypa/cibuildwheel@v2.15.0 env: CIBW_ARCHS_MACOS: "${{ matrix.macos_arch }}" From 7c4fe83bd87ccef21f8c5a0cd5d122c5b004bb15 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:51:26 -0700 Subject: [PATCH 081/279] Make pre-commit do less (#3838) --- .pre-commit-config.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 60a092f8b29..a7ae7761494 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -63,3 +63,6 @@ repos: hooks: - id: end-of-file-fixer - id: trailing-whitespace + +ci: + autoupdate_schedule: quarterly From ade371fd1c7118b8a82b281c28425fefb8cb719e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 16 Aug 2023 00:01:21 -0700 Subject: [PATCH 082/279] [pre-commit.ci] pre-commit autoupdate (#3837) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a7ae7761494..6301526a445 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: exclude: ^src/blib2to3/ - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.4.1 + rev: v1.5.0 hooks: - id: mypy exclude: ^docs/conf.py From 793c2b5f9f7c7ca267fbcab58d30997ac6b9497d Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 18 Aug 2023 18:32:47 -0700 Subject: [PATCH 083/279] Pin tox to fix CI (#3843) --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4bf687435b4..8a139387c5b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,7 +45,7 @@ jobs: - name: Install tox run: | python -m pip install --upgrade pip - python -m pip install --upgrade tox + python -m pip install --upgrade 'tox<4.7' - name: Unit tests if: "!startsWith(matrix.python-version, 'pypy')" From c6a031e623c7991ac9129f578dc21dffe2d7ede3 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sat, 19 Aug 2023 04:26:36 +0200 Subject: [PATCH 084/279] Improve caching by comparing file hashes as fallback for mtime and size (#3821) Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- CHANGES.md | 1 + .../reference/reference_classes.rst | 7 + .../reference/reference_functions.rst | 8 - pyproject.toml | 2 +- src/black/__init__.py | 11 +- src/black/cache.py | 160 +++++++++++------- src/black/concurrency.py | 9 +- tests/test_black.py | 155 +++++++++++------ 8 files changed, 219 insertions(+), 134 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8bf4188606c..a14a55a03ac 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -33,6 +33,7 @@ - Avoid importing `IPython` if notebook cells do not contain magics (#3782) +- Improve caching by comparing file hashes as fallback for mtime and size. (#3821) ### Output diff --git a/docs/contributing/reference/reference_classes.rst b/docs/contributing/reference/reference_classes.rst index 29b25003af2..dc615579e30 100644 --- a/docs/contributing/reference/reference_classes.rst +++ b/docs/contributing/reference/reference_classes.rst @@ -186,6 +186,13 @@ Black Classes :show-inheritance: :members: +:class:`Cache` +------------------------ + +.. autoclass:: black.cache.Cache + :show-inheritance: + :members: + Enum Classes ~~~~~~~~~~~~~ diff --git a/docs/contributing/reference/reference_functions.rst b/docs/contributing/reference/reference_functions.rst index 09517f73961..dd92e37a7d4 100644 --- a/docs/contributing/reference/reference_functions.rst +++ b/docs/contributing/reference/reference_functions.rst @@ -94,18 +94,10 @@ Split functions Caching ------- -.. autofunction:: black.cache.filter_cached - .. autofunction:: black.cache.get_cache_dir .. autofunction:: black.cache.get_cache_file -.. autofunction:: black.cache.get_cache_info - -.. autofunction:: black.cache.read_cache - -.. autofunction:: black.cache.write_cache - Utilities --------- diff --git a/pyproject.toml b/pyproject.toml index d29b768c289..6cd3f34bc10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,7 +67,7 @@ dependencies = [ "pathspec>=0.9.0", "platformdirs>=2", "tomli>=1.1.0; python_version < '3.11'", - "typing_extensions>=3.10.0.0; python_version < '3.10'", + "typing_extensions>=4.0.1; python_version < '3.11'", ] dynamic = ["readme", "version"] diff --git a/src/black/__init__.py b/src/black/__init__.py index 923a51867b5..dc06eab8dd0 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -34,7 +34,7 @@ from pathspec.patterns.gitwildmatch import GitWildMatchPatternError from _black_version import version as __version__ -from black.cache import Cache, get_cache_info, read_cache, write_cache +from black.cache import Cache from black.comments import normalize_fmt_off from black.const import ( DEFAULT_EXCLUDES, @@ -775,12 +775,9 @@ def reformat_one( if format_stdin_to_stdout(fast=fast, write_back=write_back, mode=mode): changed = Changed.YES else: - cache: Cache = {} + cache = Cache.read(mode) if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF): - cache = read_cache(mode) - res_src = src.resolve() - res_src_s = str(res_src) - if res_src_s in cache and cache[res_src_s] == get_cache_info(res_src): + if not cache.is_changed(src): changed = Changed.CACHED if changed is not Changed.CACHED and format_file_in_place( src, fast=fast, write_back=write_back, mode=mode @@ -789,7 +786,7 @@ def reformat_one( if (write_back is WriteBack.YES and changed is not Changed.CACHED) or ( write_back is WriteBack.CHECK and changed is Changed.NO ): - write_cache(cache, [src], mode) + cache.write([src]) report.done(src, changed) except Exception as exc: if report.verbose: diff --git a/src/black/cache.py b/src/black/cache.py index 9455ff44772..ff15da2a94e 100644 --- a/src/black/cache.py +++ b/src/black/cache.py @@ -1,21 +1,28 @@ """Caching of formatted files with feature-based invalidation.""" - +import hashlib import os import pickle +import sys import tempfile +from dataclasses import dataclass, field from pathlib import Path -from typing import Dict, Iterable, Set, Tuple +from typing import Dict, Iterable, NamedTuple, Set, Tuple from platformdirs import user_cache_dir from _black_version import version as __version__ from black.mode import Mode -# types -Timestamp = float -FileSize = int -CacheInfo = Tuple[Timestamp, FileSize] -Cache = Dict[str, CacheInfo] +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self + + +class FileData(NamedTuple): + st_mtime: float + st_size: int + hash: str def get_cache_dir() -> Path: @@ -37,61 +44,92 @@ def get_cache_dir() -> Path: CACHE_DIR = get_cache_dir() -def read_cache(mode: Mode) -> Cache: - """Read the cache if it exists and is well formed. - - If it is not well formed, the call to write_cache later should resolve the issue. - """ - cache_file = get_cache_file(mode) - if not cache_file.exists(): - return {} - - with cache_file.open("rb") as fobj: - try: - cache: Cache = pickle.load(fobj) - except (pickle.UnpicklingError, ValueError, IndexError): - return {} - - return cache - - def get_cache_file(mode: Mode) -> Path: return CACHE_DIR / f"cache.{mode.get_cache_key()}.pickle" -def get_cache_info(path: Path) -> CacheInfo: - """Return the information used to check if a file is already formatted or not.""" - stat = path.stat() - return stat.st_mtime, stat.st_size - - -def filter_cached(cache: Cache, sources: Iterable[Path]) -> Tuple[Set[Path], Set[Path]]: - """Split an iterable of paths in `sources` into two sets. - - The first contains paths of files that modified on disk or are not in the - cache. The other contains paths to non-modified files. - """ - todo, done = set(), set() - for src in sources: - res_src = src.resolve() - if cache.get(str(res_src)) != get_cache_info(res_src): - todo.add(src) - else: - done.add(src) - return todo, done - - -def write_cache(cache: Cache, sources: Iterable[Path], mode: Mode) -> None: - """Update the cache file.""" - cache_file = get_cache_file(mode) - try: - CACHE_DIR.mkdir(parents=True, exist_ok=True) - new_cache = { - **cache, - **{str(src.resolve()): get_cache_info(src) for src in sources}, - } - with tempfile.NamedTemporaryFile(dir=str(cache_file.parent), delete=False) as f: - pickle.dump(new_cache, f, protocol=4) - os.replace(f.name, cache_file) - except OSError: - pass +@dataclass +class Cache: + mode: Mode + cache_file: Path + file_data: Dict[str, FileData] = field(default_factory=dict) + + @classmethod + def read(cls, mode: Mode) -> Self: + """Read the cache if it exists and is well formed. + + If it is not well formed, the call to write later should + resolve the issue. + """ + cache_file = get_cache_file(mode) + if not cache_file.exists(): + return cls(mode, cache_file) + + with cache_file.open("rb") as fobj: + try: + file_data: Dict[str, FileData] = pickle.load(fobj) + except (pickle.UnpicklingError, ValueError, IndexError): + return cls(mode, cache_file) + + return cls(mode, cache_file, file_data) + + @staticmethod + def hash_digest(path: Path) -> str: + """Return hash digest for path.""" + + data = path.read_bytes() + return hashlib.sha256(data).hexdigest() + + @staticmethod + def get_file_data(path: Path) -> FileData: + """Return file data for path.""" + + stat = path.stat() + hash = Cache.hash_digest(path) + return FileData(stat.st_mtime, stat.st_size, hash) + + def is_changed(self, source: Path) -> bool: + """Check if source has changed compared to cached version.""" + res_src = source.resolve() + old = self.file_data.get(str(res_src)) + if old is None: + return True + + st = res_src.stat() + if st.st_size != old.st_size: + return True + if int(st.st_mtime) != int(old.st_mtime): + new_hash = Cache.hash_digest(res_src) + if new_hash != old.hash: + return True + return False + + def filtered_cached(self, sources: Iterable[Path]) -> Tuple[Set[Path], Set[Path]]: + """Split an iterable of paths in `sources` into two sets. + + The first contains paths of files that modified on disk or are not in the + cache. The other contains paths to non-modified files. + """ + changed: Set[Path] = set() + done: Set[Path] = set() + for src in sources: + if self.is_changed(src): + changed.add(src) + else: + done.add(src) + return changed, done + + def write(self, sources: Iterable[Path]) -> None: + """Update the cache file data and write a new cache file.""" + self.file_data.update( + **{str(src.resolve()): Cache.get_file_data(src) for src in sources} + ) + try: + CACHE_DIR.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + dir=str(self.cache_file.parent), delete=False + ) as f: + pickle.dump(self.file_data, f, protocol=4) + os.replace(f.name, self.cache_file) + except OSError: + pass diff --git a/src/black/concurrency.py b/src/black/concurrency.py index 893eba6675a..ce016578399 100644 --- a/src/black/concurrency.py +++ b/src/black/concurrency.py @@ -17,7 +17,7 @@ from mypy_extensions import mypyc_attr from black import WriteBack, format_file_in_place -from black.cache import Cache, filter_cached, read_cache, write_cache +from black.cache import Cache from black.mode import Mode from black.output import err from black.report import Changed, Report @@ -133,10 +133,9 @@ async def schedule_formatting( `write_back`, `fast`, and `mode` options are passed to :func:`format_file_in_place`. """ - cache: Cache = {} + cache = Cache.read(mode) if write_back not in (WriteBack.DIFF, WriteBack.COLOR_DIFF): - cache = read_cache(mode) - sources, cached = filter_cached(cache, sources) + sources, cached = cache.filtered_cached(sources) for src in sorted(cached): report.done(src, Changed.CACHED) if not sources: @@ -185,4 +184,4 @@ async def schedule_formatting( if cancelled: await asyncio.gather(*cancelled, return_exceptions=True) if sources_to_cache: - write_cache(cache, sources_to_cache, mode) + cache.write(sources_to_cache) diff --git a/tests/test_black.py b/tests/test_black.py index 3b3ab721c5f..8ae92172d43 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -41,7 +41,7 @@ import black.files from black import Feature, TargetVersion from black import re_compile_maybe_verbose as compile_pattern -from black.cache import get_cache_dir, get_cache_file +from black.cache import FileData, get_cache_dir, get_cache_file from black.debug import DebugVisitor from black.output import color_diff, diff from black.report import Report @@ -1121,10 +1121,10 @@ def test_single_file_force_pyi(self) -> None: self.invokeBlack([str(path), "--pyi"]) actual = path.read_text(encoding="utf-8") # verify cache with --pyi is separate - pyi_cache = black.read_cache(pyi_mode) - self.assertIn(str(path), pyi_cache) - normal_cache = black.read_cache(DEFAULT_MODE) - self.assertNotIn(str(path), normal_cache) + pyi_cache = black.Cache.read(pyi_mode) + assert not pyi_cache.is_changed(path) + normal_cache = black.Cache.read(DEFAULT_MODE) + assert normal_cache.is_changed(path) self.assertFormatEqual(expected, actual) black.assert_equivalent(contents, actual) black.assert_stable(contents, actual, pyi_mode) @@ -1146,11 +1146,11 @@ def test_multi_file_force_pyi(self) -> None: actual = path.read_text(encoding="utf-8") self.assertEqual(actual, expected) # verify cache with --pyi is separate - pyi_cache = black.read_cache(pyi_mode) - normal_cache = black.read_cache(reg_mode) + pyi_cache = black.Cache.read(pyi_mode) + normal_cache = black.Cache.read(reg_mode) for path in paths: - self.assertIn(str(path), pyi_cache) - self.assertNotIn(str(path), normal_cache) + assert not pyi_cache.is_changed(path) + assert normal_cache.is_changed(path) def test_pipe_force_pyi(self) -> None: source, expected = read_data("miscellaneous", "force_pyi") @@ -1171,10 +1171,10 @@ def test_single_file_force_py36(self) -> None: self.invokeBlack([str(path), *PY36_ARGS]) actual = path.read_text(encoding="utf-8") # verify cache with --target-version is separate - py36_cache = black.read_cache(py36_mode) - self.assertIn(str(path), py36_cache) - normal_cache = black.read_cache(reg_mode) - self.assertNotIn(str(path), normal_cache) + py36_cache = black.Cache.read(py36_mode) + assert not py36_cache.is_changed(path) + normal_cache = black.Cache.read(reg_mode) + assert normal_cache.is_changed(path) self.assertEqual(actual, expected) @event_loop() @@ -1194,11 +1194,11 @@ def test_multi_file_force_py36(self) -> None: actual = path.read_text(encoding="utf-8") self.assertEqual(actual, expected) # verify cache with --target-version is separate - pyi_cache = black.read_cache(py36_mode) - normal_cache = black.read_cache(reg_mode) + pyi_cache = black.Cache.read(py36_mode) + normal_cache = black.Cache.read(reg_mode) for path in paths: - self.assertIn(str(path), pyi_cache) - self.assertNotIn(str(path), normal_cache) + assert not pyi_cache.is_changed(path) + assert normal_cache.is_changed(path) def test_pipe_force_py36(self) -> None: source, expected = read_data("miscellaneous", "force_py36") @@ -1953,19 +1953,20 @@ def test_cache_broken_file(self) -> None: with cache_dir() as workspace: cache_file = get_cache_file(mode) cache_file.write_text("this is not a pickle", encoding="utf-8") - assert black.read_cache(mode) == {} + assert black.Cache.read(mode).file_data == {} src = (workspace / "test.py").resolve() src.write_text("print('hello')", encoding="utf-8") invokeBlack([str(src)]) - cache = black.read_cache(mode) - assert str(src) in cache + cache = black.Cache.read(mode) + assert not cache.is_changed(src) def test_cache_single_file_already_cached(self) -> None: mode = DEFAULT_MODE with cache_dir() as workspace: src = (workspace / "test.py").resolve() src.write_text("print('hello')", encoding="utf-8") - black.write_cache({}, [src], mode) + cache = black.Cache.read(mode) + cache.write([src]) invokeBlack([str(src)]) assert src.read_text(encoding="utf-8") == "print('hello')" @@ -1979,13 +1980,14 @@ def test_cache_multiple_files(self) -> None: one.write_text("print('hello')", encoding="utf-8") two = (workspace / "two.py").resolve() two.write_text("print('hello')", encoding="utf-8") - black.write_cache({}, [one], mode) + cache = black.Cache.read(mode) + cache.write([one]) invokeBlack([str(workspace)]) assert one.read_text(encoding="utf-8") == "print('hello')" assert two.read_text(encoding="utf-8") == 'print("hello")\n' - cache = black.read_cache(mode) - assert str(one) in cache - assert str(two) in cache + cache = black.Cache.read(mode) + assert not cache.is_changed(one) + assert not cache.is_changed(two) @pytest.mark.parametrize("color", [False, True], ids=["no-color", "with-color"]) def test_no_cache_when_writeback_diff(self, color: bool) -> None: @@ -1993,8 +1995,8 @@ def test_no_cache_when_writeback_diff(self, color: bool) -> None: with cache_dir() as workspace: src = (workspace / "test.py").resolve() src.write_text("print('hello')", encoding="utf-8") - with patch("black.read_cache") as read_cache, patch( - "black.write_cache" + with patch.object(black.Cache, "read") as read_cache, patch.object( + black.Cache, "write" ) as write_cache: cmd = [str(src), "--diff"] if color: @@ -2002,8 +2004,8 @@ def test_no_cache_when_writeback_diff(self, color: bool) -> None: invokeBlack(cmd) cache_file = get_cache_file(mode) assert cache_file.exists() is False + read_cache.assert_called_once() write_cache.assert_not_called() - read_cache.assert_not_called() @pytest.mark.parametrize("color", [False, True], ids=["no-color", "with-color"]) @event_loop() @@ -2036,17 +2038,17 @@ def test_no_cache_when_stdin(self) -> None: def test_read_cache_no_cachefile(self) -> None: mode = DEFAULT_MODE with cache_dir(): - assert black.read_cache(mode) == {} + assert black.Cache.read(mode).file_data == {} def test_write_cache_read_cache(self) -> None: mode = DEFAULT_MODE with cache_dir() as workspace: src = (workspace / "test.py").resolve() src.touch() - black.write_cache({}, [src], mode) - cache = black.read_cache(mode) - assert str(src) in cache - assert cache[str(src)] == black.get_cache_info(src) + write_cache = black.Cache.read(mode) + write_cache.write([src]) + read_cache = black.Cache.read(mode) + assert not read_cache.is_changed(src) def test_filter_cached(self) -> None: with TemporaryDirectory() as workspace: @@ -2057,21 +2059,67 @@ def test_filter_cached(self) -> None: uncached.touch() cached.touch() cached_but_changed.touch() - cache = { - str(cached): black.get_cache_info(cached), - str(cached_but_changed): (0.0, 0), - } - todo, done = black.cache.filter_cached( - cache, {uncached, cached, cached_but_changed} - ) + cache = black.Cache.read(DEFAULT_MODE) + + orig_func = black.Cache.get_file_data + + def wrapped_func(path: Path) -> FileData: + if path == cached: + return orig_func(path) + if path == cached_but_changed: + return FileData(0.0, 0, "") + raise AssertionError + + with patch.object(black.Cache, "get_file_data", side_effect=wrapped_func): + cache.write([cached, cached_but_changed]) + todo, done = cache.filtered_cached({uncached, cached, cached_but_changed}) assert todo == {uncached, cached_but_changed} assert done == {cached} + def test_filter_cached_hash(self) -> None: + with TemporaryDirectory() as workspace: + path = Path(workspace) + src = (path / "test.py").resolve() + src.write_text("print('hello')", encoding="utf-8") + st = src.stat() + cache = black.Cache.read(DEFAULT_MODE) + cache.write([src]) + cached_file_data = cache.file_data[str(src)] + + todo, done = cache.filtered_cached([src]) + assert todo == set() + assert done == {src} + assert cached_file_data.st_mtime == st.st_mtime + + # Modify st_mtime + cached_file_data = cache.file_data[str(src)] = FileData( + cached_file_data.st_mtime - 1, + cached_file_data.st_size, + cached_file_data.hash, + ) + todo, done = cache.filtered_cached([src]) + assert todo == set() + assert done == {src} + assert cached_file_data.st_mtime < st.st_mtime + assert cached_file_data.st_size == st.st_size + assert cached_file_data.hash == black.Cache.hash_digest(src) + + # Modify contents + src.write_text("print('hello world')", encoding="utf-8") + new_st = src.stat() + todo, done = cache.filtered_cached([src]) + assert todo == {src} + assert done == set() + assert cached_file_data.st_mtime < new_st.st_mtime + assert cached_file_data.st_size != new_st.st_size + assert cached_file_data.hash != black.Cache.hash_digest(src) + def test_write_cache_creates_directory_if_needed(self) -> None: mode = DEFAULT_MODE with cache_dir(exists=False) as workspace: assert not workspace.exists() - black.write_cache({}, [], mode) + cache = black.Cache.read(mode) + cache.write([]) assert workspace.exists() @event_loop() @@ -2085,15 +2133,17 @@ def test_failed_formatting_does_not_get_cached(self) -> None: clean = (workspace / "clean.py").resolve() clean.write_text('print("hello")\n', encoding="utf-8") invokeBlack([str(workspace)], exit_code=123) - cache = black.read_cache(mode) - assert str(failing) not in cache - assert str(clean) in cache + cache = black.Cache.read(mode) + assert cache.is_changed(failing) + assert not cache.is_changed(clean) def test_write_cache_write_fail(self) -> None: mode = DEFAULT_MODE - with cache_dir(), patch.object(Path, "open") as mock: - mock.side_effect = OSError - black.write_cache({}, [], mode) + with cache_dir(): + cache = black.Cache.read(mode) + with patch.object(Path, "open") as mock: + mock.side_effect = OSError + cache.write([]) def test_read_cache_line_lengths(self) -> None: mode = DEFAULT_MODE @@ -2101,11 +2151,12 @@ def test_read_cache_line_lengths(self) -> None: with cache_dir() as workspace: path = (workspace / "file.py").resolve() path.touch() - black.write_cache({}, [path], mode) - one = black.read_cache(mode) - assert str(path) in one - two = black.read_cache(short_mode) - assert str(path) not in two + cache = black.Cache.read(mode) + cache.write([path]) + one = black.Cache.read(mode) + assert not one.is_changed(path) + two = black.Cache.read(short_mode) + assert two.is_changed(path) def assert_collected_sources( From 066aa9210ac7815cbb9b4a25075f54d614b0afc7 Mon Sep 17 00:00:00 2001 From: Marc Mueller <30130371+cdce8p@users.noreply.github.com> Date: Sat, 19 Aug 2023 17:09:59 +0200 Subject: [PATCH 085/279] Remove tox pin (#3844) --- .github/workflows/test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8a139387c5b..216b0ba5236 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,11 +45,12 @@ jobs: - name: Install tox run: | python -m pip install --upgrade pip - python -m pip install --upgrade 'tox<4.7' + python -m pip install --upgrade tox - name: Unit tests if: "!startsWith(matrix.python-version, 'pypy')" - run: tox -e ci-py -- -v --color=yes + run: + tox -e ci-py$(echo ${{ matrix.python-version }} | tr -d '.') -- -v --color=yes - name: Unit tests (pypy) if: "startsWith(matrix.python-version, 'pypy')" From 6310a405f6663948f7e0b9411cb54e5db2b712a6 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sat, 19 Aug 2023 08:13:05 -0700 Subject: [PATCH 086/279] Improve handling of root to get_sources (#3847) This is a little more type safe and a little cleaner --- src/black/__init__.py | 14 ++++++++------ tests/test_black.py | 28 ++++++++-------------------- 2 files changed, 16 insertions(+), 26 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index dc06eab8dd0..6fc91d2e6d3 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -560,9 +560,10 @@ def main( # noqa: C901 content=code, fast=fast, write_back=write_back, mode=mode, report=report ) else: + assert root is not None # root is only None if code is not None try: sources = get_sources( - ctx=ctx, + root=root, src=src, quiet=quiet, verbose=verbose, @@ -615,7 +616,7 @@ def main( # noqa: C901 def get_sources( *, - ctx: click.Context, + root: Path, src: Tuple[str, ...], quiet: bool, verbose: bool, @@ -628,7 +629,6 @@ def get_sources( ) -> Set[Path]: """Compute the set of files to be formatted.""" sources: Set[Path] = set() - root = ctx.obj["root"] using_default_exclude = exclude is None exclude = re_compile_maybe_verbose(DEFAULT_EXCLUDES) if exclude is None else exclude @@ -645,7 +645,7 @@ def get_sources( if is_stdin or p.is_file(): normalized_path: Optional[str] = normalize_path_maybe_ignore( - p, ctx.obj["root"], report + p, root, report ) if normalized_path is None: if verbose: @@ -674,7 +674,9 @@ def get_sources( sources.add(p) elif p.is_dir(): - p = root / normalize_path_maybe_ignore(p, ctx.obj["root"], report) + p_relative = normalize_path_maybe_ignore(p, root, report) + assert p_relative is not None + p = root / p_relative if verbose: out(f'Found input source directory: "{p}"', fg="blue") @@ -686,7 +688,7 @@ def get_sources( sources.update( gen_python_files( p.iterdir(), - ctx.obj["root"], + root, include, exclude, extend_exclude, diff --git a/tests/test_black.py b/tests/test_black.py index 8ae92172d43..79930fabf1f 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -507,13 +507,11 @@ def _mocked_calls() -> bool: with patch("pathlib.Path.iterdir", return_value=target_contents), patch( "pathlib.Path.cwd", return_value=working_directory ), patch("pathlib.Path.is_dir", side_effect=mock_n_calls([True])): - ctx = FakeContext() # Note that the root folder (project_root) isn't the folder # named "root" (aka working_directory) - ctx.obj["root"] = project_root report = MagicMock(verbose=True) black.get_sources( - ctx=ctx, + root=project_root, src=("./child",), quiet=False, verbose=True, @@ -2163,7 +2161,7 @@ def assert_collected_sources( src: Sequence[Union[str, Path]], expected: Sequence[Union[str, Path]], *, - ctx: Optional[FakeContext] = None, + root: Optional[Path] = None, exclude: Optional[str] = None, include: Optional[str] = None, extend_exclude: Optional[str] = None, @@ -2179,7 +2177,7 @@ def assert_collected_sources( ) gs_force_exclude = None if force_exclude is None else compile_pattern(force_exclude) collected = black.get_sources( - ctx=ctx or FakeContext(), + root=root or THIS_DIR, src=gs_src, quiet=False, verbose=False, @@ -2215,9 +2213,7 @@ def test_gitignore_used_as_default(self) -> None: base / "b/.definitely_exclude/a.pyi", ] src = [base / "b/"] - ctx = FakeContext() - ctx.obj["root"] = base - assert_collected_sources(src, expected, ctx=ctx, extend_exclude=r"/exclude/") + assert_collected_sources(src, expected, root=base, extend_exclude=r"/exclude/") def test_gitignore_used_on_multiple_sources(self) -> None: root = Path(DATA_DIR / "gitignore_used_on_multiple_sources") @@ -2225,10 +2221,8 @@ def test_gitignore_used_on_multiple_sources(self) -> None: root / "dir1" / "b.py", root / "dir2" / "b.py", ] - ctx = FakeContext() - ctx.obj["root"] = root src = [root / "dir1", root / "dir2"] - assert_collected_sources(src, expected, ctx=ctx) + assert_collected_sources(src, expected, root=root) @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None)) def test_exclude_for_issue_1572(self) -> None: @@ -2334,9 +2328,7 @@ def test_gitignore_that_ignores_subfolders(self) -> None: # If gitignore with */* is in root root = Path(DATA_DIR / "ignore_subfolders_gitignore_tests" / "subdir") expected = [root / "b.py"] - ctx = FakeContext() - ctx.obj["root"] = root - assert_collected_sources([root], expected, ctx=ctx) + assert_collected_sources([root], expected, root=root) # If .gitignore with */* is nested root = Path(DATA_DIR / "ignore_subfolders_gitignore_tests") @@ -2344,17 +2336,13 @@ def test_gitignore_that_ignores_subfolders(self) -> None: root / "a.py", root / "subdir" / "b.py", ] - ctx = FakeContext() - ctx.obj["root"] = root - assert_collected_sources([root], expected, ctx=ctx) + assert_collected_sources([root], expected, root=root) # If command is executed from outer dir root = Path(DATA_DIR / "ignore_subfolders_gitignore_tests") target = root / "subdir" expected = [target / "b.py"] - ctx = FakeContext() - ctx.obj["root"] = root - assert_collected_sources([target], expected, ctx=ctx) + assert_collected_sources([target], expected, root=root) def test_empty_include(self) -> None: path = DATA_DIR / "include_exclude_tests" From d9c249c25a77f75e70278aab9ec65c10ce08b0a8 Mon Sep 17 00:00:00 2001 From: Kjell-Magnus Date: Tue, 22 Aug 2023 21:40:10 +0200 Subject: [PATCH 087/279] Fix download badge link (#3853) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9d0b29af215..b257c333f0d 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Coverage Status License: MIT PyPI -Downloads +Downloads conda-forge Code style: black

From 47676bf5939ae5c8e670d947917bc8af4732eab6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 26 Aug 2023 08:44:17 -0500 Subject: [PATCH 088/279] Bump furo from 2023.7.26 to 2023.8.19 in /docs + sphinx to 7.2.3 (#3848) * Bump furo from 2023.7.26 to 2023.8.19 in /docs Bumps [furo](https://github.com/pradyunsg/furo) from 2023.7.26 to 2023.8.19. - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2023.07.26...2023.08.19) --- updated-dependencies: - dependency-name: furo dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Move to sphinx 7.2.3 + fix intersphinx_mapping --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Cooper Ry Lees --- docs/conf.py | 2 +- docs/requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 7fc4f8f589e..f7cf1b42842 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -213,4 +213,4 @@ def make_pypi_svg(version: str) -> None: # -- Options for intersphinx extension --------------------------------------- # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {"https://docs.python.org/3/": None} +intersphinx_mapping = {"": ("https://docs.python.org/3/", None)} diff --git a/docs/requirements.txt b/docs/requirements.txt index ff179f3805e..dad39f67ed3 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,9 +1,9 @@ # Used by ReadTheDocs; pinned requirements for stability. myst-parser==2.0.0 -Sphinx==6.1.3 +Sphinx==7.2.3 # Older versions break Sphinx even though they're declared to be supported. docutils==0.19 sphinxcontrib-programoutput==0.17 sphinx_copybutton==0.5.2 -furo==2023.7.26 +furo==2023.8.19 From 58f1bf69d2ed2f6e3e5fa6a31e01ae58c9ffcff9 Mon Sep 17 00:00:00 2001 From: "Johnny.H" Date: Sun, 3 Sep 2023 10:46:23 +0800 Subject: [PATCH 089/279] Move coverage configurations to `pyproject.toml` (#3858) --- .coveragerc | 9 --------- .github/workflows/test.yml | 4 ++-- pyproject.toml | 9 +++++++++ 3 files changed, 11 insertions(+), 11 deletions(-) delete mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 5577e496a57..00000000000 --- a/.coveragerc +++ /dev/null @@ -1,9 +0,0 @@ -[report] -omit = - src/blib2to3/* - tests/data/* - */site-packages/* - .tox/* - -[run] -relative_files = True diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 216b0ba5236..7daa31ee903 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -62,7 +62,7 @@ jobs: if: github.repository == 'psf/black' && matrix.os == 'ubuntu-latest' && !startsWith(matrix.python-version, 'pypy') - uses: AndreMiras/coveralls-python-action@v20201129 + uses: AndreMiras/coveralls-python-action@8799c9f4443ac4201d2e2f2c725d577174683b99 with: github-token: ${{ secrets.GITHUB_TOKEN }} parallel: true @@ -77,7 +77,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Send finished signal to Coveralls - uses: AndreMiras/coveralls-python-action@v20201129 + uses: AndreMiras/coveralls-python-action@8799c9f4443ac4201d2e2f2c725d577174683b99 with: parallel-finished: true debug: true diff --git a/pyproject.toml b/pyproject.toml index 6cd3f34bc10..ea5c9f84684 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -210,3 +210,12 @@ filterwarnings = [ # Work around https://github.com/pytest-dev/pytest/issues/10977 for Python 3.12 '''ignore:(Attribute s|Attribute n|ast.Str|ast.Bytes|ast.NameConstant|ast.Num) is deprecated and will be removed in Python 3.14:DeprecationWarning''' ] +[tool.coverage.report] +omit = [ + "src/blib2to3/*", + "tests/data/*", + "*/site-packages/*", + ".tox/*" +] +[tool.coverage.run] +relative_files = true From df50fee7fd85018f8db462774512a83031f00322 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Wed, 6 Sep 2023 21:06:07 -0700 Subject: [PATCH 090/279] Apply ignore logic before symlink resolution (#3846) This means, for instance, that a gitignored symlink cannot affect your formatting. Fixes #3527, fixes #3826 --- CHANGES.md | 2 ++ src/black/files.py | 20 ++++++++------- tests/test_black.py | 62 +++++++++++++++++++++++++-------------------- 3 files changed, 48 insertions(+), 36 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a14a55a03ac..2168c1b90ce 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,6 +20,8 @@ +- Black now applies exclusion and ignore logic before resolving symlinks (#3846) + ### Packaging diff --git a/src/black/files.py b/src/black/files.py index 368e4170d47..362898dc0fd 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -330,35 +330,37 @@ def gen_python_files( assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}" for child in paths: - normalized_path = normalize_path_maybe_ignore(child, root, report) - if normalized_path is None: - continue + root_relative_path = child.absolute().relative_to(root).as_posix() # First ignore files matching .gitignore, if passed if gitignore_dict and _path_is_ignored( - normalized_path, root, gitignore_dict, report + root_relative_path, root, gitignore_dict, report ): continue # Then ignore with `--exclude` `--extend-exclude` and `--force-exclude` options. - normalized_path = "/" + normalized_path + root_relative_path = "/" + root_relative_path if child.is_dir(): - normalized_path += "/" + root_relative_path += "/" - if path_is_excluded(normalized_path, exclude): + if path_is_excluded(root_relative_path, exclude): report.path_ignored(child, "matches the --exclude regular expression") continue - if path_is_excluded(normalized_path, extend_exclude): + if path_is_excluded(root_relative_path, extend_exclude): report.path_ignored( child, "matches the --extend-exclude regular expression" ) continue - if path_is_excluded(normalized_path, force_exclude): + if path_is_excluded(root_relative_path, force_exclude): report.path_ignored(child, "matches the --force-exclude regular expression") continue + normalized_path = normalize_path_maybe_ignore(child, root, report) + if normalized_path is None: + continue + if child.is_dir(): # If gitignore is None, gitignore usage is disabled, while a Falsey # gitignore is when the directory doesn't have a .gitignore file. diff --git a/tests/test_black.py b/tests/test_black.py index 79930fabf1f..4fb6aef9bca 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -492,9 +492,7 @@ def test_false_positive_symlink_output_issue_3384(self) -> None: project_root = Path(THIS_DIR / "data" / "nested_gitignore_tests") working_directory = project_root / "root" target_abspath = working_directory / "child" - target_contents = ( - src.relative_to(working_directory) for src in target_abspath.iterdir() - ) + target_contents = list(target_abspath.iterdir()) def mock_n_calls(responses: List[bool]) -> Callable[[], bool]: def _mocked_calls() -> bool: @@ -2375,38 +2373,48 @@ def test_extend_exclude(self) -> None: ) @pytest.mark.incompatible_with_mypyc - def test_symlink_out_of_root_directory(self) -> None: + def test_symlinks(self) -> None: path = MagicMock() root = THIS_DIR.resolve() - child = MagicMock() include = re.compile(black.DEFAULT_INCLUDES) exclude = re.compile(black.DEFAULT_EXCLUDES) report = black.Report() gitignore = PathSpec.from_lines("gitwildmatch", []) - # `child` should behave like a symlink which resolved path is clearly - # outside of the `root` directory. - path.iterdir.return_value = [child] - child.resolve.return_value = Path("/a/b/c") - child.as_posix.return_value = "/a/b/c" - try: - list( - black.gen_python_files( - path.iterdir(), - root, - include, - exclude, - None, - None, - report, - {path: gitignore}, - verbose=False, - quiet=False, - ) + + regular = MagicMock() + outside_root_symlink = MagicMock() + ignored_symlink = MagicMock() + + path.iterdir.return_value = [regular, outside_root_symlink, ignored_symlink] + + regular.absolute.return_value = root / "regular.py" + regular.resolve.return_value = root / "regular.py" + regular.is_dir.return_value = False + + outside_root_symlink.absolute.return_value = root / "symlink.py" + outside_root_symlink.resolve.return_value = Path("/nowhere") + + ignored_symlink.absolute.return_value = root / ".mypy_cache" / "symlink.py" + + files = list( + black.gen_python_files( + path.iterdir(), + root, + include, + exclude, + None, + None, + report, + {path: gitignore}, + verbose=False, + quiet=False, ) - except ValueError as ve: - pytest.fail(f"`get_python_files_in_dir()` failed: {ve}") + ) + assert files == [regular] + path.iterdir.assert_called_once() - child.resolve.assert_called_once() + outside_root_symlink.resolve.assert_called_once() + ignored_symlink.resolve.assert_not_called() @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None)) def test_get_sources_with_stdin(self) -> None: From 8daa64a2e10907539094df51f4c51306bb426f07 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+KotlinIsland@users.noreply.github.com> Date: Thu, 7 Sep 2023 17:11:50 +1000 Subject: [PATCH 091/279] blackd: fix mishandling of single character input (#3558) --- CHANGES.md | 2 ++ src/blackd/__init__.py | 3 ++- tests/test_blackd.py | 6 ++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 2168c1b90ce..af9fc490acf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -45,6 +45,8 @@ +- Fix an issue in `blackd` with single character input (#3558) + ### Integrations diff --git a/src/blackd/__init__.py b/src/blackd/__init__.py index 4f2d87d0fca..6b0f3d33295 100644 --- a/src/blackd/__init__.py +++ b/src/blackd/__init__.py @@ -152,7 +152,8 @@ async def handle(request: web.Request, executor: Executor) -> web.Response: ) # Preserve CRLF line endings - if req_str[req_str.find("\n") - 1] == "\r": + nl = req_str.find("\n") + if nl > 0 and req_str[nl - 1] == "\r": formatted_str = formatted_str.replace("\n", "\r\n") # If, after swapping line endings, nothing changed, then say so if formatted_str == req_str: diff --git a/tests/test_blackd.py b/tests/test_blackd.py index 325bd7dd5aa..dd2126e6bc2 100644 --- a/tests/test_blackd.py +++ b/tests/test_blackd.py @@ -240,3 +240,9 @@ async def test_normalizes_line_endings(self) -> None: response = await self.client.post("/", data=data) self.assertEqual(await response.text(), expected) self.assertEqual(response.status, 200) + + @unittest_run_loop + async def test_single_character(self) -> None: + response = await self.client.post("/", data="1") + self.assertEqual(await response.text(), "1\n") + self.assertEqual(response.status, 200) From 74d3009ba480a871df57197144578f1ae4016210 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Fri, 8 Sep 2023 03:35:07 +0200 Subject: [PATCH 092/279] Add Black PyCharm 2023.2 integration instructions (#3839) --- docs/integrations/editors.md | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/docs/integrations/editors.md b/docs/integrations/editors.md index ff563068e79..cebe2b0721e 100644 --- a/docs/integrations/editors.md +++ b/docs/integrations/editors.md @@ -10,16 +10,26 @@ Options include the following: ## PyCharm/IntelliJ IDEA -There are three different ways you can use _Black_ from PyCharm: +There are several different ways you can use _Black_ from PyCharm: -1. As local server using the BlackConnect plugin -1. As external tool -1. As file watcher +1. Using the built-in _Black_ integration (PyCharm 2023.2 and later). This option is the + simplest to set up. +1. As local server using the BlackConnect plugin. This option formats the fastest. It + spins up {doc}`Black's HTTP server `, to + avoid the startup cost on subsequent formats. +1. As external tool. +1. As file watcher. -The first option is the simplest to set up and formats the fastest (by spinning up -{doc}`Black's HTTP server `, avoiding the -startup cost on subsequent formats), but if you would prefer to not install a -third-party plugin or blackd's extra dependencies, the other two are also great options. +### Built-in _Black_ integration + +1. Install `black`. + + ```console + $ pip install black + ``` + +1. Go to `Preferences or Settings -> Tools -> Black` and configure _Black_ to your + liking. ### As local server From a20338cf100ff20a24e2058c6f6014e82efdf914 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Fri, 8 Sep 2023 16:37:13 +0200 Subject: [PATCH 093/279] Avoid removing whitespace for walrus operators within subscripts (#3823) Co-authored-by: hauntsaninja Co-authored-by: Jelle Zijlstra --- CHANGES.md | 1 + src/black/lines.py | 4 +++- src/black/mode.py | 1 + src/black/nodes.py | 8 +++++++- tests/data/preview/pep_572.py | 6 ++++++ tests/data/preview_py_310/pep_572.py | 12 ++++++++++++ tests/test_format.py | 7 +++++++ 7 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 tests/data/preview/pep_572.py create mode 100644 tests/data/preview_py_310/pep_572.py diff --git a/CHANGES.md b/CHANGES.md index af9fc490acf..4aa3123fab6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -80,6 +80,7 @@ (#3740) - Fix error in AST validation when _Black_ removes trailing whitespace in a type comment (#3773) +- Fix a bug whereby spaces were removed from walrus operators within subscript (#3823) ### Preview style diff --git a/src/black/lines.py b/src/black/lines.py index 0a307b45eff..f3044ce47b8 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -81,7 +81,9 @@ def append( # Note: at this point leaf.prefix should be empty except for # imports, for which we only preserve newlines. leaf.prefix += whitespace( - leaf, complex_subscript=self.is_complex_subscript(leaf) + leaf, + complex_subscript=self.is_complex_subscript(leaf), + mode=self.mode, ) if self.inside_brackets or not preformatted or track_bracket: self.bracket_tracker.mark(leaf) diff --git a/src/black/mode.py b/src/black/mode.py index 282c1669da7..06d20b7a7d6 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -183,6 +183,7 @@ class Preview(Enum): wrap_long_dict_values_in_parens = auto() wrap_multiple_context_managers_in_parens = auto() dummy_implementations = auto() + walrus_subscript = auto() class Deprecated(UserWarning): diff --git a/src/black/nodes.py b/src/black/nodes.py index 45423b2596b..edd201a21e9 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -13,6 +13,7 @@ from mypy_extensions import mypyc_attr from black.cache import CACHE_DIR +from black.mode import Mode, Preview from black.strings import has_triple_quotes from blib2to3 import pygram from blib2to3.pgen2 import token @@ -171,7 +172,7 @@ def visit_default(self, node: LN) -> Iterator[T]: yield from self.visit(child) -def whitespace(leaf: Leaf, *, complex_subscript: bool) -> str: # noqa: C901 +def whitespace(leaf: Leaf, *, complex_subscript: bool, mode: Mode) -> str: # noqa: C901 """Return whitespace prefix if needed for the given `leaf`. `complex_subscript` signals whether the given leaf is part of a subscription @@ -345,6 +346,11 @@ def whitespace(leaf: Leaf, *, complex_subscript: bool) -> str: # noqa: C901 return NO + elif Preview.walrus_subscript in mode and ( + t == token.COLONEQUAL or prev.type == token.COLONEQUAL + ): + return SPACE + elif not complex_subscript: return NO diff --git a/tests/data/preview/pep_572.py b/tests/data/preview/pep_572.py new file mode 100644 index 00000000000..a50e130ad9c --- /dev/null +++ b/tests/data/preview/pep_572.py @@ -0,0 +1,6 @@ +x[(a:=0):] +x[:(a:=0)] + +# output +x[(a := 0):] +x[:(a := 0)] diff --git a/tests/data/preview_py_310/pep_572.py b/tests/data/preview_py_310/pep_572.py new file mode 100644 index 00000000000..78d4e9e4506 --- /dev/null +++ b/tests/data/preview_py_310/pep_572.py @@ -0,0 +1,12 @@ +x[a:=0] +x[a := 0] +x[a := 0, b := 1] +x[5, b := 0] +x[a:=0,b:=1] + +# output +x[a := 0] +x[a := 0] +x[a := 0, b := 1] +x[5, b := 0] +x[a := 0, b := 1] diff --git a/tests/test_format.py b/tests/test_format.py index fb4d8eb4346..0650a2d6e53 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -56,6 +56,13 @@ def test_preview_context_managers_targeting_py39() -> None: assert_format(source, expected, mode, minimum_version=(3, 9)) +@pytest.mark.parametrize("filename", all_data_cases("preview_py_310")) +def test_preview_python_310(filename: str) -> None: + source, expected = read_data("preview_py_310", filename) + mode = black.Mode(target_versions={black.TargetVersion.PY310}, preview=True) + assert_format(source, expected, mode, minimum_version=(3, 10)) + + @pytest.mark.parametrize( "filename", all_data_cases("preview_context_managers/auto_detect") ) From b40b01ffe3dbf1fa989acd6050ef5e61c085b5da Mon Sep 17 00:00:00 2001 From: konsti Date: Sat, 9 Sep 2023 03:51:27 +0200 Subject: [PATCH 094/279] Blank line between nested and function def in stub files. (#3862) The idea behind this change is that we stop looking into previous body to determine if there should be a blank before a function or class definition. Input: ```python import sys if sys.version_info > (3, 7): class Nested1: assignment = 1 def function_definition(self): ... def f1(self) -> str: ... class Nested2: def function_definition(self): ... assignment = 1 def f2(self) -> str: ... if sys.version_info > (3, 7): def nested1(): assignment = 1 def function_definition(self): ... def f1(self) -> str: ... def nested2(): def function_definition(self): ... assignment = 1 def f2(self) -> str: ... ``` Stable style ```python import sys if sys.version_info > (3, 7): class Nested1: assignment = 1 def function_definition(self): ... def f1(self) -> str: ... class Nested2: def function_definition(self): ... assignment = 1 def f2(self) -> str: ... if sys.version_info > (3, 7): def nested1(): assignment = 1 def function_definition(self): ... def f1(self) -> str: ... def nested2(): def function_definition(self): ... assignment = 1 def f2(self) -> str: ... ``` In the stable formatting, we have a blank line sometimes, not depending on the previous statement on the same level, but on the last (potentially nested) statement in the previous body. #2783/#3564 fixes this for classes in preview style: ```python import sys if sys.version_info > (3, 7): class Nested1: assignment = 1 def function_definition(self): ... def f1(self) -> str: ... class Nested2: def function_definition(self): ... assignment = 1 def f2(self) -> str: ... if sys.version_info > (3, 7): def nested1(): assignment = 1 def function_definition(self): ... def f1(self) -> str: ... def nested2(): def function_definition(self): ... assignment = 1 def f2(self) -> str: ... ``` This PR additionally fixes this for function definitions: ```python if sys.version_info > (3, 7): if sys.platform == "win32": assignment = 1 def function_definition(self): ... def f1(self) -> str: ... if sys.platform != "win32": def function_definition(self): ... assignment = 1 def f2(self) -> str: ... if sys.version_info > (3, 8): if sys.platform == "win32": assignment = 1 def function_definition(self): ... class F1: ... if sys.platform != "win32": def function_definition(self): ... assignment = 1 class F2: ... ``` You can see the effect of this change on typeshed in https://github.com/konstin/typeshed/pull/1/files. As baseline, the preview mode changes without this PR are at https://github.com/konstin/typeshed/pull/2. Co-authored-by: Jelle Zijlstra --- CHANGES.md | 2 + src/black/lines.py | 11 +++++ src/black/mode.py | 1 + .../data/miscellaneous/nested_class_stub.pyi | 16 ------- tests/data/miscellaneous/nested_stub.pyi | 43 +++++++++++++++++++ tests/test_format.py | 4 +- 6 files changed, 59 insertions(+), 18 deletions(-) delete mode 100644 tests/data/miscellaneous/nested_class_stub.pyi create mode 100644 tests/data/miscellaneous/nested_stub.pyi diff --git a/CHANGES.md b/CHANGES.md index 4aa3123fab6..b0fa5f8745e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -171,6 +171,8 @@ expected to become part of Black's stable style in January 2024. - For stubs, enforce one blank line after a nested class with a body other than just `...` (#3564) - Improve handling of multiline strings by changing line split behavior (#1879) +- In stub files, add a blank line between a statement with a body (e.g an + `if sys.version_info > (3, x):`) and a function definition on the same level. (#3862) ### Parser diff --git a/src/black/lines.py b/src/black/lines.py index f3044ce47b8..71b657a0654 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -711,6 +711,17 @@ def _maybe_empty_lines_for_class_or_def( # noqa: C901 newlines = 0 else: newlines = 1 + # Remove case `self.previous_line.depth > current_line.depth` below when + # this becomes stable. + # + # Don't inspect the previous line if it's part of the body of the previous + # statement in the same level, we always want a blank line if there's + # something with a body preceding. + elif ( + Preview.blank_line_between_nested_and_def_stub_file in current_line.mode + and self.previous_line.depth > current_line.depth + ): + newlines = 1 elif ( current_line.is_def or current_line.is_decorator ) and not self.previous_line.is_def: diff --git a/src/black/mode.py b/src/black/mode.py index 06d20b7a7d6..8a855ac495a 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -170,6 +170,7 @@ class Preview(Enum): add_trailing_comma_consistently = auto() blank_line_after_nested_stub_class = auto() + blank_line_between_nested_and_def_stub_file = auto() hex_codes_in_unicode_sequences = auto() improved_async_statements_handling = auto() multiline_string_handling = auto() diff --git a/tests/data/miscellaneous/nested_class_stub.pyi b/tests/data/miscellaneous/nested_class_stub.pyi deleted file mode 100644 index daf281b517b..00000000000 --- a/tests/data/miscellaneous/nested_class_stub.pyi +++ /dev/null @@ -1,16 +0,0 @@ -class Outer: - class InnerStub: ... - outer_attr_after_inner_stub: int - class Inner: - inner_attr: int - outer_attr: int - -# output -class Outer: - class InnerStub: ... - outer_attr_after_inner_stub: int - - class Inner: - inner_attr: int - - outer_attr: int diff --git a/tests/data/miscellaneous/nested_stub.pyi b/tests/data/miscellaneous/nested_stub.pyi new file mode 100644 index 00000000000..15e69d854db --- /dev/null +++ b/tests/data/miscellaneous/nested_stub.pyi @@ -0,0 +1,43 @@ +import sys + +class Outer: + class InnerStub: ... + outer_attr_after_inner_stub: int + class Inner: + inner_attr: int + outer_attr: int + +if sys.version_info > (3, 7): + if sys.platform == "win32": + assignment = 1 + def function_definition(self): ... + def f1(self) -> str: ... + if sys.platform != "win32": + def function_definition(self): ... + assignment = 1 + def f2(self) -> str: ... + +# output + +import sys + +class Outer: + class InnerStub: ... + outer_attr_after_inner_stub: int + + class Inner: + inner_attr: int + + outer_attr: int + +if sys.version_info > (3, 7): + if sys.platform == "win32": + assignment = 1 + def function_definition(self): ... + + def f1(self) -> str: ... + if sys.platform != "win32": + def function_definition(self): ... + assignment = 1 + + def f2(self) -> str: ... \ No newline at end of file diff --git a/tests/test_format.py b/tests/test_format.py index 0650a2d6e53..f3db423b637 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -201,9 +201,9 @@ def test_stub() -> None: assert_format(source, expected, mode) -def test_nested_class_stub() -> None: +def test_nested_stub() -> None: mode = replace(DEFAULT_MODE, is_pyi=True, preview=True) - source, expected = read_data("miscellaneous", "nested_class_stub.pyi") + source, expected = read_data("miscellaneous", "nested_stub.pyi") assert_format(source, expected, mode) From b70b2c619671f0c6adc722742181bd2fa6e2a2f4 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 8 Sep 2023 20:24:49 -0700 Subject: [PATCH 095/279] Prepare release 23.9.0 (#3863) --- CHANGES.md | 48 +++++++++++++-------- docs/contributing/release_process.md | 2 + docs/integrations/source_version_control.md | 4 +- docs/usage_and_configuration/the_basics.md | 6 +-- 4 files changed, 37 insertions(+), 23 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b0fa5f8745e..3829526871e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,14 +14,10 @@ -- More concise formatting for dummy implementations (#3796) - ### Configuration -- Black now applies exclusion and ignore logic before resolving symlinks (#3846) - ### Packaging @@ -34,9 +30,6 @@ -- Avoid importing `IPython` if notebook cells do not contain magics (#3782) -- Improve caching by comparing file hashes as fallback for mtime and size. (#3821) - ### Output @@ -45,23 +38,45 @@ -- Fix an issue in `blackd` with single character input (#3558) - ### Integrations +### Documentation + + + +## 23.9.0 + +### Preview style + +- More concise formatting for dummy implementations (#3796) +- In stub files, add a blank line between a statement with a body (e.g an + `if sys.version_info > (3, x):`) and a function definition on the same level (#3862) +- Fix a bug whereby spaces were removed from walrus operators within subscript(#3823) + +### Configuration + +- Black now applies exclusion and ignore logic before resolving symlinks (#3846) + +### Performance + +- Avoid importing `IPython` if notebook cells do not contain magics (#3782) +- Improve caching by comparing file hashes as fallback for mtime and size (#3821) + +### _Blackd_ + +- Fix an issue in `blackd` with single character input (#3558) + +### Integrations + - Black now has an [official pre-commit mirror](https://github.com/psf/black-pre-commit-mirror). Swapping `https://github.com/psf/black` to `https://github.com/psf/black-pre-commit-mirror` in your `.pre-commit-config.yaml` will make Black about 2x faster (#3828) - The `.black.env` folder specified by `ENV_PATH` will now be removed on the completion - of the GitHub Action. (#3759) - -### Documentation - - + of the GitHub Action (#3759) ## 23.7.0 @@ -80,7 +95,6 @@ (#3740) - Fix error in AST validation when _Black_ removes trailing whitespace in a type comment (#3773) -- Fix a bug whereby spaces were removed from walrus operators within subscript (#3823) ### Preview style @@ -171,8 +185,6 @@ expected to become part of Black's stable style in January 2024. - For stubs, enforce one blank line after a nested class with a body other than just `...` (#3564) - Improve handling of multiline strings by changing line split behavior (#1879) -- In stub files, add a blank line between a statement with a body (e.g an - `if sys.version_info > (3, x):`) and a function definition on the same level. (#3862) ### Parser diff --git a/docs/contributing/release_process.md b/docs/contributing/release_process.md index be9b08a6c82..02865d6f4bd 100644 --- a/docs/contributing/release_process.md +++ b/docs/contributing/release_process.md @@ -43,6 +43,8 @@ To cut a release: 1. Remove any empty sections for the current release 1. (_optional_) Read through and copy-edit the changelog (eg. by moving entries, fixing typos, or rephrasing entries) + 1. Double-check that no changelog entries since the last release were put in the + wrong section (e.g., run `git diff CHANGES.md`) 1. Add a new empty template for the next release above ([template below](#changelog-template)) 1. Update references to the latest version in diff --git a/docs/integrations/source_version_control.md b/docs/integrations/source_version_control.md index 24e732848f1..28414973ff5 100644 --- a/docs/integrations/source_version_control.md +++ b/docs/integrations/source_version_control.md @@ -8,7 +8,7 @@ Use [pre-commit](https://pre-commit.com/). Once you repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.7.0 + rev: 23.9.0 hooks: - id: black # It is recommended to specify the latest version of Python @@ -35,7 +35,7 @@ include Jupyter Notebooks. To use this hook, simply replace the hook's `id: blac repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.7.0 + rev: 23.9.0 hooks: - id: black-jupyter # It is recommended to specify the latest version of Python diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 5efb50a9a12..6ae9441fb69 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -193,8 +193,8 @@ configuration file for consistent results across environments. ```console $ black --version -black, 23.7.0 (compiled: yes) -$ black --required-version 23.7.0 -c "format = 'this'" +black, 23.9.0 (compiled: yes) +$ black --required-version 23.9.0 -c "format = 'this'" format = "this" $ black --required-version 31.5b2 -c "still = 'beta?!'" Oh no! 💥 💔 💥 The required version does not match the running version! @@ -285,7 +285,7 @@ You can check the version of _Black_ you have installed using the `--version` fl ```console $ black --version -black, 23.7.0 +black, 23.9.0 ``` #### `--config` From 716fa08090b6a51e4c72dd0a14b6c45f7da4a9d0 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 8 Sep 2023 22:16:15 -0700 Subject: [PATCH 096/279] Upgrade mypy (#3864) --- CHANGES.md | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 3829526871e..1334efefe7b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -22,6 +22,8 @@ +- Upgrade to mypy 1.5.1 (#3864) + ### Parser diff --git a/pyproject.toml b/pyproject.toml index ea5c9f84684..8585f4efbed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -117,7 +117,7 @@ sources = ["src"] enable-by-default = false dependencies = [ "hatch-mypyc>=0.16.0", - "mypy==1.3", + "mypy==1.5.1", "click==8.1.3", # avoid https://github.com/pallets/click/issues/2558 ] require-runtime-dependencies = true From 4e93f2aa01606154dc6af6e494b9f2b7e4c8c7fa Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 8 Sep 2023 22:16:25 -0700 Subject: [PATCH 097/279] Add classifier for 3.12 (#3866) --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 8585f4efbed..ebfbede8559 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Software Development :: Quality Assurance", ] From add161b367a0d5a3cc395ec8e045f7b965edaef8 Mon Sep 17 00:00:00 2001 From: Richard Si Date: Sat, 9 Sep 2023 12:08:28 -0400 Subject: [PATCH 098/279] Bump RTD Python version from 3.8 to 3.11 (#3868) Recent ReadTheDocs builds have been failing as our documentation dependencies (notably Sphinx) require Python 3.9+. --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index fff2d6ed341..fa612668850 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -6,7 +6,7 @@ formats: build: os: ubuntu-22.04 tools: - python: "3.8" + python: "3.11" python: install: From 4eebfd1a7a4aa2652cfc674cf301d1f2f48098aa Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 10 Sep 2023 07:53:27 -0700 Subject: [PATCH 099/279] Add mypyc test marks to new tests that patch (#3871) This is enough for me to get a clean test run on Python 3.9 with mypyc. I have not been able to repro the pickle failures on either Linux or macOS. --- tests/test_black.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_black.py b/tests/test_black.py index 4fb6aef9bca..badb8fff5fb 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1985,6 +1985,7 @@ def test_cache_multiple_files(self) -> None: assert not cache.is_changed(one) assert not cache.is_changed(two) + @pytest.mark.incompatible_with_mypyc @pytest.mark.parametrize("color", [False, True], ids=["no-color", "with-color"]) def test_no_cache_when_writeback_diff(self, color: bool) -> None: mode = DEFAULT_MODE @@ -2046,6 +2047,7 @@ def test_write_cache_read_cache(self) -> None: read_cache = black.Cache.read(mode) assert not read_cache.is_changed(src) + @pytest.mark.incompatible_with_mypyc def test_filter_cached(self) -> None: with TemporaryDirectory() as workspace: path = Path(workspace) From c83ad6c077e7bb281cfd3fbdd89bbeb4c980e563 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 10 Sep 2023 15:36:25 -0600 Subject: [PATCH 100/279] Upgrade to Furo 2023.9.10 to fix docs build (#3873) --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index dad39f67ed3..b8bab4393ff 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,4 +6,4 @@ Sphinx==7.2.3 docutils==0.19 sphinxcontrib-programoutput==0.17 sphinx_copybutton==0.5.2 -furo==2023.8.19 +furo==2023.9.10 From 0b62b9c9a44a995e44d64ecf7cc08d8d7037642d Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 10 Sep 2023 15:45:13 -0700 Subject: [PATCH 101/279] Ignore aiohttp DeprecationWarning for 3.12 (#3876) Co-authored-by: Marc Mueller <30130371+cdce8p@users.noreply.github.com> --- pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ebfbede8559..3d81e8f5ba9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -209,7 +209,10 @@ filterwarnings = [ # https://github.com/aio-libs/aiohttp/issues/6905 '''ignore:'cgi' is deprecated and slated for removal in Python 3.13:DeprecationWarning''', # Work around https://github.com/pytest-dev/pytest/issues/10977 for Python 3.12 - '''ignore:(Attribute s|Attribute n|ast.Str|ast.Bytes|ast.NameConstant|ast.Num) is deprecated and will be removed in Python 3.14:DeprecationWarning''' + '''ignore:(Attribute s|Attribute n|ast.Str|ast.Bytes|ast.NameConstant|ast.Num) is deprecated and will be removed in Python 3.14:DeprecationWarning''', + # Will be fixed with aiohttp 3.9.0 + # https://github.com/aio-libs/aiohttp/pull/7302 + "ignore:datetime.*utcfromtimestamp\\(\\) is deprecated and scheduled for removal:DeprecationWarning", ] [tool.coverage.report] omit = [ From f7917453c99f8183ffd0397affcccb2c37594771 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 10 Sep 2023 16:12:20 -0700 Subject: [PATCH 102/279] Re-export black.Mode (#3875) --- src/black/__init__.py | 11 +++-------- tests/test_black.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 6fc91d2e6d3..188a4f79f0e 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -63,14 +63,9 @@ ) from black.linegen import LN, LineGenerator, transform_line from black.lines import EmptyLineTracker, LinesBlock -from black.mode import ( - FUTURE_FLAG_TO_FEATURE, - VERSION_TO_FEATURES, - Feature, - Mode, - TargetVersion, - supports_feature, -) +from black.mode import FUTURE_FLAG_TO_FEATURE, VERSION_TO_FEATURES, Feature +from black.mode import Mode as Mode # re-exported +from black.mode import TargetVersion, supports_feature from black.nodes import ( STARS, is_number_token, diff --git a/tests/test_black.py b/tests/test_black.py index badb8fff5fb..d22b6859607 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -2482,6 +2482,41 @@ def test_get_sources_with_stdin_filename_and_force_exclude(self) -> None: ) +class TestDeFactoAPI: + """Test that certain symbols that are commonly used externally keep working. + + We don't (yet) formally expose an API (see issue #779), but we should endeavor to + keep certain functions that external users commonly rely on working. + + """ + + def test_format_str(self) -> None: + # format_str and Mode should keep working + assert ( + black.format_str("print('hello')", mode=black.Mode()) == 'print("hello")\n' + ) + + # you can pass line length + assert ( + black.format_str("print('hello')", mode=black.Mode(line_length=42)) + == 'print("hello")\n' + ) + + # invalid input raises InvalidInput + with pytest.raises(black.InvalidInput): + black.format_str("syntax error", mode=black.Mode()) + + def test_format_file_contents(self) -> None: + # You probably should be using format_str() instead, but let's keep + # this one around since people do use it + assert ( + black.format_file_contents("x=1", fast=True, mode=black.Mode()) == "x = 1\n" + ) + + with pytest.raises(black.NothingChanged): + black.format_file_contents("x = 1\n", fast=True, mode=black.Mode()) + + try: with open(black.__file__, "r", encoding="utf-8") as _bf: black_source_lines = _bf.readlines() From 751583a1dfc691423d96d7711a4c8d9cfe3ee9c8 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 10 Sep 2023 16:16:24 -0700 Subject: [PATCH 103/279] Pickle raw tuples in FileData cache (#3877) Co-authored-by: Marc Mueller <30130371+cdce8p@users.noreply.github.com> --- CHANGES.md | 3 +++ src/black/cache.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1334efefe7b..9fa14f3ebc4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -32,6 +32,9 @@ +- Store raw tuples instead of NamedTuples in Black's cache, improving performance and + decreasing the size of the cache (#3877) + ### Output diff --git a/src/black/cache.py b/src/black/cache.py index ff15da2a94e..77f66cc34a9 100644 --- a/src/black/cache.py +++ b/src/black/cache.py @@ -67,7 +67,8 @@ def read(cls, mode: Mode) -> Self: with cache_file.open("rb") as fobj: try: - file_data: Dict[str, FileData] = pickle.load(fobj) + data: Dict[str, Tuple[float, int, str]] = pickle.load(fobj) + file_data = {k: FileData(*v) for k, v in data.items()} except (pickle.UnpicklingError, ValueError, IndexError): return cls(mode, cache_file) @@ -129,7 +130,12 @@ def write(self, sources: Iterable[Path]) -> None: with tempfile.NamedTemporaryFile( dir=str(self.cache_file.parent), delete=False ) as f: - pickle.dump(self.file_data, f, protocol=4) + # We store raw tuples in the cache because pickling NamedTuples + # doesn't work with mypyc on Python 3.8, and because it's faster. + data: Dict[str, Tuple[float, int, str]] = { + k: (*v,) for k, v in self.file_data.items() + } + pickle.dump(data, f, protocol=4) os.replace(f.name, self.cache_file) except OSError: pass From 62dca32dc55a850f39d78ba8b9c21cc4261a2ddf Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 10 Sep 2023 16:47:08 -0700 Subject: [PATCH 104/279] mypyc builds on PRs, skip mypyc wheels for 3.12 (#3870) Co-authored-by: Jelle Zijlstra --- .github/workflows/pypi_upload.yml | 12 ++++++++---- pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index 9be231dd305..ab2c6402c23 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -1,8 +1,9 @@ -name: Publish to PyPI +name: Build wheels and publish to PyPI on: release: types: [published] + pull_request: permissions: contents: read @@ -28,7 +29,8 @@ jobs: - name: Build wheel and source distributions run: python -m build - - name: Upload to PyPI via Twine + - if: github.event_name == 'release' + name: Upload to PyPI via Twine env: TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} run: twine upload --verbose -u '__token__' dist/* @@ -68,7 +70,8 @@ jobs: name: ${{ matrix.name }}-mypyc-wheels path: ./wheelhouse/*.whl - - name: Upload wheels to PyPI via Twine + - if: github.event_name == 'release' + name: Upload wheels to PyPI via Twine env: TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} run: pipx run twine upload --verbose -u '__token__' wheelhouse/*.whl @@ -87,7 +90,8 @@ jobs: ref: stable fetch-depth: 0 - - name: Update stable branch to release tag & push + - if: github.event_name == 'release' + name: Update stable branch to release tag & push run: | git reset --hard ${{ github.event.release.tag_name }} git push diff --git a/pyproject.toml b/pyproject.toml index 3d81e8f5ba9..159907ac8ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -146,7 +146,7 @@ build-verbosity = 1 # - Architecture (64-bit only): amd64 / x86_64, universal2, and arm64 # - OS: Linux (no musl), Windows, and macOS build = "cp3*-*" -skip = ["*-manylinux_i686", "*-musllinux_*", "*-win32", "pp-*"] +skip = ["*-manylinux_i686", "*-musllinux_*", "*-win32", "pp-*", "cp312-*"] # This is the bare minimum needed to run the test suite. Pulling in the full # test_requirements.txt would download a bunch of other packages not necessary # here and would slow down the testing step a fair bit. From e87737140f32d3cd7c44ede75f02dcd58e55820e Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 10 Sep 2023 17:35:41 -0700 Subject: [PATCH 105/279] Prepare release 23.9.1 (#3878) --- CHANGES.md | 23 ++++++++++++++++----- docs/integrations/source_version_control.md | 4 ++-- docs/usage_and_configuration/the_basics.md | 6 +++--- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9fa14f3ebc4..a68106ad23f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -22,8 +22,6 @@ -- Upgrade to mypy 1.5.1 (#3864) - ### Parser @@ -32,9 +30,6 @@ -- Store raw tuples instead of NamedTuples in Black's cache, improving performance and - decreasing the size of the cache (#3877) - ### Output @@ -52,6 +47,24 @@ +## 23.9.1 + +Due to various issues, the previous release (23.9.0) did not include compiled mypyc +wheels, which make Black significantly faster. These issues have now been fixed, and +this release should come with compiled wheels once again. + +There will be no wheels for Python 3.12 due to a bug in mypyc. We will provide 3.12 +wheels in a future release as soon as the mypyc bug is fixed. + +### Packaging + +- Upgrade to mypy 1.5.1 (#3864) + +### Performance + +- Store raw tuples instead of NamedTuples in Black's cache, improving performance and + decreasing the size of the cache (#3877) + ## 23.9.0 ### Preview style diff --git a/docs/integrations/source_version_control.md b/docs/integrations/source_version_control.md index 28414973ff5..2afcc02f3cd 100644 --- a/docs/integrations/source_version_control.md +++ b/docs/integrations/source_version_control.md @@ -8,7 +8,7 @@ Use [pre-commit](https://pre-commit.com/). Once you repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.9.0 + rev: 23.9.1 hooks: - id: black # It is recommended to specify the latest version of Python @@ -35,7 +35,7 @@ include Jupyter Notebooks. To use this hook, simply replace the hook's `id: blac repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.9.0 + rev: 23.9.1 hooks: - id: black-jupyter # It is recommended to specify the latest version of Python diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 6ae9441fb69..57fafd87654 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -193,8 +193,8 @@ configuration file for consistent results across environments. ```console $ black --version -black, 23.9.0 (compiled: yes) -$ black --required-version 23.9.0 -c "format = 'this'" +black, 23.9.1 (compiled: yes) +$ black --required-version 23.9.1 -c "format = 'this'" format = "this" $ black --required-version 31.5b2 -c "still = 'beta?!'" Oh no! 💥 💔 💥 The required version does not match the running version! @@ -285,7 +285,7 @@ You can check the version of _Black_ you have installed using the `--version` fl ```console $ black --version -black, 23.9.0 +black, 23.9.1 ``` #### `--config` From 213cb655188fd56c548be3f0d9191c30595407ca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 09:34:36 -0700 Subject: [PATCH 106/279] Bump actions/checkout from 3 to 4 (#3883) Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/changelog.yml | 2 +- .github/workflows/diff_shades.yml | 4 ++-- .github/workflows/diff_shades_comment.yml | 2 +- .github/workflows/doc.yml | 2 +- .github/workflows/docker.yml | 2 +- .github/workflows/fuzz.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/pypi_upload.yml | 6 +++--- .github/workflows/test.yml | 6 +++--- .github/workflows/upload_binary.yml | 2 +- 10 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index b3e1f0b9024..a1804597d7d 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Grep CHANGES.md for PR number if: contains(github.event.pull_request.labels.*.name, 'skip news') != true diff --git a/.github/workflows/diff_shades.yml b/.github/workflows/diff_shades.yml index d685ef9456d..637bd527eaa 100644 --- a/.github/workflows/diff_shades.yml +++ b/.github/workflows/diff_shades.yml @@ -19,7 +19,7 @@ jobs: matrix: ${{ steps.set-config.outputs.matrix }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: "*" @@ -52,7 +52,7 @@ jobs: steps: - name: Checkout this repository (full clone) - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: # The baseline revision could be rather old so a full clone is ideal. fetch-depth: 0 diff --git a/.github/workflows/diff_shades_comment.yml b/.github/workflows/diff_shades_comment.yml index 22c293f91d2..b86bd93410e 100644 --- a/.github/workflows/diff_shades_comment.yml +++ b/.github/workflows/diff_shades_comment.yml @@ -12,7 +12,7 @@ jobs: comment: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: "*" diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index fc94dea62d9..fa3d87c70f5 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -21,7 +21,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up latest Python uses: actions/setup-python@v4 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8baace940ba..566fc880785 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v2 diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 4439148a1c7..1b5a50c0e0b 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -25,7 +25,7 @@ jobs: python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 064d4745a53..3eaf5785f5a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Assert PR target is main if: github.event_name == 'pull_request' && github.repository == 'psf/black' diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index ab2c6402c23..bf4d8349c95 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up latest Python uses: actions/setup-python@v4 @@ -57,7 +57,7 @@ jobs: macos_arch: "universal2" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build wheels via cibuildwheel uses: pypa/cibuildwheel@v2.15.0 @@ -85,7 +85,7 @@ jobs: steps: - name: Checkout stable branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: stable fetch-depth: 0 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7daa31ee903..1f33f2b814f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,7 @@ jobs: os: [ubuntu-latest, macOS-latest, windows-latest] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 @@ -75,7 +75,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Send finished signal to Coveralls uses: AndreMiras/coveralls-python-action@8799c9f4443ac4201d2e2f2c725d577174683b99 with: @@ -93,7 +93,7 @@ jobs: os: [ubuntu-latest, macOS-latest] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up latest Python uses: actions/setup-python@v4 diff --git a/.github/workflows/upload_binary.yml b/.github/workflows/upload_binary.yml index 22535a64c67..bb19d48158c 100644 --- a/.github/workflows/upload_binary.yml +++ b/.github/workflows/upload_binary.yml @@ -29,7 +29,7 @@ jobs: executable_mime: "application/x-mach-binary" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up latest Python uses: actions/setup-python@v4 From e73662ca7cfd6d4760e11a6ab489a1ec585d1cd4 Mon Sep 17 00:00:00 2001 From: Simon Alinder <92031780+AlinderS@users.noreply.github.com> Date: Mon, 11 Sep 2023 18:47:47 +0200 Subject: [PATCH 107/279] Fix broken url in editors.md (#3885) * Fix broken url in editors.md Update a link pointing to the Arch Linux repos. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- docs/integrations/editors.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/integrations/editors.md b/docs/integrations/editors.md index cebe2b0721e..ab8c6a743e5 100644 --- a/docs/integrations/editors.md +++ b/docs/integrations/editors.md @@ -288,8 +288,8 @@ $ git checkout origin/stable -b stable ##### Arch Linux On Arch Linux, the plugin is shipped with the -[`python-black`](https://archlinux.org/packages/community/any/python-black/) package, so -you can start using it in Vim after install with no additional setup. +[`python-black`](https://archlinux.org/packages/extra/any/python-black/) package, so you +can start using it in Vim after install with no additional setup. ##### Vim 8 Native Plugin Management From b2f03f913282b359d6301c426093b59b04303cff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 13:19:57 -0700 Subject: [PATCH 108/279] Bump sphinx from 7.2.3 to 7.2.5 in /docs (#3882) Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 7.2.3 to 7.2.5. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v7.2.3...v7.2.5) --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index b8bab4393ff..e4471b76031 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,7 @@ # Used by ReadTheDocs; pinned requirements for stability. myst-parser==2.0.0 -Sphinx==7.2.3 +Sphinx==7.2.5 # Older versions break Sphinx even though they're declared to be supported. docutils==0.19 sphinxcontrib-programoutput==0.17 From 14f60c84c84d6f872c09ec2171a873cac75f4c0f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 13:20:36 -0700 Subject: [PATCH 109/279] Bump docutils from 0.19 to 0.20.1 in /docs (#3699) Bumps [docutils](https://docutils.sourceforge.io/) from 0.19 to 0.20.1. --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index e4471b76031..a29e295ae49 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -3,7 +3,7 @@ myst-parser==2.0.0 Sphinx==7.2.5 # Older versions break Sphinx even though they're declared to be supported. -docutils==0.19 +docutils==0.20.1 sphinxcontrib-programoutput==0.17 sphinx_copybutton==0.5.2 furo==2023.9.10 From 004fb79706a02c9a06abd5c416b033340f99e558 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 11 Sep 2023 13:36:37 -0700 Subject: [PATCH 110/279] mypyc build improvements (#3881) Build in separate jobs. This makes it clearer if e.g. a single Python version is failing. It also potentially gets you more parallelism. Build everything on push to master. Only build Linux 3.8 and 3.11 wheels on PRs. --- .github/workflows/pypi_upload.yml | 71 ++++++++++++++++++++++--------- pyproject.toml | 4 +- 2 files changed, 52 insertions(+), 23 deletions(-) diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index bf4d8349c95..813ac39186f 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -1,9 +1,12 @@ -name: Build wheels and publish to PyPI +name: Build and publish on: release: types: [published] pull_request: + push: + branches: + - main permissions: contents: read @@ -12,6 +15,7 @@ jobs: main: name: sdist + pure wheel runs-on: ubuntu-latest + if: github.event_name == 'release' steps: - uses: actions/checkout@v4 @@ -35,34 +39,58 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} run: twine upload --verbose -u '__token__' dist/* + generate_wheels_matrix: + name: generate wheels matrix + runs-on: ubuntu-latest + outputs: + include: ${{ steps.set-matrix.outputs.include }} + steps: + - uses: actions/checkout@v3 + - name: Install cibuildwheel and pypyp + run: | + pipx install cibuildwheel==2.15.0 + pipx install pypyp==1 + - name: generate matrix + if: github.event_name != 'pull_request' + run: | + { + cibuildwheel --print-build-identifiers --platform linux \ + | pyp 'json.dumps({"only": x, "os": "ubuntu-latest"})' \ + && cibuildwheel --print-build-identifiers --platform macos \ + | pyp 'json.dumps({"only": x, "os": "macos-latest"})' \ + && cibuildwheel --print-build-identifiers --platform windows \ + | pyp 'json.dumps({"only": x, "os": "windows-latest"})' + } | pyp 'json.dumps(list(map(json.loads, lines)))' > /tmp/matrix + env: + CIBW_ARCHS_LINUX: x86_64 + CIBW_ARCHS_MACOS: x86_64 arm64 + CIBW_ARCHS_WINDOWS: AMD64 + - name: generate matrix (PR) + if: github.event_name == 'pull_request' + run: | + cibuildwheel --print-build-identifiers --platform linux \ + | pyp 'json.dumps({"only": x, "os": "ubuntu-latest"})' \ + | pyp 'json.dumps(list(map(json.loads, lines)))' > /tmp/matrix + env: + CIBW_BUILD: "cp38-* cp311-*" + CIBW_ARCHS_LINUX: x86_64 + - id: set-matrix + run: echo "include=$(cat /tmp/matrix)" | tee -a $GITHUB_OUTPUT + mypyc: - name: mypyc wheels (${{ matrix.name }}) + name: mypyc wheels ${{ matrix.only }} + needs: generate_wheels_matrix runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - include: - - os: ubuntu-latest - name: linux-x86_64 - - os: windows-2019 - name: windows-amd64 - - os: macos-11 - name: macos-x86_64 - macos_arch: "x86_64" - - os: macos-11 - name: macos-arm64 - macos_arch: "arm64" - - os: macos-11 - name: macos-universal2 - macos_arch: "universal2" + include: ${{ fromJson(needs.generate_wheels_matrix.outputs.include) }} steps: - uses: actions/checkout@v4 - - - name: Build wheels via cibuildwheel - uses: pypa/cibuildwheel@v2.15.0 - env: - CIBW_ARCHS_MACOS: "${{ matrix.macos_arch }}" + - uses: pypa/cibuildwheel@v2.15.0 + with: + only: ${{ matrix.only }} - name: Upload wheels as workflow artifacts uses: actions/upload-artifact@v3 @@ -80,6 +108,7 @@ jobs: name: Update stable branch needs: [main, mypyc] runs-on: ubuntu-latest + if: github.event_name == 'release' permissions: contents: write diff --git a/pyproject.toml b/pyproject.toml index 159907ac8ec..d246eb0b272 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -145,8 +145,8 @@ build-verbosity = 1 # - Python: CPython 3.8+ only # - Architecture (64-bit only): amd64 / x86_64, universal2, and arm64 # - OS: Linux (no musl), Windows, and macOS -build = "cp3*-*" -skip = ["*-manylinux_i686", "*-musllinux_*", "*-win32", "pp-*", "cp312-*"] +build = "cp3*" +skip = ["*-manylinux_i686", "*-musllinux_*", "*-win32", "pp*", "cp312-*"] # This is the bare minimum needed to run the test suite. Pulling in the full # test_requirements.txt would download a bunch of other packages not necessary # here and would slow down the testing step a fair bit. From e9356c1ff0083aea4416bf1d3e29748634bb4f7f Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Wed, 13 Sep 2023 00:40:41 -0700 Subject: [PATCH 111/279] Document disabling E704 (#3888) Linking #3887 --- .flake8 | 2 +- docs/guides/using_black_with_other_tools.md | 2 +- docs/the_black_code_style/current_style.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.flake8 b/.flake8 index 7bc346a09c1..85f51cf9f05 100644 --- a/.flake8 +++ b/.flake8 @@ -1,6 +1,6 @@ [flake8] # B905 should be enabled when we drop support for 3.9 -ignore = E203, E266, E501, W503, B905, B907 +ignore = E203, E266, E501, E704, W503, B905, B907 # line length is intentionally set to 80 here because black uses Bugbear # See https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#line-length for more details max-line-length = 80 diff --git a/docs/guides/using_black_with_other_tools.md b/docs/guides/using_black_with_other_tools.md index 6c6fbb88174..22c641a7420 100644 --- a/docs/guides/using_black_with_other_tools.md +++ b/docs/guides/using_black_with_other_tools.md @@ -173,7 +173,7 @@ limit of `88`, _Black_'s default. This explains `max-line-length = 88`. ```ini [flake8] max-line-length = 88 -extend-ignore = E203 +extend-ignore = E203, E704 ``` diff --git a/docs/the_black_code_style/current_style.md b/docs/the_black_code_style/current_style.md index e1a8078bf2c..ff757a8276b 100644 --- a/docs/the_black_code_style/current_style.md +++ b/docs/the_black_code_style/current_style.md @@ -173,7 +173,7 @@ If you use Flake8, you have a few options: max-line-length = 80 ... select = C,E,F,W,B,B950 - extend-ignore = E203, E501 + extend-ignore = E203, E501, E704 ``` The rationale for E950 is explained in @@ -184,7 +184,7 @@ If you use Flake8, you have a few options: ```ini [flake8] max-line-length = 88 - extend-ignore = E203 + extend-ignore = E203, E704 ``` An explanation of why E203 is disabled can be found in the [Slices section](#slices) of From 5a0615a7edd3339718a346577f03cf07da364025 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 06:47:02 -0700 Subject: [PATCH 112/279] Bump sphinx from 7.2.5 to 7.2.6 in /docs (#3895) Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 7.2.5 to 7.2.6. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES.rst) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v7.2.5...v7.2.6) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index a29e295ae49..b5b9e22fc84 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,7 @@ # Used by ReadTheDocs; pinned requirements for stability. myst-parser==2.0.0 -Sphinx==7.2.5 +Sphinx==7.2.6 # Older versions break Sphinx even though they're declared to be supported. docutils==0.20.1 sphinxcontrib-programoutput==0.17 From 34ed4cf8fd14eb423e3eb0fabf558aee93868a35 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 06:47:26 -0700 Subject: [PATCH 113/279] Bump docker/build-push-action from 4 to 5 (#3894) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 4 to 5. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v4...v5) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 566fc880785..41f92b58853 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -36,7 +36,7 @@ jobs: latest_non_release)" >> $GITHUB_ENV - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . platforms: linux/amd64,linux/arm64 @@ -47,7 +47,7 @@ jobs: if: ${{ github.event_name == 'release' && github.event.action == 'published' && !github.event.release.prerelease }} - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . platforms: linux/amd64,linux/arm64 @@ -58,7 +58,7 @@ jobs: if: ${{ github.event_name == 'release' && github.event.action == 'published' && github.event.release.prerelease }} - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . platforms: linux/amd64,linux/arm64 From ab92daf408727718849d16fcd13590006e52c1bd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 06:47:43 -0700 Subject: [PATCH 114/279] Bump docker/login-action from 2 to 3 (#3891) Bumps [docker/login-action](https://github.com/docker/login-action) from 2 to 3. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 41f92b58853..a5992f758c1 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -25,7 +25,7 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} From edf66baa21d337ae069be3871e95f88b68a3ffcf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 06:48:03 -0700 Subject: [PATCH 115/279] Bump docker/setup-buildx-action from 2 to 3 (#3892) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 2 to 3. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a5992f758c1..80930ccfee8 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -22,7 +22,7 @@ jobs: uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub uses: docker/login-action@v3 From f5990e85474f7641717e70bafb797462995d974f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 06:48:11 -0700 Subject: [PATCH 116/279] Bump docker/setup-qemu-action from 2 to 3 (#3890) Bumps [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) from 2 to 3. - [Release notes](https://github.com/docker/setup-qemu-action/releases) - [Commits](https://github.com/docker/setup-qemu-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/setup-qemu-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 80930ccfee8..ee858236fcf 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 From 7316a793187eedd94c288f1df648ecca0d8763dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 06:48:27 -0700 Subject: [PATCH 117/279] Bump actions/checkout from 3 to 4 (#3893) Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pypi_upload.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index 813ac39186f..2a74b7c6a55 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -45,7 +45,7 @@ jobs: outputs: include: ${{ steps.set-matrix.outputs.include }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install cibuildwheel and pypyp run: | pipx install cibuildwheel==2.15.0 From e974fc3c52959e9f01bf62bdcd0d8c100ce78985 Mon Sep 17 00:00:00 2001 From: Eero Vaher Date: Mon, 18 Sep 2023 20:35:07 +0300 Subject: [PATCH 118/279] Remove outdated mentions of runtime support of Python 3.7 (#3896) Remove mentions of runtime support of Python 3.7 Runtime support of Python 3.7 was removed in b4dca26c7d93f930bbd5a7b552807370b60d4298 but a few mentions of it being supported have remained until now. --- README.md | 2 +- autoload/black.vim | 4 ++-- docs/faq.md | 6 +++--- docs/getting_started.md | 2 +- docs/integrations/editors.md | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index b257c333f0d..cad8184f7bc 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ Try it out now using the [Black Playground](https://black.vercel.app). Watch the ### Installation -_Black_ can be installed by running `pip install black`. It requires Python 3.7+ to run. +_Black_ can be installed by running `pip install black`. It requires Python 3.8+ to run. If you want to format Jupyter Notebooks, install with `pip install "black[jupyter]"`. If you can't wait for the latest _hotness_ and want to install from GitHub, use: diff --git a/autoload/black.vim b/autoload/black.vim index 4eb9b25db28..051fea05c3b 100644 --- a/autoload/black.vim +++ b/autoload/black.vim @@ -75,8 +75,8 @@ def _initialize_black_env(upgrade=False): return True pyver = sys.version_info[:3] - if pyver < (3, 7): - print("Sorry, Black requires Python 3.7+ to run.") + if pyver < (3, 8): + print("Sorry, Black requires Python 3.8+ to run.") return False from pathlib import Path diff --git a/docs/faq.md b/docs/faq.md index 8941ca3fe4d..c62e1b504b5 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -86,7 +86,7 @@ disabled-by-default counterpart W504. E203 should be disabled while changes are ## Which Python versions does Black support? -Currently the runtime requires Python 3.7-3.11. Formatting is supported for files +Currently the runtime requires Python 3.8-3.11. Formatting is supported for files containing syntax from Python 3.3 to 3.11. We promise to support at least all Python versions that have not reached their end of life. This is the case for both running _Black_ and formatting code. @@ -95,7 +95,7 @@ Support for formatting Python 2 code was removed in version 22.0. While we've ma plans to stop supporting older Python 3 minor versions immediately, their support might also be removed some time in the future without a deprecation period. -Runtime support for 3.6 was removed in version 22.10.0. +Runtime support for 3.7 was removed in version 23.7.0. ## Why does my linter or typechecker complain after I format my code? @@ -107,7 +107,7 @@ codebase with _Black_. ## Can I run Black with PyPy? -Yes, there is support for PyPy 3.7 and higher. +Yes, there is support for PyPy 3.8 and higher. ## Why does Black not detect syntax errors in my code? diff --git a/docs/getting_started.md b/docs/getting_started.md index 33fb2f978bb..15b7646a509 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -16,7 +16,7 @@ Also, you can try out _Black_ online for minimal fuss on the ## Installation -_Black_ can be installed by running `pip install black`. It requires Python 3.7+ to run. +_Black_ can be installed by running `pip install black`. It requires Python 3.8+ to run. If you want to format Jupyter Notebooks, install with `pip install "black[jupyter]"`. If you can't wait for the latest _hotness_ and want to install from GitHub, use: diff --git a/docs/integrations/editors.md b/docs/integrations/editors.md index ab8c6a743e5..9c7cfe19083 100644 --- a/docs/integrations/editors.md +++ b/docs/integrations/editors.md @@ -236,7 +236,7 @@ Configuration: #### Installation -This plugin **requires Vim 7.0+ built with Python 3.7+ support**. It needs Python 3.7 to +This plugin **requires Vim 7.0+ built with Python 3.8+ support**. It needs Python 3.8 to be able to run _Black_ inside the Vim process which is much faster than calling an external command. From 8c5d96ffd3da6d621e67639dadd26a1d7d0227cd Mon Sep 17 00:00:00 2001 From: John Litborn <11260241+jakkdl@users.noreply.github.com> Date: Fri, 22 Sep 2023 17:38:51 +0200 Subject: [PATCH 119/279] fix indentation of line breaks in long type hints by adding parens (#3899) * fix indentation of line breaks in long type hints by adding parentheses, and remove unnecessary parentheses * add entry in CHANGES.md, make the style change only in preview mode --- CHANGES.md | 3 + src/black/linegen.py | 30 ++- src/black/mode.py | 1 + .../preview/long_strings__type_annotations.py | 2 +- .../pep604_union_types_line_breaks.py | 187 ++++++++++++++++++ 5 files changed, 220 insertions(+), 3 deletions(-) create mode 100644 tests/data/preview_py_310/pep604_union_types_line_breaks.py diff --git a/CHANGES.md b/CHANGES.md index a68106ad23f..a879ab3e8da 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,6 +12,9 @@ ### Preview style +- Long type hints are now wrapped in parentheses and properly indented when split across + multiple lines (#3899) + ### Configuration diff --git a/src/black/linegen.py b/src/black/linegen.py index 507e860190f..9ddd4619f69 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -397,6 +397,24 @@ def visit_factor(self, node: Node) -> Iterator[Line]: node.insert_child(index, Node(syms.atom, [lpar, operand, rpar])) yield from self.visit_default(node) + def visit_tname(self, node: Node) -> Iterator[Line]: + """ + Add potential parentheses around types in function parameter lists to be made + into real parentheses in case the type hint is too long to fit on a line + Examples: + def foo(a: int, b: float = 7): ... + + -> + + def foo(a: (int), b: (float) = 7): ... + """ + if Preview.parenthesize_long_type_hints in self.mode: + assert len(node.children) == 3 + if maybe_make_parens_invisible_in_atom(node.children[2], parent=node): + wrap_in_parentheses(node, node.children[2], visible=False) + + yield from self.visit_default(node) + def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: if Preview.hex_codes_in_unicode_sequences in self.mode: normalize_unicode_escape_sequences(leaf) @@ -498,7 +516,14 @@ def __post_init__(self) -> None: self.visit_except_clause = partial(v, keywords={"except"}, parens={"except"}) self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"}) self.visit_classdef = partial(v, keywords={"class"}, parens=Ø) - self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS) + + # When this is moved out of preview, add ":" directly to ASSIGNMENTS in nodes.py + if Preview.parenthesize_long_type_hints in self.mode: + assignments = ASSIGNMENTS | {":"} + else: + assignments = ASSIGNMENTS + self.visit_expr_stmt = partial(v, keywords=Ø, parens=assignments) + self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"}) self.visit_import_from = partial(v, keywords=Ø, parens={"import"}) self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"}) @@ -1368,7 +1393,7 @@ def maybe_make_parens_invisible_in_atom( Returns whether the node should itself be wrapped in invisible parentheses. """ if ( - node.type != syms.atom + node.type not in (syms.atom, syms.expr) or is_empty_tuple(node) or is_one_tuple(node) or (is_yield(node) and parent.type != syms.expr_stmt) @@ -1392,6 +1417,7 @@ def maybe_make_parens_invisible_in_atom( syms.except_clause, syms.funcdef, syms.with_stmt, + syms.tname, # these ones aren't useful to end users, but they do please fuzzers syms.for_stmt, syms.del_stmt, diff --git a/src/black/mode.py b/src/black/mode.py index 8a855ac495a..f44a821bcd0 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -180,6 +180,7 @@ class Preview(Enum): # for https://github.com/psf/black/issues/3117 to be fixed. string_processing = auto() parenthesize_conditional_expressions = auto() + parenthesize_long_type_hints = auto() skip_magic_trailing_comma_in_subscript = auto() wrap_long_dict_values_in_parens = auto() wrap_multiple_context_managers_in_parens = auto() diff --git a/tests/data/preview/long_strings__type_annotations.py b/tests/data/preview/long_strings__type_annotations.py index 41d7ee2b67b..45de882d02c 100644 --- a/tests/data/preview/long_strings__type_annotations.py +++ b/tests/data/preview/long_strings__type_annotations.py @@ -54,6 +54,6 @@ def func( def func( - argument: ("int |" "str"), + argument: "int |" "str", ) -> Set["int |" " str"]: pass diff --git a/tests/data/preview_py_310/pep604_union_types_line_breaks.py b/tests/data/preview_py_310/pep604_union_types_line_breaks.py new file mode 100644 index 00000000000..9c4ab870766 --- /dev/null +++ b/tests/data/preview_py_310/pep604_union_types_line_breaks.py @@ -0,0 +1,187 @@ +# This has always worked +z= Loooooooooooooooooooooooong | Loooooooooooooooooooooooong | Loooooooooooooooooooooooong | Loooooooooooooooooooooooong + +# "AnnAssign"s now also work +z: Loooooooooooooooooooooooong | Loooooooooooooooooooooooong | Loooooooooooooooooooooooong | Loooooooooooooooooooooooong +z: (Short + | Short2 + | Short3 + | Short4) +z: (int) +z: ((int)) + + +z: Loooooooooooooooooooooooong | Loooooooooooooooooooooooong | Loooooooooooooooooooooooong | Loooooooooooooooooooooooong = 7 +z: (Short + | Short2 + | Short3 + | Short4) = 8 +z: (int) = 2.3 +z: ((int)) = foo() + +# In case I go for not enforcing parantheses, this might get improved at the same time +x = ( + z + == 9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999, + y + == 9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999, +) + +x = ( + z == (9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999), + y == (9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999), +) + +# handle formatting of "tname"s in parameter list + +# remove unnecessary paren +def foo(i: (int)) -> None: ... + + +# this is a syntax error in the type annotation according to mypy, but it's not invalid *python* code, so make sure we don't mess with it and make it so. +def foo(i: (int,)) -> None: ... + +def foo( + i: int, + x: Loooooooooooooooooooooooong + | Looooooooooooooooong + | Looooooooooooooooooooong + | Looooooong, + *, + s: str, +) -> None: + pass + + +@app.get("/path/") +async def foo( + q: str + | None = Query(None, title="Some long title", description="Some long description") +): + pass + + +def f( + max_jobs: int + | None = Option( + None, help="Maximum number of jobs to launch. And some additional text." + ), + another_option: bool = False + ): + ... + + +# output +# This has always worked +z = ( + Loooooooooooooooooooooooong + | Loooooooooooooooooooooooong + | Loooooooooooooooooooooooong + | Loooooooooooooooooooooooong +) + +# "AnnAssign"s now also work +z: ( + Loooooooooooooooooooooooong + | Loooooooooooooooooooooooong + | Loooooooooooooooooooooooong + | Loooooooooooooooooooooooong +) +z: Short | Short2 | Short3 | Short4 +z: int +z: int + + +z: ( + Loooooooooooooooooooooooong + | Loooooooooooooooooooooooong + | Loooooooooooooooooooooooong + | Loooooooooooooooooooooooong +) = 7 +z: Short | Short2 | Short3 | Short4 = 8 +z: int = 2.3 +z: int = foo() + +# In case I go for not enforcing parantheses, this might get improved at the same time +x = ( + z + == 9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999, + y + == 9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999, +) + +x = ( + z + == ( + 9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999 + | 9999999999999999999999999999999999999999 + ), + y + == ( + 9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999 + + 9999999999999999999999999999999999999999 + ), +) + +# handle formatting of "tname"s in parameter list + + +# remove unnecessary paren +def foo(i: int) -> None: ... + + +# this is a syntax error in the type annotation according to mypy, but it's not invalid *python* code, so make sure we don't mess with it and make it so. +def foo(i: (int,)) -> None: ... + + +def foo( + i: int, + x: ( + Loooooooooooooooooooooooong + | Looooooooooooooooong + | Looooooooooooooooooooong + | Looooooong + ), + *, + s: str, +) -> None: + pass + + +@app.get("/path/") +async def foo( + q: str | None = Query( + None, title="Some long title", description="Some long description" + ) +): + pass + + +def f( + max_jobs: int | None = Option( + None, help="Maximum number of jobs to launch. And some additional text." + ), + another_option: bool = False, +): ... From 5f6ea5ff20100290ba8e8803a924caea12d2d0b6 Mon Sep 17 00:00:00 2001 From: Syed Mohammad Ibrahim <8592115+iamibi@users.noreply.github.com> Date: Sun, 24 Sep 2023 07:53:03 +0530 Subject: [PATCH 120/279] added the py311 to target-version config (#3898) --- docs/usage_and_configuration/the_basics.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 57fafd87654..36119f225e6 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -52,18 +52,19 @@ See also [the style documentation](labels/line-length). #### `-t`, `--target-version` -Python versions that should be supported by Black's output. You should include all -versions that your code supports. If you support Python 3.7 through 3.10, you should -write: +Python versions that should be supported by Black's output. You can run `black --help` +and look for the `--target-version` option to see the full list of supported versions. +You should include all versions that your code supports. If you support Python 3.8 +through 3.11, you should write: ```console -$ black -t py37 -t py38 -t py39 -t py310 +$ black -t py38 -t py39 -t py310 -t py311 ``` In a [configuration file](#configuration-via-a-file), you can write: ```toml -target-version = ["py37", "py38", "py39", "py310"] +target-version = ["py38", "py39", "py310", "py311"] ``` _Black_ uses this option to decide what grammar to use to parse your code. In addition, From 3dcacdda0d7f69a1705f3e2a151c24a6cf004171 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Sep 2023 09:32:58 -0700 Subject: [PATCH 121/279] Bump pypa/cibuildwheel from 2.15.0 to 2.16.0 (#3901) Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.15.0 to 2.16.0. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.15.0...v2.16.0) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/pypi_upload.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index 2a74b7c6a55..026f74e1c9f 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -88,7 +88,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: pypa/cibuildwheel@v2.15.0 + - uses: pypa/cibuildwheel@v2.16.0 with: only: ${{ matrix.only }} From 9b82120ddb81373377b58da5de7caa9495a1551e Mon Sep 17 00:00:00 2001 From: John Litborn <11260241+jakkdl@users.noreply.github.com> Date: Thu, 28 Sep 2023 16:03:24 +0200 Subject: [PATCH 122/279] add support for printing the diff of AST trees when running tests (#3902) Co-authored-by: Jelle Zijlstra --- docs/contributing/the_basics.md | 38 +++++++++++++++++++++++++++++++++ src/black/debug.py | 21 ++++++++++++------ tests/conftest.py | 27 +++++++++++++++++++++++ tests/test_black.py | 29 ++++++++++++++++++++++--- tests/util.py | 24 ++++++++++++++++----- tox.ini | 2 +- 6 files changed, 125 insertions(+), 16 deletions(-) diff --git a/docs/contributing/the_basics.md b/docs/contributing/the_basics.md index 40d233257e3..864894b491f 100644 --- a/docs/contributing/the_basics.md +++ b/docs/contributing/the_basics.md @@ -37,6 +37,44 @@ the root of the black repo: (.venv)$ tox -e run_self ``` +### Development + +Further examples of invoking the tests + +```console +# Run all of the above mentioned, in parallel +(.venv)$ tox --parallel=auto + +# Run tests on a specific python version +(.venv)$ tox -e py39 + +# pass arguments to pytest +(.venv)$ tox -e py -- --no-cov + +# print full tree diff, see documentation below +(.venv)$ tox -e py -- --print-full-tree + +# disable diff printing, see documentation below +(.venv)$ tox -e py -- --print-tree-diff=False +``` + +`Black` has two pytest command-line options affecting test files in `tests/data/` that +are split into an input part, and an output part, separated by a line with`# output`. +These can be passed to `pytest` through `tox`, or directly into pytest if not using +`tox`. + +#### `--print-full-tree` + +Upon a failing test, print the full concrete syntax tree (CST) as it is after processing +the input ("actual"), and the tree that's yielded after parsing the output ("expected"). +Note that a test can fail with different output with the same CST. This used to be the +default, but now defaults to `False`. + +#### `--print-tree-diff` + +Upon a failing test, print the diff of the trees as described above. This is the +default. To turn it off pass `--print-tree-diff=False`. + ### News / Changelog Requirement `Black` has CI that will check for an entry corresponding to your PR in `CHANGES.md`. If diff --git a/src/black/debug.py b/src/black/debug.py index 150b44842dd..cebc48765ba 100644 --- a/src/black/debug.py +++ b/src/black/debug.py @@ -1,5 +1,5 @@ -from dataclasses import dataclass -from typing import Iterator, TypeVar, Union +from dataclasses import dataclass, field +from typing import Any, Iterator, List, TypeVar, Union from black.nodes import Visitor from black.output import out @@ -14,26 +14,33 @@ @dataclass class DebugVisitor(Visitor[T]): tree_depth: int = 0 + list_output: List[str] = field(default_factory=list) + print_output: bool = True + + def out(self, message: str, *args: Any, **kwargs: Any) -> None: + self.list_output.append(message) + if self.print_output: + out(message, *args, **kwargs) def visit_default(self, node: LN) -> Iterator[T]: indent = " " * (2 * self.tree_depth) if isinstance(node, Node): _type = type_repr(node.type) - out(f"{indent}{_type}", fg="yellow") + self.out(f"{indent}{_type}", fg="yellow") self.tree_depth += 1 for child in node.children: yield from self.visit(child) self.tree_depth -= 1 - out(f"{indent}/{_type}", fg="yellow", bold=False) + self.out(f"{indent}/{_type}", fg="yellow", bold=False) else: _type = token.tok_name.get(node.type, str(node.type)) - out(f"{indent}{_type}", fg="blue", nl=False) + self.out(f"{indent}{_type}", fg="blue", nl=False) if node.prefix: # We don't have to handle prefixes for `Node` objects since # that delegates to the first child anyway. - out(f" {node.prefix!r}", fg="green", bold=False, nl=False) - out(f" {node.value!r}", fg="blue", bold=False) + self.out(f" {node.prefix!r}", fg="green", bold=False, nl=False) + self.out(f" {node.value!r}", fg="blue", bold=False) @classmethod def show(cls, code: Union[str, Leaf, Node]) -> None: diff --git a/tests/conftest.py b/tests/conftest.py index 67517268d1b..1a0dd747d8e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1 +1,28 @@ +import pytest + pytest_plugins = ["tests.optional"] + +PRINT_FULL_TREE: bool = False +PRINT_TREE_DIFF: bool = True + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--print-full-tree", + action="store_true", + default=False, + help="print full syntax trees on failed tests", + ) + parser.addoption( + "--print-tree-diff", + action="store_true", + default=True, + help="print diff of syntax trees on failed tests", + ) + + +def pytest_configure(config: pytest.Config) -> None: + global PRINT_FULL_TREE + global PRINT_TREE_DIFF + PRINT_FULL_TREE = config.getoption("--print-full-tree") + PRINT_TREE_DIFF = config.getoption("--print-tree-diff") diff --git a/tests/test_black.py b/tests/test_black.py index d22b6859607..c665eee3a6c 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -9,7 +9,6 @@ import re import sys import types -import unittest from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager, redirect_stderr from dataclasses import replace @@ -1047,9 +1046,10 @@ def test_endmarker(self) -> None: self.assertEqual(len(n.children), 1) self.assertEqual(n.children[0].type, black.token.ENDMARKER) + @patch("tests.conftest.PRINT_FULL_TREE", True) + @patch("tests.conftest.PRINT_TREE_DIFF", False) @pytest.mark.incompatible_with_mypyc - @unittest.skipIf(os.environ.get("SKIP_AST_PRINT"), "user set SKIP_AST_PRINT") - def test_assertFormatEqual(self) -> None: + def test_assertFormatEqual_print_full_tree(self) -> None: out_lines = [] err_lines = [] @@ -1068,6 +1068,29 @@ def err(msg: str, **kwargs: Any) -> None: self.assertIn("Actual tree:", out_str) self.assertEqual("".join(err_lines), "") + @patch("tests.conftest.PRINT_FULL_TREE", False) + @patch("tests.conftest.PRINT_TREE_DIFF", True) + @pytest.mark.incompatible_with_mypyc + def test_assertFormatEqual_print_tree_diff(self) -> None: + out_lines = [] + err_lines = [] + + def out(msg: str, **kwargs: Any) -> None: + out_lines.append(msg) + + def err(msg: str, **kwargs: Any) -> None: + err_lines.append(msg) + + with patch("black.output._out", out), patch("black.output._err", err): + with self.assertRaises(AssertionError): + self.assertFormatEqual("j = [1, 2, 3]\n", "j = [1, 2, 3,]\n") + + out_str = "".join(out_lines) + self.assertIn("Tree Diff:", out_str) + self.assertIn("+ COMMA", out_str) + self.assertIn("+ ','", out_str) + self.assertEqual("".join(err_lines), "") + @event_loop() @patch("concurrent.futures.ProcessPoolExecutor", MagicMock(side_effect=OSError)) def test_works_in_mono_process_only_environment(self) -> None: diff --git a/tests/util.py b/tests/util.py index 967d576fafe..541d21da4df 100644 --- a/tests/util.py +++ b/tests/util.py @@ -12,6 +12,8 @@ from black.mode import TargetVersion from black.output import diff, err, out +from . import conftest + PYTHON_SUFFIX = ".py" ALLOWED_SUFFIXES = (PYTHON_SUFFIX, ".pyi", ".out", ".diff", ".ipynb") @@ -34,22 +36,34 @@ def _assert_format_equal(expected: str, actual: str) -> None: - if actual != expected and not os.environ.get("SKIP_AST_PRINT"): + if actual != expected and (conftest.PRINT_FULL_TREE or conftest.PRINT_TREE_DIFF): bdv: DebugVisitor[Any] - out("Expected tree:", fg="green") + actual_out: str = "" + expected_out: str = "" + if conftest.PRINT_FULL_TREE: + out("Expected tree:", fg="green") try: exp_node = black.lib2to3_parse(expected) - bdv = DebugVisitor() + bdv = DebugVisitor(print_output=conftest.PRINT_FULL_TREE) list(bdv.visit(exp_node)) + expected_out = "\n".join(bdv.list_output) except Exception as ve: err(str(ve)) - out("Actual tree:", fg="red") + if conftest.PRINT_FULL_TREE: + out("Actual tree:", fg="red") try: exp_node = black.lib2to3_parse(actual) - bdv = DebugVisitor() + bdv = DebugVisitor(print_output=conftest.PRINT_FULL_TREE) list(bdv.visit(exp_node)) + actual_out = "\n".join(bdv.list_output) except Exception as ve: err(str(ve)) + if conftest.PRINT_TREE_DIFF: + out("Tree Diff:") + out( + diff(expected_out, actual_out, "expected tree", "actual tree") + or "Trees do not differ" + ) if actual != expected: out(diff(expected, actual, "expected", "actual")) diff --git a/tox.ini b/tox.ini index d34dbbc71db..018cef993c0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] isolated_build = true -envlist = {,ci-}py{37,38,39,310,311,py3},fuzz,run_self +envlist = {,ci-}py{38,39,310,311,py3},fuzz,run_self [testenv] setenv = From e7c3368c1316c38338cef34fffc42ea3252b1802 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 28 Sep 2023 09:10:01 -0700 Subject: [PATCH 123/279] Try newer clang in diff-shades job (#3904) --- .github/workflows/diff_shades.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/diff_shades.yml b/.github/workflows/diff_shades.yml index 637bd527eaa..97db907abc8 100644 --- a/.github/workflows/diff_shades.yml +++ b/.github/workflows/diff_shades.yml @@ -44,7 +44,7 @@ jobs: HATCH_BUILD_HOOKS_ENABLE: "1" # Clang is less picky with the C code it's given than gcc (and may # generate faster binaries too). - CC: clang-12 + CC: clang-14 strategy: fail-fast: false matrix: From a91eb73064c9bef76c3a961ab662bb5f75a1543d Mon Sep 17 00:00:00 2001 From: Eddie Darling Date: Sun, 1 Oct 2023 15:35:42 -0700 Subject: [PATCH 124/279] Fix comments getting removed from inside parenthesized strings (#3909) Since the id of the old leaf may be the key to comments, the new leaf must adopt the old comments --- CHANGES.md | 2 ++ src/black/trans.py | 3 +++ tests/data/preview/comments7.py | 24 ++++++++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index a879ab3e8da..028a01acd62 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,8 @@ +- Fix comments getting removed from inside parenthesized strings (#3909) + ### Configuration diff --git a/src/black/trans.py b/src/black/trans.py index daed26427d7..c0cc92613ac 100644 --- a/src/black/trans.py +++ b/src/black/trans.py @@ -942,6 +942,9 @@ def _transform_to_new_line( LL[lpar_or_rpar_idx].remove() # Remove lpar. replace_child(LL[idx], string_leaf) new_line.append(string_leaf) + # replace comments + old_comments = new_line.comments.pop(id(LL[idx]), []) + new_line.comments.setdefault(id(string_leaf), []).extend(old_comments) else: LL[lpar_or_rpar_idx].remove() # This is a rpar. diff --git a/tests/data/preview/comments7.py b/tests/data/preview/comments7.py index 8b1224017e5..0655de999ec 100644 --- a/tests/data/preview/comments7.py +++ b/tests/data/preview/comments7.py @@ -131,6 +131,18 @@ def test_fails_invalid_post_data( square = Square(4) # type: Optional[Square] +# Regression test for https://github.com/psf/black/issues/3756. +[ + ( + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" # aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + ), +] +[ + ( # aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" # aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + ), +] + # output from .config import ( @@ -282,3 +294,15 @@ def test_fails_invalid_post_data( square = Square(4) # type: Optional[Square] + +# Regression test for https://github.com/psf/black/issues/3756. +[ + ( # aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ), +] +[ + ( # aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" # aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + ), +] From f99ef6e190785b3e6a58e83f1382e7d6d3c4881e Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 1 Oct 2023 15:41:32 -0700 Subject: [PATCH 125/279] Fix up changelog (#3910) --- CHANGES.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 028a01acd62..5e518497c92 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,14 +10,14 @@ -### Preview style +- Fix comments getting removed from inside parenthesized strings (#3909) -- Long type hints are now wrapped in parentheses and properly indented when split across - multiple lines (#3899) +### Preview style -- Fix comments getting removed from inside parenthesized strings (#3909) +- Long type hints are now wrapped in parentheses and properly indented when split across + multiple lines (#3899) ### Configuration From 1b08cbc63400a8b8e0fbb620b6b2a4dab35e1e7b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Oct 2023 23:40:00 -0700 Subject: [PATCH 126/279] Bump pypa/cibuildwheel from 2.16.0 to 2.16.1 (#3911) --- .github/workflows/pypi_upload.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index 026f74e1c9f..41ab6460793 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -88,7 +88,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: pypa/cibuildwheel@v2.16.0 + - uses: pypa/cibuildwheel@v2.16.1 with: only: ${{ matrix.only }} From 9e9fdce9a81a53fd3771e1825de523a6413b3c35 Mon Sep 17 00:00:00 2001 From: Shreya Agarwal Date: Mon, 2 Oct 2023 20:05:57 +0530 Subject: [PATCH 127/279] docs: use LSP for SublimeText 4 (#3913) --- docs/integrations/editors.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/integrations/editors.md b/docs/integrations/editors.md index 9c7cfe19083..83904144c48 100644 --- a/docs/integrations/editors.md +++ b/docs/integrations/editors.md @@ -399,9 +399,10 @@ close and reopen your File, _Black_ will be done with its job. server for Black. Formatting is much more responsive using this extension, **but the minimum supported version of Black is 22.3.0**. -## SublimeText 3 +## SublimeText -Use [sublack plugin](https://github.com/jgirardet/sublack). +For SublimeText 3, use [sublack plugin](https://github.com/jgirardet/sublack). For +higher versions, it is recommended to use [LSP](#python-lsp-server) as documented below. ## Python LSP Server From 947bd3825e5dc67f16f48f916462c4470b7a5247 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Oct 2023 10:19:53 -0700 Subject: [PATCH 128/279] [pre-commit.ci] pre-commit autoupdate (#3915) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/mirrors-mypy: v1.5.0 → v1.5.1](https://github.com/pre-commit/mirrors-mypy/compare/v1.5.0...v1.5.1) - [github.com/pre-commit/mirrors-prettier: v3.0.1 → v3.0.3](https://github.com/pre-commit/mirrors-prettier/compare/v3.0.1...v3.0.3) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6301526a445..99b3565ed0e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: exclude: ^src/blib2to3/ - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.5.0 + rev: v1.5.1 hooks: - id: mypy exclude: ^docs/conf.py @@ -53,7 +53,7 @@ repos: - hypothesis - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.1 + rev: v3.0.3 hooks: - id: prettier exclude: \.github/workflows/diff_shades\.yml From 36078bc83f24dcd5f74e021a105429595a3fd63c Mon Sep 17 00:00:00 2001 From: John Litborn <11260241+jakkdl@users.noreply.github.com> Date: Thu, 5 Oct 2023 01:42:35 +0200 Subject: [PATCH 129/279] respect magic trailing commas in return types (#3916) --- CHANGES.md | 1 + src/black/linegen.py | 36 ++- src/black/mode.py | 1 + .../return_annotation_brackets_string.py | 11 + .../funcdef_return_type_trailing_comma.py | 300 ++++++++++++++++++ .../return_annotation_brackets.py | 13 + 6 files changed, 361 insertions(+), 1 deletion(-) create mode 100644 tests/data/preview_py_310/funcdef_return_type_trailing_comma.py diff --git a/CHANGES.md b/CHANGES.md index 5e518497c92..888824ee055 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,6 +18,7 @@ - Long type hints are now wrapped in parentheses and properly indented when split across multiple lines (#3899) +- Magic trailing commas are now respected in return types. (#3916) ### Configuration diff --git a/src/black/linegen.py b/src/black/linegen.py index 9ddd4619f69..bdc4ee54ab2 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -573,7 +573,7 @@ def transform_line( transformers = [string_merge, string_paren_strip] else: transformers = [] - elif line.is_def: + elif line.is_def and not should_split_funcdef_with_rhs(line, mode): transformers = [left_hand_split] else: @@ -652,6 +652,40 @@ def _rhs( yield line +def should_split_funcdef_with_rhs(line: Line, mode: Mode) -> bool: + """If a funcdef has a magic trailing comma in the return type, then we should first + split the line with rhs to respect the comma. + """ + if Preview.respect_magic_trailing_comma_in_return_type not in mode: + return False + + return_type_leaves: List[Leaf] = [] + in_return_type = False + + for leaf in line.leaves: + if leaf.type == token.COLON: + in_return_type = False + if in_return_type: + return_type_leaves.append(leaf) + if leaf.type == token.RARROW: + in_return_type = True + + # using `bracket_split_build_line` will mess with whitespace, so we duplicate a + # couple lines from it. + result = Line(mode=line.mode, depth=line.depth) + leaves_to_track = get_leaves_inside_matching_brackets(return_type_leaves) + for leaf in return_type_leaves: + result.append( + leaf, + preformatted=True, + track_bracket=id(leaf) in leaves_to_track, + ) + + # we could also return true if the line is too long, and the return type is longer + # than the param list. Or if `should_split_rhs` returns True. + return result.magic_trailing_comma is not None + + class _BracketSplitComponent(Enum): head = auto() body = auto() diff --git a/src/black/mode.py b/src/black/mode.py index f44a821bcd0..30c5d2f1b2f 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -181,6 +181,7 @@ class Preview(Enum): string_processing = auto() parenthesize_conditional_expressions = auto() parenthesize_long_type_hints = auto() + respect_magic_trailing_comma_in_return_type = auto() skip_magic_trailing_comma_in_subscript = auto() wrap_long_dict_values_in_parens = auto() wrap_multiple_context_managers_in_parens = auto() diff --git a/tests/data/preview/return_annotation_brackets_string.py b/tests/data/preview/return_annotation_brackets_string.py index 6978829fd5c..9148bd045bc 100644 --- a/tests/data/preview/return_annotation_brackets_string.py +++ b/tests/data/preview/return_annotation_brackets_string.py @@ -2,6 +2,10 @@ def frobnicate() -> "ThisIsTrulyUnreasonablyExtremelyLongClassName | list[ThisIsTrulyUnreasonablyExtremelyLongClassName]": pass +# splitting the string breaks if there's any parameters +def frobnicate(a) -> "ThisIsTrulyUnreasonablyExtremelyLongClassName | list[ThisIsTrulyUnreasonablyExtremelyLongClassName]": + pass + # output # Long string example @@ -10,3 +14,10 @@ def frobnicate() -> ( " list[ThisIsTrulyUnreasonablyExtremelyLongClassName]" ): pass + + +# splitting the string breaks if there's any parameters +def frobnicate( + a, +) -> "ThisIsTrulyUnreasonablyExtremelyLongClassName | list[ThisIsTrulyUnreasonablyExtremelyLongClassName]": + pass diff --git a/tests/data/preview_py_310/funcdef_return_type_trailing_comma.py b/tests/data/preview_py_310/funcdef_return_type_trailing_comma.py new file mode 100644 index 00000000000..15db772f01e --- /dev/null +++ b/tests/data/preview_py_310/funcdef_return_type_trailing_comma.py @@ -0,0 +1,300 @@ +# normal, short, function definition +def foo(a, b) -> tuple[int, float]: ... + + +# normal, short, function definition w/o return type +def foo(a, b): ... + + +# no splitting +def foo(a: A, b: B) -> list[p, q]: + pass + + +# magic trailing comma in param list +def foo(a, b,): ... + + +# magic trailing comma in nested params in param list +def foo(a, b: tuple[int, float,]): ... + + +# magic trailing comma in return type, no params +def a() -> tuple[ + a, + b, +]: ... + + +# magic trailing comma in return type, params +def foo(a: A, b: B) -> list[ + p, + q, +]: + pass + + +# magic trailing comma in param list and in return type +def foo( + a: a, + b: b, +) -> list[ + a, + a, +]: + pass + + +# long function definition, param list is longer +def aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa( + bbbbbbbbbbbbbbbbbb, +) -> cccccccccccccccccccccccccccccc: ... + + +# long function definition, return type is longer +# this should maybe split on rhs? +def aaaaaaaaaaaaaaaaa(bbbbbbbbbbbbbbbbbb) -> list[ + Ccccccccccccccccccccccccccccccccccccccccccccccccccc, Dddddd +]: ... + + +# long return type, no param list +def foo() -> list[ + Loooooooooooooooooooooooooooooooooooong, + Loooooooooooooooooooong, + Looooooooooooong, +]: ... + + +# long function name, no param list, no return value +def thiiiiiiiiiiiiiiiiiis_iiiiiiiiiiiiiiiiiiiiiiiiiiiiiis_veeeeeeeeeeeeeeeeeeeeeeery_looooooong(): + pass + + +# long function name, no param list +def thiiiiiiiiiiiiiiiiiis_iiiiiiiiiiiiiiiiiiiiiiiiiiiiiis_veeeeeeeeeeeeeeeeeeeeeeery_looooooong() -> ( + list[int, float] +): ... + + +# long function name, no return value +def thiiiiiiiiiiiiiiiiiis_iiiiiiiiiiiiiiiiiiiiiiiiiiiiiis_veeeeeeeeeeeeeeeeeeeeeeery_looooooong( + a, b +): ... + + +# unskippable type hint (??) +def foo(a) -> list[aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa]: # type: ignore + pass + + +def foo(a) -> list[ + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +]: # abpedeifnore + pass + +def foo(a, b: list[Bad],): ... # type: ignore + +# don't lose any comments (no magic) +def foo( # 1 + a, # 2 + b) -> list[ # 3 + a, # 4 + b]: # 5 + ... # 6 + + +# don't lose any comments (param list magic) +def foo( # 1 + a, # 2 + b,) -> list[ # 3 + a, # 4 + b]: # 5 + ... # 6 + + +# don't lose any comments (return type magic) +def foo( # 1 + a, # 2 + b) -> list[ # 3 + a, # 4 + b,]: # 5 + ... # 6 + + +# don't lose any comments (both magic) +def foo( # 1 + a, # 2 + b,) -> list[ # 3 + a, # 4 + b,]: # 5 + ... # 6 + +# real life example +def SimplePyFn( + context: hl.GeneratorContext, + buffer_input: Buffer[UInt8, 2], + func_input: Buffer[Int32, 2], + float_arg: Scalar[Float32], + offset: int = 0, +) -> tuple[ + Buffer[UInt8, 2], + Buffer[UInt8, 2], +]: ... +# output +# normal, short, function definition +def foo(a, b) -> tuple[int, float]: ... + + +# normal, short, function definition w/o return type +def foo(a, b): ... + + +# no splitting +def foo(a: A, b: B) -> list[p, q]: + pass + + +# magic trailing comma in param list +def foo( + a, + b, +): ... + + +# magic trailing comma in nested params in param list +def foo( + a, + b: tuple[ + int, + float, + ], +): ... + + +# magic trailing comma in return type, no params +def a() -> tuple[ + a, + b, +]: ... + + +# magic trailing comma in return type, params +def foo(a: A, b: B) -> list[ + p, + q, +]: + pass + + +# magic trailing comma in param list and in return type +def foo( + a: a, + b: b, +) -> list[ + a, + a, +]: + pass + + +# long function definition, param list is longer +def aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa( + bbbbbbbbbbbbbbbbbb, +) -> cccccccccccccccccccccccccccccc: ... + + +# long function definition, return type is longer +# this should maybe split on rhs? +def aaaaaaaaaaaaaaaaa( + bbbbbbbbbbbbbbbbbb, +) -> list[Ccccccccccccccccccccccccccccccccccccccccccccccccccc, Dddddd]: ... + + +# long return type, no param list +def foo() -> list[ + Loooooooooooooooooooooooooooooooooooong, + Loooooooooooooooooooong, + Looooooooooooong, +]: ... + + +# long function name, no param list, no return value +def thiiiiiiiiiiiiiiiiiis_iiiiiiiiiiiiiiiiiiiiiiiiiiiiiis_veeeeeeeeeeeeeeeeeeeeeeery_looooooong(): + pass + + +# long function name, no param list +def thiiiiiiiiiiiiiiiiiis_iiiiiiiiiiiiiiiiiiiiiiiiiiiiiis_veeeeeeeeeeeeeeeeeeeeeeery_looooooong() -> ( + list[int, float] +): ... + + +# long function name, no return value +def thiiiiiiiiiiiiiiiiiis_iiiiiiiiiiiiiiiiiiiiiiiiiiiiiis_veeeeeeeeeeeeeeeeeeeeeeery_looooooong( + a, b +): ... + + +# unskippable type hint (??) +def foo(a) -> list[aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa]: # type: ignore + pass + + +def foo( + a, +) -> list[ + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +]: # abpedeifnore + pass + + +def foo( + a, + b: list[Bad], +): ... # type: ignore + + +# don't lose any comments (no magic) +def foo(a, b) -> list[a, b]: # 1 # 2 # 3 # 4 # 5 + ... # 6 + + +# don't lose any comments (param list magic) +def foo( # 1 + a, # 2 + b, +) -> list[a, b]: # 3 # 4 # 5 + ... # 6 + + +# don't lose any comments (return type magic) +def foo(a, b) -> list[ # 1 # 2 # 3 + a, # 4 + b, +]: # 5 + ... # 6 + + +# don't lose any comments (both magic) +def foo( # 1 + a, # 2 + b, +) -> list[ # 3 + a, # 4 + b, +]: # 5 + ... # 6 + + +# real life example +def SimplePyFn( + context: hl.GeneratorContext, + buffer_input: Buffer[UInt8, 2], + func_input: Buffer[Int32, 2], + float_arg: Scalar[Float32], + offset: int = 0, +) -> tuple[ + Buffer[UInt8, 2], + Buffer[UInt8, 2], +]: ... diff --git a/tests/data/simple_cases/return_annotation_brackets.py b/tests/data/simple_cases/return_annotation_brackets.py index 265c30220d8..8509ecdb92c 100644 --- a/tests/data/simple_cases/return_annotation_brackets.py +++ b/tests/data/simple_cases/return_annotation_brackets.py @@ -87,6 +87,11 @@ def foo() -> tuple[loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo def foo() -> tuple[int, int, int,]: return 2 +# Magic trailing comma example, with params +# this is broken - the trailing comma is transferred to the param list. Fixed in preview +def foo(a,b) -> tuple[int, int, int,]: + return 2 + # output # Control def double(a: int) -> int: @@ -208,3 +213,11 @@ def foo() -> ( ] ): return 2 + + +# Magic trailing comma example, with params +# this is broken - the trailing comma is transferred to the param list. Fixed in preview +def foo( + a, b +) -> tuple[int, int, int,]: + return 2 From 6c88e8e46e19aa9f1986ab9d1f0ee4cf95f49956 Mon Sep 17 00:00:00 2001 From: Jake Anto <64896514+jake-anto@users.noreply.github.com> Date: Fri, 6 Oct 2023 06:44:59 +0530 Subject: [PATCH 130/279] Update link to VS Code formatting instructions (#3921) Update link --- docs/integrations/editors.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/editors.md b/docs/integrations/editors.md index 83904144c48..7d056160fcb 100644 --- a/docs/integrations/editors.md +++ b/docs/integrations/editors.md @@ -391,7 +391,7 @@ close and reopen your File, _Black_ will be done with its job. - Use the [Python extension](https://marketplace.visualstudio.com/items?itemName=ms-python.python) - ([instructions](https://code.visualstudio.com/docs/python/editing#_formatting)). + ([instructions](https://code.visualstudio.com/docs/python/formatting)). - Alternatively the pre-release [Black Formatter](https://marketplace.visualstudio.com/items?itemName=ms-python.black-formatter) From 27c05e1e24e02c62d0c2de2a1ab0673b2c2ca653 Mon Sep 17 00:00:00 2001 From: John Litborn <11260241+jakkdl@users.noreply.github.com> Date: Fri, 6 Oct 2023 03:15:35 +0200 Subject: [PATCH 131/279] exclude tests/data/.* from mypy (#3917) --- mypy.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mypy.ini b/mypy.ini index 95ec22d65be..ad916185bce 100644 --- a/mypy.ini +++ b/mypy.ini @@ -39,3 +39,8 @@ ignore_missing_imports = True [mypy-_black_version.*] ignore_missing_imports = True + +# CI only checks src/, but in case users are running LSP or similar we explicitly ignore +# errors in test data files. +[mypy-tests.data.*] +ignore_errors = True From 3a2d76c7bcf39e852f3b379b76537d7847ed4225 Mon Sep 17 00:00:00 2001 From: David Lev <42866208+david-lev@users.noreply.github.com> Date: Fri, 6 Oct 2023 04:21:56 +0300 Subject: [PATCH 132/279] =?UTF-8?q?Remove=20`$`,=20`>>>`=20and=20other=20p?= =?UTF-8?q?rompt=20prefixes=20when=20code=20copied=20from=20the=E2=80=A6?= =?UTF-8?q?=20(#3884)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adding configurations for sphinx-copybutton in conf.py https://sphinx-copybutton.readthedocs.io/en/latest/use.html#using-regexp-prompt-identifiers --- docs/conf.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index f7cf1b42842..6b645435325 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -210,6 +210,13 @@ def make_pypi_svg(version: str) -> None: autodoc_member_order = "bysource" +# -- sphinx-copybutton configuration ---------------------------------------- +copybutton_prompt_text = ( + r">>> |\.\.\. |> |\$ |\# | In \[\d*\]: | {2,5}\.\.\.: | {5,8}: " +) +copybutton_prompt_is_regexp = True +copybutton_remove_prompts = True + # -- Options for intersphinx extension --------------------------------------- # Example configuration for intersphinx: refer to the Python standard library. From 3457ec48af0f4c481cdf35f280998bde3f484efd Mon Sep 17 00:00:00 2001 From: Cristiano Salerno <119511125+csalerno-asml@users.noreply.github.com> Date: Fri, 6 Oct 2023 19:41:36 +0200 Subject: [PATCH 133/279] Update output display to job summary (#3914) * Update output display to job summary * fix: handled exit-code of script * added changelog message --- CHANGES.md | 2 ++ action.yml | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 888824ee055..062a195717d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -48,6 +48,8 @@ +- The action output displayed in the job summary is now wrapped in Markdown (#3914) + ### Documentation +- Black no longer attempts to provide special errors for attempting to format Python 2 + code (#3933) + ### _Blackd_ diff --git a/src/black/parsing.py b/src/black/parsing.py index e98e019cac6..03e767a333b 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -3,7 +3,7 @@ """ import ast import sys -from typing import Final, Iterable, Iterator, List, Set, Tuple +from typing import Iterable, Iterator, List, Set, Tuple from black.mode import VERSION_TO_FEATURES, Feature, TargetVersion, supports_feature from black.nodes import syms @@ -14,8 +14,6 @@ from blib2to3.pgen2.tokenize import TokenError from blib2to3.pytree import Leaf, Node -PY2_HINT: Final = "Python 2 support was removed in version 22.0." - class InvalidInput(ValueError): """Raised when input source code fails all parse attempts.""" @@ -26,9 +24,9 @@ def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]: # No target_version specified, so try all grammars. return [ # Python 3.7-3.9 - pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords, + pygram.python_grammar_async_keywords, # Python 3.0-3.6 - pygram.python_grammar_no_print_statement_no_exec_statement, + pygram.python_grammar, # Python 3.10+ pygram.python_grammar_soft_keywords, ] @@ -39,12 +37,10 @@ def get_grammars(target_versions: Set[TargetVersion]) -> List[Grammar]: target_versions, Feature.ASYNC_IDENTIFIERS ) and not supports_feature(target_versions, Feature.PATTERN_MATCHING): # Python 3.7-3.9 - grammars.append( - pygram.python_grammar_no_print_statement_no_exec_statement_async_keywords - ) + grammars.append(pygram.python_grammar_async_keywords) if not supports_feature(target_versions, Feature.ASYNC_KEYWORDS): # Python 3.0-3.6 - grammars.append(pygram.python_grammar_no_print_statement_no_exec_statement) + grammars.append(pygram.python_grammar) if any(Feature.PATTERN_MATCHING in VERSION_TO_FEATURES[v] for v in target_versions): # Python 3.10+ grammars.append(pygram.python_grammar_soft_keywords) @@ -89,14 +85,6 @@ def lib2to3_parse(src_txt: str, target_versions: Iterable[TargetVersion] = ()) - # Choose the latest version when raising the actual parsing error. assert len(errors) >= 1 exc = errors[max(errors)] - - if matches_grammar(src_txt, pygram.python_grammar) or matches_grammar( - src_txt, pygram.python_grammar_no_print_statement - ): - original_msg = exc.args[0] - msg = f"{original_msg}\n{PY2_HINT}" - raise InvalidInput(msg) from None - raise exc from None if isinstance(result, Leaf): diff --git a/src/blib2to3/Grammar.txt b/src/blib2to3/Grammar.txt index e48e66363fb..be91df75740 100644 --- a/src/blib2to3/Grammar.txt +++ b/src/blib2to3/Grammar.txt @@ -80,8 +80,8 @@ vfplist: vfpdef (',' vfpdef)* [','] stmt: simple_stmt | compound_stmt simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE -small_stmt: (type_stmt | expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | - import_stmt | global_stmt | exec_stmt | assert_stmt) +small_stmt: (type_stmt | expr_stmt | del_stmt | pass_stmt | flow_stmt | + import_stmt | global_stmt | assert_stmt) expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) | ('=' (yield_expr|testlist_star_expr))*) annassign: ':' test ['=' (yield_expr|testlist_star_expr)] @@ -89,8 +89,6 @@ testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//=') # For normal and annotated assignments, additional restrictions enforced by the interpreter -print_stmt: 'print' ( [ test (',' test)* [','] ] | - '>>' test [ (',' test)+ [','] ] ) del_stmt: 'del' exprlist pass_stmt: 'pass' flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt @@ -109,7 +107,6 @@ import_as_names: import_as_name (',' import_as_name)* [','] dotted_as_names: dotted_as_name (',' dotted_as_name)* dotted_name: NAME ('.' NAME)* global_stmt: ('global' | 'nonlocal') NAME (',' NAME)* -exec_stmt: 'exec' expr ['in' test [',' test]] assert_stmt: 'assert' test [',' test] type_stmt: "type" NAME [typeparams] '=' expr diff --git a/src/blib2to3/pygram.py b/src/blib2to3/pygram.py index c30c630e816..2b43b4c112b 100644 --- a/src/blib2to3/pygram.py +++ b/src/blib2to3/pygram.py @@ -63,7 +63,6 @@ class _python_symbols(Symbols): encoding_decl: int eval_input: int except_clause: int - exec_stmt: int expr: int expr_stmt: int exprlist: int @@ -97,7 +96,6 @@ class _python_symbols(Symbols): pattern: int patterns: int power: int - print_stmt: int raise_stmt: int return_stmt: int shift_expr: int @@ -153,22 +151,16 @@ class _pattern_symbols(Symbols): python_grammar: Grammar -python_grammar_no_print_statement: Grammar -python_grammar_no_print_statement_no_exec_statement: Grammar -python_grammar_no_print_statement_no_exec_statement_async_keywords: Grammar -python_grammar_no_exec_statement: Grammar -pattern_grammar: Grammar +python_grammar_async_keywords: Grammar python_grammar_soft_keywords: Grammar - +pattern_grammar: Grammar python_symbols: _python_symbols pattern_symbols: _pattern_symbols def initialize(cache_dir: Union[str, "os.PathLike[str]", None] = None) -> None: global python_grammar - global python_grammar_no_print_statement - global python_grammar_no_print_statement_no_exec_statement - global python_grammar_no_print_statement_no_exec_statement_async_keywords + global python_grammar_async_keywords global python_grammar_soft_keywords global python_symbols global pattern_grammar @@ -180,38 +172,25 @@ def initialize(cache_dir: Union[str, "os.PathLike[str]", None] = None) -> None: os.path.dirname(__file__), "PatternGrammar.txt" ) - # Python 2 python_grammar = driver.load_packaged_grammar("blib2to3", _GRAMMAR_FILE, cache_dir) - python_grammar.version = (2, 0) + assert "print" not in python_grammar.keywords + assert "exec" not in python_grammar.keywords soft_keywords = python_grammar.soft_keywords.copy() python_grammar.soft_keywords.clear() python_symbols = _python_symbols(python_grammar) - # Python 2 + from __future__ import print_function - python_grammar_no_print_statement = python_grammar.copy() - del python_grammar_no_print_statement.keywords["print"] - # Python 3.0-3.6 - python_grammar_no_print_statement_no_exec_statement = python_grammar.copy() - del python_grammar_no_print_statement_no_exec_statement.keywords["print"] - del python_grammar_no_print_statement_no_exec_statement.keywords["exec"] - python_grammar_no_print_statement_no_exec_statement.version = (3, 0) + python_grammar.version = (3, 0) # Python 3.7+ - python_grammar_no_print_statement_no_exec_statement_async_keywords = ( - python_grammar_no_print_statement_no_exec_statement.copy() - ) - python_grammar_no_print_statement_no_exec_statement_async_keywords.async_keywords = ( - True - ) - python_grammar_no_print_statement_no_exec_statement_async_keywords.version = (3, 7) + python_grammar_async_keywords = python_grammar.copy() + python_grammar_async_keywords.async_keywords = True + python_grammar_async_keywords.version = (3, 7) # Python 3.10+ - python_grammar_soft_keywords = ( - python_grammar_no_print_statement_no_exec_statement_async_keywords.copy() - ) + python_grammar_soft_keywords = python_grammar_async_keywords.copy() python_grammar_soft_keywords.soft_keywords = soft_keywords python_grammar_soft_keywords.version = (3, 10) diff --git a/tests/test_format.py b/tests/test_format.py index f3db423b637..ff358d59c94 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -155,12 +155,6 @@ def test_fast_cases(filename: str) -> None: assert_format(source, expected, fast=True) -def test_python_2_hint() -> None: - with pytest.raises(black.parsing.InvalidInput) as exc_info: - assert_format("print 'daylily'", "print 'daylily'") - exc_info.match(black.parsing.PY2_HINT) - - @pytest.mark.filterwarnings("ignore:invalid escape sequence.*:DeprecationWarning") def test_docstring_no_string_normalization() -> None: """Like test_docstring but with string normalization off.""" From a69bda3b9bde208d5489eb2e37fc982b6bc1d8df Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 9 Oct 2023 18:43:47 -0700 Subject: [PATCH 137/279] Use inline flags for test cases (#3931) Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- docs/contributing/the_basics.md | 21 +- .../attribute_access_on_number_literals.py | 0 .../beginning_backslash.py | 0 .../{simple_cases => cases}/bracketmatch.py | 0 .../class_blank_parentheses.py | 0 .../class_methods_new_line.py | 0 .../{simple_cases => cases}/collections.py | 0 .../comment_after_escaped_newline.py | 0 .../data/{simple_cases => cases}/comments.py | 0 .../data/{simple_cases => cases}/comments2.py | 0 .../data/{simple_cases => cases}/comments3.py | 0 .../data/{simple_cases => cases}/comments4.py | 0 .../data/{simple_cases => cases}/comments5.py | 0 .../data/{simple_cases => cases}/comments6.py | 0 .../data/{simple_cases => cases}/comments8.py | 0 .../data/{simple_cases => cases}/comments9.py | 0 .../comments_non_breaking_space.py | 0 .../{simple_cases => cases}/composition.py | 0 .../composition_no_trailing_comma.py | 0 .../data/{simple_cases => cases}/docstring.py | 0 ...ocstring_no_extra_empty_line_before_eof.py | 0 .../docstring_no_string_normalization.py | 1 + .../docstring_preview.py | 0 ...cstring_preview_no_string_normalization.py | 1 + .../{simple_cases => cases}/empty_lines.py | 0 .../{simple_cases => cases}/expression.diff | 0 .../{simple_cases => cases}/expression.py | 0 .../data/{simple_cases => cases}/fmtonoff.py | 0 .../data/{simple_cases => cases}/fmtonoff2.py | 0 .../data/{simple_cases => cases}/fmtonoff3.py | 0 .../data/{simple_cases => cases}/fmtonoff4.py | 0 .../data/{simple_cases => cases}/fmtonoff5.py | 0 .../fmtpass_imports.py | 0 tests/data/{simple_cases => cases}/fmtskip.py | 0 .../data/{simple_cases => cases}/fmtskip2.py | 0 .../data/{simple_cases => cases}/fmtskip3.py | 0 .../data/{simple_cases => cases}/fmtskip4.py | 0 .../data/{simple_cases => cases}/fmtskip5.py | 0 .../data/{simple_cases => cases}/fmtskip6.py | 0 .../data/{simple_cases => cases}/fmtskip7.py | 0 .../data/{simple_cases => cases}/fmtskip8.py | 0 tests/data/{simple_cases => cases}/fstring.py | 0 .../funcdef_return_type_trailing_comma.py | 1 + .../data/{simple_cases => cases}/function.py | 0 .../data/{simple_cases => cases}/function2.py | 0 .../function_trailing_comma.py | 0 .../{simple_cases => cases}/ignore_pyi.py | 1 + .../{simple_cases => cases}/import_spacing.py | 0 .../{miscellaneous => cases}/linelength6.py | 1 + .../long_strings_flag_disabled.py | 0 ...ine_consecutive_open_parentheses_ignore.py | 0 .../nested_stub.pyi => cases/nested_stub.py} | 1 + .../data/{py_36 => cases}/numeric_literals.py | 5 - .../numeric_literals_skip_underscores.py | 4 - .../one_element_subscript.py | 0 .../parenthesized_context_managers.py | 1 + .../pattern_matching_complex.py | 1 + .../pattern_matching_extras.py | 1 + .../pattern_matching_generic.py | 1 + .../pattern_matching_simple.py | 1 + .../pattern_matching_style.py | 1 + .../pep604_union_types_line_breaks.py | 1 + tests/data/{py_38 => cases}/pep_570.py | 1 + tests/data/{py_38 => cases}/pep_572.py | 1 + .../pep_572_do_not_remove_parens.py | 1 + tests/data/{py_310 => cases}/pep_572_py310.py | 1 + tests/data/{py_39 => cases}/pep_572_py39.py | 1 + .../{py_38 => cases}/pep_572_remove_parens.py | 1 + tests/data/{simple_cases => cases}/pep_604.py | 0 tests/data/{py_311 => cases}/pep_646.py | 1 + tests/data/{py_311 => cases}/pep_654.py | 1 + tests/data/{py_311 => cases}/pep_654_style.py | 1 + .../power_op_newline.py | 1 + .../power_op_spacing.py | 0 .../prefer_rhs_split_reformatted.py | 0 .../preview_async_stmts.py} | 1 + .../cantfit.py => cases/preview_cantfit.py} | 1 + .../preview_comments7.py} | 1 + .../preview_context_managers_38.py} | 1 + .../preview_context_managers_39.py} | 1 + ...review_context_managers_autodetect_310.py} | 1 + ...review_context_managers_autodetect_311.py} | 1 + ...preview_context_managers_autodetect_38.py} | 1 + ...preview_context_managers_autodetect_39.py} | 1 + .../preview_dummy_implementations.py} | 1 + .../preview_format_unicode_escape_seq.py} | 1 + .../preview_long_dict_values.py} | 1 + .../preview_long_strings.py} | 1 + ...preview_long_strings__east_asian_width.py} | 1 + .../preview_long_strings__edge_case.py} | 1 + .../preview_long_strings__regression.py} | 1 + ...preview_long_strings__type_annotations.py} | 1 + .../preview_multiline_strings.py} | 1 + ...preview_no_blank_line_before_docstring.py} | 1 + .../pep_572.py => cases/preview_pep_572.py} | 1 + .../preview_percent_precedence.py} | 1 + .../preview_prefer_rhs_split.py} | 1 + ...view_return_annotation_brackets_string.py} | 1 + .../preview_trailing_comma.py} | 1 + .../pep_572.py => cases/py310_pep572.py} | 1 + tests/data/{py_37 => cases}/python37.py | 5 +- tests/data/{py_38 => cases}/python38.py | 5 +- tests/data/{py_39 => cases}/python39.py | 6 +- .../remove_await_parens.py | 0 .../remove_except_parens.py | 0 .../remove_for_brackets.py | 0 .../remove_newline_after_code_block_open.py | 0 .../remove_newline_after_match.py | 1 + .../{simple_cases => cases}/remove_parens.py | 0 .../{py_39 => cases}/remove_with_brackets.py | 1 + .../return_annotation_brackets.py | 0 .../skip_magic_trailing_comma.py | 1 + tests/data/{simple_cases => cases}/slices.py | 0 .../{py_310 => cases}/starred_for_target.py | 1 + .../string_prefixes.py | 0 .../{miscellaneous/stub.pyi => cases/stub.py} | 1 + tests/data/{simple_cases => cases}/torture.py | 0 .../trailing_comma_optional_parens1.py | 0 .../trailing_comma_optional_parens2.py | 0 .../trailing_comma_optional_parens3.py | 0 .../trailing_commas_in_leading_parts.py | 0 .../tricky_unicode_symbols.py | 0 .../{simple_cases => cases}/tupleassign.py | 0 tests/data/{py_312 => cases}/type_aliases.py | 1 + .../type_comment_syntax_error.py | 0 tests/data/{py_312 => cases}/type_params.py | 1 + .../{simple_cases => cases}/whitespace.py | 0 tests/data/miscellaneous/force_pyi.py | 1 + tests/test_black.py | 42 ++-- tests/test_blackd.py | 2 +- tests/test_format.py | 194 ++---------------- tests/util.py | 86 +++++++- 132 files changed, 206 insertions(+), 220 deletions(-) rename tests/data/{simple_cases => cases}/attribute_access_on_number_literals.py (100%) rename tests/data/{simple_cases => cases}/beginning_backslash.py (100%) rename tests/data/{simple_cases => cases}/bracketmatch.py (100%) rename tests/data/{simple_cases => cases}/class_blank_parentheses.py (100%) rename tests/data/{simple_cases => cases}/class_methods_new_line.py (100%) rename tests/data/{simple_cases => cases}/collections.py (100%) rename tests/data/{simple_cases => cases}/comment_after_escaped_newline.py (100%) rename tests/data/{simple_cases => cases}/comments.py (100%) rename tests/data/{simple_cases => cases}/comments2.py (100%) rename tests/data/{simple_cases => cases}/comments3.py (100%) rename tests/data/{simple_cases => cases}/comments4.py (100%) rename tests/data/{simple_cases => cases}/comments5.py (100%) rename tests/data/{simple_cases => cases}/comments6.py (100%) rename tests/data/{simple_cases => cases}/comments8.py (100%) rename tests/data/{simple_cases => cases}/comments9.py (100%) rename tests/data/{simple_cases => cases}/comments_non_breaking_space.py (100%) rename tests/data/{simple_cases => cases}/composition.py (100%) rename tests/data/{simple_cases => cases}/composition_no_trailing_comma.py (100%) rename tests/data/{simple_cases => cases}/docstring.py (100%) rename tests/data/{simple_cases => cases}/docstring_no_extra_empty_line_before_eof.py (100%) rename tests/data/{miscellaneous => cases}/docstring_no_string_normalization.py (98%) rename tests/data/{simple_cases => cases}/docstring_preview.py (100%) rename tests/data/{miscellaneous => cases}/docstring_preview_no_string_normalization.py (88%) rename tests/data/{simple_cases => cases}/empty_lines.py (100%) rename tests/data/{simple_cases => cases}/expression.diff (100%) rename tests/data/{simple_cases => cases}/expression.py (100%) rename tests/data/{simple_cases => cases}/fmtonoff.py (100%) rename tests/data/{simple_cases => cases}/fmtonoff2.py (100%) rename tests/data/{simple_cases => cases}/fmtonoff3.py (100%) rename tests/data/{simple_cases => cases}/fmtonoff4.py (100%) rename tests/data/{simple_cases => cases}/fmtonoff5.py (100%) rename tests/data/{simple_cases => cases}/fmtpass_imports.py (100%) rename tests/data/{simple_cases => cases}/fmtskip.py (100%) rename tests/data/{simple_cases => cases}/fmtskip2.py (100%) rename tests/data/{simple_cases => cases}/fmtskip3.py (100%) rename tests/data/{simple_cases => cases}/fmtskip4.py (100%) rename tests/data/{simple_cases => cases}/fmtskip5.py (100%) rename tests/data/{simple_cases => cases}/fmtskip6.py (100%) rename tests/data/{simple_cases => cases}/fmtskip7.py (100%) rename tests/data/{simple_cases => cases}/fmtskip8.py (100%) rename tests/data/{simple_cases => cases}/fstring.py (100%) rename tests/data/{preview_py_310 => cases}/funcdef_return_type_trailing_comma.py (99%) rename tests/data/{simple_cases => cases}/function.py (100%) rename tests/data/{simple_cases => cases}/function2.py (100%) rename tests/data/{simple_cases => cases}/function_trailing_comma.py (100%) rename tests/data/{simple_cases => cases}/ignore_pyi.py (97%) rename tests/data/{simple_cases => cases}/import_spacing.py (100%) rename tests/data/{miscellaneous => cases}/linelength6.py (80%) rename tests/data/{miscellaneous => cases}/long_strings_flag_disabled.py (100%) rename tests/data/{simple_cases => cases}/multiline_consecutive_open_parentheses_ignore.py (100%) rename tests/data/{miscellaneous/nested_stub.pyi => cases/nested_stub.py} (97%) rename tests/data/{py_36 => cases}/numeric_literals.py (91%) rename tests/data/{py_36 => cases}/numeric_literals_skip_underscores.py (80%) rename tests/data/{simple_cases => cases}/one_element_subscript.py (100%) rename tests/data/{py_310 => cases}/parenthesized_context_managers.py (95%) rename tests/data/{py_310 => cases}/pattern_matching_complex.py (98%) rename tests/data/{py_310 => cases}/pattern_matching_extras.py (98%) rename tests/data/{py_310 => cases}/pattern_matching_generic.py (98%) rename tests/data/{py_310 => cases}/pattern_matching_simple.py (98%) rename tests/data/{py_310 => cases}/pattern_matching_style.py (97%) rename tests/data/{preview_py_310 => cases}/pep604_union_types_line_breaks.py (99%) rename tests/data/{py_38 => cases}/pep_570.py (95%) rename tests/data/{py_38 => cases}/pep_572.py (96%) rename tests/data/{fast => cases}/pep_572_do_not_remove_parens.py (96%) rename tests/data/{py_310 => cases}/pep_572_py310.py (93%) rename tests/data/{py_39 => cases}/pep_572_py39.py (89%) rename tests/data/{py_38 => cases}/pep_572_remove_parens.py (98%) rename tests/data/{simple_cases => cases}/pep_604.py (100%) rename tests/data/{py_311 => cases}/pep_646.py (98%) rename tests/data/{py_311 => cases}/pep_654.py (96%) rename tests/data/{py_311 => cases}/pep_654_style.py (98%) rename tests/data/{miscellaneous => cases}/power_op_newline.py (73%) rename tests/data/{simple_cases => cases}/power_op_spacing.py (100%) rename tests/data/{simple_cases => cases}/prefer_rhs_split_reformatted.py (100%) rename tests/data/{preview/async_stmts.py => cases/preview_async_stmts.py} (93%) rename tests/data/{preview/cantfit.py => cases/preview_cantfit.py} (99%) rename tests/data/{preview/comments7.py => cases/preview_comments7.py} (99%) rename tests/data/{preview_context_managers/targeting_py38.py => cases/preview_context_managers_38.py} (96%) rename tests/data/{preview_context_managers/targeting_py39.py => cases/preview_context_managers_39.py} (98%) rename tests/data/{preview_context_managers/auto_detect/features_3_10.py => cases/preview_context_managers_autodetect_310.py} (93%) rename tests/data/{preview_context_managers/auto_detect/features_3_11.py => cases/preview_context_managers_autodetect_311.py} (92%) rename tests/data/{preview_context_managers/auto_detect/features_3_8.py => cases/preview_context_managers_autodetect_38.py} (98%) rename tests/data/{preview_context_managers/auto_detect/features_3_9.py => cases/preview_context_managers_autodetect_39.py} (93%) rename tests/data/{preview/dummy_implementations.py => cases/preview_dummy_implementations.py} (98%) rename tests/data/{preview/format_unicode_escape_seq.py => cases/preview_format_unicode_escape_seq.py} (96%) rename tests/data/{preview/long_dict_values.py => cases/preview_long_dict_values.py} (99%) rename tests/data/{preview/long_strings.py => cases/preview_long_strings.py} (99%) rename tests/data/{preview/long_strings__east_asian_width.py => cases/preview_long_strings__east_asian_width.py} (96%) rename tests/data/{preview/long_strings__edge_case.py => cases/preview_long_strings__edge_case.py} (99%) rename tests/data/{preview/long_strings__regression.py => cases/preview_long_strings__regression.py} (99%) rename tests/data/{preview/long_strings__type_annotations.py => cases/preview_long_strings__type_annotations.py} (98%) rename tests/data/{preview/multiline_strings.py => cases/preview_multiline_strings.py} (99%) rename tests/data/{preview/no_blank_line_before_docstring.py => cases/preview_no_blank_line_before_docstring.py} (97%) rename tests/data/{preview/pep_572.py => cases/preview_pep_572.py} (75%) rename tests/data/{preview/percent_precedence.py => cases/preview_percent_precedence.py} (96%) rename tests/data/{preview/prefer_rhs_split.py => cases/preview_prefer_rhs_split.py} (99%) rename tests/data/{preview/return_annotation_brackets_string.py => cases/preview_return_annotation_brackets_string.py} (97%) rename tests/data/{preview/trailing_comma.py => cases/preview_trailing_comma.py} (97%) rename tests/data/{preview_py_310/pep_572.py => cases/py310_pep572.py} (77%) rename tests/data/{py_37 => cases}/python37.py (95%) rename tests/data/{py_38 => cases}/python38.py (93%) rename tests/data/{py_39 => cases}/python39.py (92%) rename tests/data/{simple_cases => cases}/remove_await_parens.py (100%) rename tests/data/{simple_cases => cases}/remove_except_parens.py (100%) rename tests/data/{simple_cases => cases}/remove_for_brackets.py (100%) rename tests/data/{simple_cases => cases}/remove_newline_after_code_block_open.py (100%) rename tests/data/{py_310 => cases}/remove_newline_after_match.py (94%) rename tests/data/{simple_cases => cases}/remove_parens.py (100%) rename tests/data/{py_39 => cases}/remove_with_brackets.py (98%) rename tests/data/{simple_cases => cases}/return_annotation_brackets.py (100%) rename tests/data/{simple_cases => cases}/skip_magic_trailing_comma.py (97%) rename tests/data/{simple_cases => cases}/slices.py (100%) rename tests/data/{py_310 => cases}/starred_for_target.py (92%) rename tests/data/{simple_cases => cases}/string_prefixes.py (100%) rename tests/data/{miscellaneous/stub.pyi => cases/stub.py} (99%) rename tests/data/{simple_cases => cases}/torture.py (100%) rename tests/data/{simple_cases => cases}/trailing_comma_optional_parens1.py (100%) rename tests/data/{simple_cases => cases}/trailing_comma_optional_parens2.py (100%) rename tests/data/{simple_cases => cases}/trailing_comma_optional_parens3.py (100%) rename tests/data/{simple_cases => cases}/trailing_commas_in_leading_parts.py (100%) rename tests/data/{simple_cases => cases}/tricky_unicode_symbols.py (100%) rename tests/data/{simple_cases => cases}/tupleassign.py (100%) rename tests/data/{py_312 => cases}/type_aliases.py (81%) rename tests/data/{type_comments => cases}/type_comment_syntax_error.py (100%) rename tests/data/{py_312 => cases}/type_params.py (97%) rename tests/data/{simple_cases => cases}/whitespace.py (100%) diff --git a/docs/contributing/the_basics.md b/docs/contributing/the_basics.md index 864894b491f..bc1680eecfd 100644 --- a/docs/contributing/the_basics.md +++ b/docs/contributing/the_basics.md @@ -58,7 +58,26 @@ Further examples of invoking the tests (.venv)$ tox -e py -- --print-tree-diff=False ``` -`Black` has two pytest command-line options affecting test files in `tests/data/` that +### Testing + +All aspects of the _Black_ style should be tested. Normally, tests should be created as +files in the `tests/data/cases` directory. These files consist of up to three parts: + +- A line that starts with `# flags: ` followed by a set of command-line options. For + example, if the line is `# flags: --preview --skip-magic-trailing-comma`, the test + case will be run with preview mode on and the magic trailing comma off. The options + accepted are mostly a subset of those of _Black_ itself, except for the + `--minimum-version=` flag, which should be used when testing a grammar feature that + works only in newer versions of Python. This flag ensures that we don't try to + validate the AST on older versions and tests that we autodetect the Python version + correctly when the feature is used. For the exact flags accepted, see the function + `get_flags_parser` in `tests/util.py`. If this line is omitted, the default options + are used. +- A block of Python code used as input for the formatter. +- The line `# output`, followed by the output of _Black_ when run on the previous block. + If this is omitted, the test asserts that _Black_ will leave the input code unchanged. + +_Black_ has two pytest command-line options affecting test files in `tests/data/` that are split into an input part, and an output part, separated by a line with`# output`. These can be passed to `pytest` through `tox`, or directly into pytest if not using `tox`. diff --git a/tests/data/simple_cases/attribute_access_on_number_literals.py b/tests/data/cases/attribute_access_on_number_literals.py similarity index 100% rename from tests/data/simple_cases/attribute_access_on_number_literals.py rename to tests/data/cases/attribute_access_on_number_literals.py diff --git a/tests/data/simple_cases/beginning_backslash.py b/tests/data/cases/beginning_backslash.py similarity index 100% rename from tests/data/simple_cases/beginning_backslash.py rename to tests/data/cases/beginning_backslash.py diff --git a/tests/data/simple_cases/bracketmatch.py b/tests/data/cases/bracketmatch.py similarity index 100% rename from tests/data/simple_cases/bracketmatch.py rename to tests/data/cases/bracketmatch.py diff --git a/tests/data/simple_cases/class_blank_parentheses.py b/tests/data/cases/class_blank_parentheses.py similarity index 100% rename from tests/data/simple_cases/class_blank_parentheses.py rename to tests/data/cases/class_blank_parentheses.py diff --git a/tests/data/simple_cases/class_methods_new_line.py b/tests/data/cases/class_methods_new_line.py similarity index 100% rename from tests/data/simple_cases/class_methods_new_line.py rename to tests/data/cases/class_methods_new_line.py diff --git a/tests/data/simple_cases/collections.py b/tests/data/cases/collections.py similarity index 100% rename from tests/data/simple_cases/collections.py rename to tests/data/cases/collections.py diff --git a/tests/data/simple_cases/comment_after_escaped_newline.py b/tests/data/cases/comment_after_escaped_newline.py similarity index 100% rename from tests/data/simple_cases/comment_after_escaped_newline.py rename to tests/data/cases/comment_after_escaped_newline.py diff --git a/tests/data/simple_cases/comments.py b/tests/data/cases/comments.py similarity index 100% rename from tests/data/simple_cases/comments.py rename to tests/data/cases/comments.py diff --git a/tests/data/simple_cases/comments2.py b/tests/data/cases/comments2.py similarity index 100% rename from tests/data/simple_cases/comments2.py rename to tests/data/cases/comments2.py diff --git a/tests/data/simple_cases/comments3.py b/tests/data/cases/comments3.py similarity index 100% rename from tests/data/simple_cases/comments3.py rename to tests/data/cases/comments3.py diff --git a/tests/data/simple_cases/comments4.py b/tests/data/cases/comments4.py similarity index 100% rename from tests/data/simple_cases/comments4.py rename to tests/data/cases/comments4.py diff --git a/tests/data/simple_cases/comments5.py b/tests/data/cases/comments5.py similarity index 100% rename from tests/data/simple_cases/comments5.py rename to tests/data/cases/comments5.py diff --git a/tests/data/simple_cases/comments6.py b/tests/data/cases/comments6.py similarity index 100% rename from tests/data/simple_cases/comments6.py rename to tests/data/cases/comments6.py diff --git a/tests/data/simple_cases/comments8.py b/tests/data/cases/comments8.py similarity index 100% rename from tests/data/simple_cases/comments8.py rename to tests/data/cases/comments8.py diff --git a/tests/data/simple_cases/comments9.py b/tests/data/cases/comments9.py similarity index 100% rename from tests/data/simple_cases/comments9.py rename to tests/data/cases/comments9.py diff --git a/tests/data/simple_cases/comments_non_breaking_space.py b/tests/data/cases/comments_non_breaking_space.py similarity index 100% rename from tests/data/simple_cases/comments_non_breaking_space.py rename to tests/data/cases/comments_non_breaking_space.py diff --git a/tests/data/simple_cases/composition.py b/tests/data/cases/composition.py similarity index 100% rename from tests/data/simple_cases/composition.py rename to tests/data/cases/composition.py diff --git a/tests/data/simple_cases/composition_no_trailing_comma.py b/tests/data/cases/composition_no_trailing_comma.py similarity index 100% rename from tests/data/simple_cases/composition_no_trailing_comma.py rename to tests/data/cases/composition_no_trailing_comma.py diff --git a/tests/data/simple_cases/docstring.py b/tests/data/cases/docstring.py similarity index 100% rename from tests/data/simple_cases/docstring.py rename to tests/data/cases/docstring.py diff --git a/tests/data/simple_cases/docstring_no_extra_empty_line_before_eof.py b/tests/data/cases/docstring_no_extra_empty_line_before_eof.py similarity index 100% rename from tests/data/simple_cases/docstring_no_extra_empty_line_before_eof.py rename to tests/data/cases/docstring_no_extra_empty_line_before_eof.py diff --git a/tests/data/miscellaneous/docstring_no_string_normalization.py b/tests/data/cases/docstring_no_string_normalization.py similarity index 98% rename from tests/data/miscellaneous/docstring_no_string_normalization.py rename to tests/data/cases/docstring_no_string_normalization.py index a90b578f09a..4ec6b8a0153 100644 --- a/tests/data/miscellaneous/docstring_no_string_normalization.py +++ b/tests/data/cases/docstring_no_string_normalization.py @@ -1,3 +1,4 @@ +# flags: --skip-string-normalization class ALonelyClass: ''' A multiline class docstring. diff --git a/tests/data/simple_cases/docstring_preview.py b/tests/data/cases/docstring_preview.py similarity index 100% rename from tests/data/simple_cases/docstring_preview.py rename to tests/data/cases/docstring_preview.py diff --git a/tests/data/miscellaneous/docstring_preview_no_string_normalization.py b/tests/data/cases/docstring_preview_no_string_normalization.py similarity index 88% rename from tests/data/miscellaneous/docstring_preview_no_string_normalization.py rename to tests/data/cases/docstring_preview_no_string_normalization.py index 338cc01d33e..712c7364f51 100644 --- a/tests/data/miscellaneous/docstring_preview_no_string_normalization.py +++ b/tests/data/cases/docstring_preview_no_string_normalization.py @@ -1,3 +1,4 @@ +# flags: --preview --skip-string-normalization def do_not_touch_this_prefix(): R"""There was a bug where docstring prefixes would be normalized even with -S.""" diff --git a/tests/data/simple_cases/empty_lines.py b/tests/data/cases/empty_lines.py similarity index 100% rename from tests/data/simple_cases/empty_lines.py rename to tests/data/cases/empty_lines.py diff --git a/tests/data/simple_cases/expression.diff b/tests/data/cases/expression.diff similarity index 100% rename from tests/data/simple_cases/expression.diff rename to tests/data/cases/expression.diff diff --git a/tests/data/simple_cases/expression.py b/tests/data/cases/expression.py similarity index 100% rename from tests/data/simple_cases/expression.py rename to tests/data/cases/expression.py diff --git a/tests/data/simple_cases/fmtonoff.py b/tests/data/cases/fmtonoff.py similarity index 100% rename from tests/data/simple_cases/fmtonoff.py rename to tests/data/cases/fmtonoff.py diff --git a/tests/data/simple_cases/fmtonoff2.py b/tests/data/cases/fmtonoff2.py similarity index 100% rename from tests/data/simple_cases/fmtonoff2.py rename to tests/data/cases/fmtonoff2.py diff --git a/tests/data/simple_cases/fmtonoff3.py b/tests/data/cases/fmtonoff3.py similarity index 100% rename from tests/data/simple_cases/fmtonoff3.py rename to tests/data/cases/fmtonoff3.py diff --git a/tests/data/simple_cases/fmtonoff4.py b/tests/data/cases/fmtonoff4.py similarity index 100% rename from tests/data/simple_cases/fmtonoff4.py rename to tests/data/cases/fmtonoff4.py diff --git a/tests/data/simple_cases/fmtonoff5.py b/tests/data/cases/fmtonoff5.py similarity index 100% rename from tests/data/simple_cases/fmtonoff5.py rename to tests/data/cases/fmtonoff5.py diff --git a/tests/data/simple_cases/fmtpass_imports.py b/tests/data/cases/fmtpass_imports.py similarity index 100% rename from tests/data/simple_cases/fmtpass_imports.py rename to tests/data/cases/fmtpass_imports.py diff --git a/tests/data/simple_cases/fmtskip.py b/tests/data/cases/fmtskip.py similarity index 100% rename from tests/data/simple_cases/fmtskip.py rename to tests/data/cases/fmtskip.py diff --git a/tests/data/simple_cases/fmtskip2.py b/tests/data/cases/fmtskip2.py similarity index 100% rename from tests/data/simple_cases/fmtskip2.py rename to tests/data/cases/fmtskip2.py diff --git a/tests/data/simple_cases/fmtskip3.py b/tests/data/cases/fmtskip3.py similarity index 100% rename from tests/data/simple_cases/fmtskip3.py rename to tests/data/cases/fmtskip3.py diff --git a/tests/data/simple_cases/fmtskip4.py b/tests/data/cases/fmtskip4.py similarity index 100% rename from tests/data/simple_cases/fmtskip4.py rename to tests/data/cases/fmtskip4.py diff --git a/tests/data/simple_cases/fmtskip5.py b/tests/data/cases/fmtskip5.py similarity index 100% rename from tests/data/simple_cases/fmtskip5.py rename to tests/data/cases/fmtskip5.py diff --git a/tests/data/simple_cases/fmtskip6.py b/tests/data/cases/fmtskip6.py similarity index 100% rename from tests/data/simple_cases/fmtskip6.py rename to tests/data/cases/fmtskip6.py diff --git a/tests/data/simple_cases/fmtskip7.py b/tests/data/cases/fmtskip7.py similarity index 100% rename from tests/data/simple_cases/fmtskip7.py rename to tests/data/cases/fmtskip7.py diff --git a/tests/data/simple_cases/fmtskip8.py b/tests/data/cases/fmtskip8.py similarity index 100% rename from tests/data/simple_cases/fmtskip8.py rename to tests/data/cases/fmtskip8.py diff --git a/tests/data/simple_cases/fstring.py b/tests/data/cases/fstring.py similarity index 100% rename from tests/data/simple_cases/fstring.py rename to tests/data/cases/fstring.py diff --git a/tests/data/preview_py_310/funcdef_return_type_trailing_comma.py b/tests/data/cases/funcdef_return_type_trailing_comma.py similarity index 99% rename from tests/data/preview_py_310/funcdef_return_type_trailing_comma.py rename to tests/data/cases/funcdef_return_type_trailing_comma.py index 15db772f01e..9b9b9c673de 100644 --- a/tests/data/preview_py_310/funcdef_return_type_trailing_comma.py +++ b/tests/data/cases/funcdef_return_type_trailing_comma.py @@ -1,3 +1,4 @@ +# flags: --preview --minimum-version=3.10 # normal, short, function definition def foo(a, b) -> tuple[int, float]: ... diff --git a/tests/data/simple_cases/function.py b/tests/data/cases/function.py similarity index 100% rename from tests/data/simple_cases/function.py rename to tests/data/cases/function.py diff --git a/tests/data/simple_cases/function2.py b/tests/data/cases/function2.py similarity index 100% rename from tests/data/simple_cases/function2.py rename to tests/data/cases/function2.py diff --git a/tests/data/simple_cases/function_trailing_comma.py b/tests/data/cases/function_trailing_comma.py similarity index 100% rename from tests/data/simple_cases/function_trailing_comma.py rename to tests/data/cases/function_trailing_comma.py diff --git a/tests/data/simple_cases/ignore_pyi.py b/tests/data/cases/ignore_pyi.py similarity index 97% rename from tests/data/simple_cases/ignore_pyi.py rename to tests/data/cases/ignore_pyi.py index 3ef61079bfe..4fae7530eb9 100644 --- a/tests/data/simple_cases/ignore_pyi.py +++ b/tests/data/cases/ignore_pyi.py @@ -1,3 +1,4 @@ +# flags: --pyi def f(): # type: ignore ... diff --git a/tests/data/simple_cases/import_spacing.py b/tests/data/cases/import_spacing.py similarity index 100% rename from tests/data/simple_cases/import_spacing.py rename to tests/data/cases/import_spacing.py diff --git a/tests/data/miscellaneous/linelength6.py b/tests/data/cases/linelength6.py similarity index 80% rename from tests/data/miscellaneous/linelength6.py rename to tests/data/cases/linelength6.py index 4fb342726f5..158038bf960 100644 --- a/tests/data/miscellaneous/linelength6.py +++ b/tests/data/cases/linelength6.py @@ -1,3 +1,4 @@ +# flags: --line-length=6 # Regression test for #3427, which reproes only with line length <= 6 def f(): """ diff --git a/tests/data/miscellaneous/long_strings_flag_disabled.py b/tests/data/cases/long_strings_flag_disabled.py similarity index 100% rename from tests/data/miscellaneous/long_strings_flag_disabled.py rename to tests/data/cases/long_strings_flag_disabled.py diff --git a/tests/data/simple_cases/multiline_consecutive_open_parentheses_ignore.py b/tests/data/cases/multiline_consecutive_open_parentheses_ignore.py similarity index 100% rename from tests/data/simple_cases/multiline_consecutive_open_parentheses_ignore.py rename to tests/data/cases/multiline_consecutive_open_parentheses_ignore.py diff --git a/tests/data/miscellaneous/nested_stub.pyi b/tests/data/cases/nested_stub.py similarity index 97% rename from tests/data/miscellaneous/nested_stub.pyi rename to tests/data/cases/nested_stub.py index 15e69d854db..b81549ec115 100644 --- a/tests/data/miscellaneous/nested_stub.pyi +++ b/tests/data/cases/nested_stub.py @@ -1,3 +1,4 @@ +# flags: --pyi --preview import sys class Outer: diff --git a/tests/data/py_36/numeric_literals.py b/tests/data/cases/numeric_literals.py similarity index 91% rename from tests/data/py_36/numeric_literals.py rename to tests/data/cases/numeric_literals.py index 254da68d330..99669328744 100644 --- a/tests/data/py_36/numeric_literals.py +++ b/tests/data/cases/numeric_literals.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3.6 - x = 123456789 x = 123456 x = .1 @@ -21,9 +19,6 @@ # output - -#!/usr/bin/env python3.6 - x = 123456789 x = 123456 x = 0.1 diff --git a/tests/data/py_36/numeric_literals_skip_underscores.py b/tests/data/cases/numeric_literals_skip_underscores.py similarity index 80% rename from tests/data/py_36/numeric_literals_skip_underscores.py rename to tests/data/cases/numeric_literals_skip_underscores.py index e345bb90276..6d60bdbb34d 100644 --- a/tests/data/py_36/numeric_literals_skip_underscores.py +++ b/tests/data/cases/numeric_literals_skip_underscores.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3.6 - x = 123456789 x = 1_2_3_4_5_6_7 x = 1E+1 @@ -11,8 +9,6 @@ # output -#!/usr/bin/env python3.6 - x = 123456789 x = 1_2_3_4_5_6_7 x = 1e1 diff --git a/tests/data/simple_cases/one_element_subscript.py b/tests/data/cases/one_element_subscript.py similarity index 100% rename from tests/data/simple_cases/one_element_subscript.py rename to tests/data/cases/one_element_subscript.py diff --git a/tests/data/py_310/parenthesized_context_managers.py b/tests/data/cases/parenthesized_context_managers.py similarity index 95% rename from tests/data/py_310/parenthesized_context_managers.py rename to tests/data/cases/parenthesized_context_managers.py index 1ef09a1bd34..16645a18baa 100644 --- a/tests/data/py_310/parenthesized_context_managers.py +++ b/tests/data/cases/parenthesized_context_managers.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.10 with (CtxManager() as example): ... diff --git a/tests/data/py_310/pattern_matching_complex.py b/tests/data/cases/pattern_matching_complex.py similarity index 98% rename from tests/data/py_310/pattern_matching_complex.py rename to tests/data/cases/pattern_matching_complex.py index 97ee194fd39..b4355c7333a 100644 --- a/tests/data/py_310/pattern_matching_complex.py +++ b/tests/data/cases/pattern_matching_complex.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.10 # Cases sampled from Lib/test/test_patma.py # case black_test_patma_098 diff --git a/tests/data/py_310/pattern_matching_extras.py b/tests/data/cases/pattern_matching_extras.py similarity index 98% rename from tests/data/py_310/pattern_matching_extras.py rename to tests/data/cases/pattern_matching_extras.py index 0242d264e5b..1e1481d7bbe 100644 --- a/tests/data/py_310/pattern_matching_extras.py +++ b/tests/data/cases/pattern_matching_extras.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.10 import match match something: diff --git a/tests/data/py_310/pattern_matching_generic.py b/tests/data/cases/pattern_matching_generic.py similarity index 98% rename from tests/data/py_310/pattern_matching_generic.py rename to tests/data/cases/pattern_matching_generic.py index 00a0e4a677d..4b4d45f0bff 100644 --- a/tests/data/py_310/pattern_matching_generic.py +++ b/tests/data/cases/pattern_matching_generic.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.10 re.match() match = a with match() as match: diff --git a/tests/data/py_310/pattern_matching_simple.py b/tests/data/cases/pattern_matching_simple.py similarity index 98% rename from tests/data/py_310/pattern_matching_simple.py rename to tests/data/cases/pattern_matching_simple.py index 5ed62415a4b..6fa2000f0de 100644 --- a/tests/data/py_310/pattern_matching_simple.py +++ b/tests/data/cases/pattern_matching_simple.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.10 # Cases sampled from PEP 636 examples match command.split(): diff --git a/tests/data/py_310/pattern_matching_style.py b/tests/data/cases/pattern_matching_style.py similarity index 97% rename from tests/data/py_310/pattern_matching_style.py rename to tests/data/cases/pattern_matching_style.py index 8e18ce2ada6..2ee6ea2b6e9 100644 --- a/tests/data/py_310/pattern_matching_style.py +++ b/tests/data/cases/pattern_matching_style.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.10 match something: case b(): print(1+1) case c( diff --git a/tests/data/preview_py_310/pep604_union_types_line_breaks.py b/tests/data/cases/pep604_union_types_line_breaks.py similarity index 99% rename from tests/data/preview_py_310/pep604_union_types_line_breaks.py rename to tests/data/cases/pep604_union_types_line_breaks.py index 9c4ab870766..fee2b840494 100644 --- a/tests/data/preview_py_310/pep604_union_types_line_breaks.py +++ b/tests/data/cases/pep604_union_types_line_breaks.py @@ -1,3 +1,4 @@ +# flags: --preview --minimum-version=3.10 # This has always worked z= Loooooooooooooooooooooooong | Loooooooooooooooooooooooong | Loooooooooooooooooooooooong | Loooooooooooooooooooooooong diff --git a/tests/data/py_38/pep_570.py b/tests/data/cases/pep_570.py similarity index 95% rename from tests/data/py_38/pep_570.py rename to tests/data/cases/pep_570.py index ca8f7ab1d95..2641c2b970e 100644 --- a/tests/data/py_38/pep_570.py +++ b/tests/data/cases/pep_570.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.8 def positional_only_arg(a, /): pass diff --git a/tests/data/py_38/pep_572.py b/tests/data/cases/pep_572.py similarity index 96% rename from tests/data/py_38/pep_572.py rename to tests/data/cases/pep_572.py index d41805f1cb1..742b6d5b7e4 100644 --- a/tests/data/py_38/pep_572.py +++ b/tests/data/cases/pep_572.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.8 (a := 1) (a := a) if (match := pattern.search(data)) is None: diff --git a/tests/data/fast/pep_572_do_not_remove_parens.py b/tests/data/cases/pep_572_do_not_remove_parens.py similarity index 96% rename from tests/data/fast/pep_572_do_not_remove_parens.py rename to tests/data/cases/pep_572_do_not_remove_parens.py index 05619ddcc2b..08dba3ffdf9 100644 --- a/tests/data/fast/pep_572_do_not_remove_parens.py +++ b/tests/data/cases/pep_572_do_not_remove_parens.py @@ -1,3 +1,4 @@ +# flags: --fast # Most of the following examples are really dumb, some of them aren't even accepted by Python, # we're fixing them only so fuzzers (which follow the grammar which actually allows these # examples matter of fact!) don't yell at us :p diff --git a/tests/data/py_310/pep_572_py310.py b/tests/data/cases/pep_572_py310.py similarity index 93% rename from tests/data/py_310/pep_572_py310.py rename to tests/data/cases/pep_572_py310.py index cb82b2d23f8..9f999deeb89 100644 --- a/tests/data/py_310/pep_572_py310.py +++ b/tests/data/cases/pep_572_py310.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.10 # Unparenthesized walruses are now allowed in indices since Python 3.10. x[a:=0] x[a:=0, b:=1] diff --git a/tests/data/py_39/pep_572_py39.py b/tests/data/cases/pep_572_py39.py similarity index 89% rename from tests/data/py_39/pep_572_py39.py rename to tests/data/cases/pep_572_py39.py index b8b081b8c45..d1614624d99 100644 --- a/tests/data/py_39/pep_572_py39.py +++ b/tests/data/cases/pep_572_py39.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.9 # Unparenthesized walruses are now allowed in set literals & set comprehensions # since Python 3.9 {x := 1, 2, 3} diff --git a/tests/data/py_38/pep_572_remove_parens.py b/tests/data/cases/pep_572_remove_parens.py similarity index 98% rename from tests/data/py_38/pep_572_remove_parens.py rename to tests/data/cases/pep_572_remove_parens.py index b952b2940c5..24f1ac29168 100644 --- a/tests/data/py_38/pep_572_remove_parens.py +++ b/tests/data/cases/pep_572_remove_parens.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.8 if (foo := 0): pass diff --git a/tests/data/simple_cases/pep_604.py b/tests/data/cases/pep_604.py similarity index 100% rename from tests/data/simple_cases/pep_604.py rename to tests/data/cases/pep_604.py diff --git a/tests/data/py_311/pep_646.py b/tests/data/cases/pep_646.py similarity index 98% rename from tests/data/py_311/pep_646.py rename to tests/data/cases/pep_646.py index e843ecf39d8..92b568a379c 100644 --- a/tests/data/py_311/pep_646.py +++ b/tests/data/cases/pep_646.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.11 A[*b] A[*b] = 1 A diff --git a/tests/data/py_311/pep_654.py b/tests/data/cases/pep_654.py similarity index 96% rename from tests/data/py_311/pep_654.py rename to tests/data/cases/pep_654.py index 387c0816f4b..12e49180e41 100644 --- a/tests/data/py_311/pep_654.py +++ b/tests/data/cases/pep_654.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.11 try: raise OSError("blah") except* ExceptionGroup as e: diff --git a/tests/data/py_311/pep_654_style.py b/tests/data/cases/pep_654_style.py similarity index 98% rename from tests/data/py_311/pep_654_style.py rename to tests/data/cases/pep_654_style.py index 9fc7c0c84db..0d34650e098 100644 --- a/tests/data/py_311/pep_654_style.py +++ b/tests/data/cases/pep_654_style.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.11 try: raise OSError("blah") except * ExceptionGroup as e: diff --git a/tests/data/miscellaneous/power_op_newline.py b/tests/data/cases/power_op_newline.py similarity index 73% rename from tests/data/miscellaneous/power_op_newline.py rename to tests/data/cases/power_op_newline.py index 85d434d63f6..d9b31403c9d 100644 --- a/tests/data/miscellaneous/power_op_newline.py +++ b/tests/data/cases/power_op_newline.py @@ -1,3 +1,4 @@ +# flags: --line-length=0 importA;()<<0**0# # output diff --git a/tests/data/simple_cases/power_op_spacing.py b/tests/data/cases/power_op_spacing.py similarity index 100% rename from tests/data/simple_cases/power_op_spacing.py rename to tests/data/cases/power_op_spacing.py diff --git a/tests/data/simple_cases/prefer_rhs_split_reformatted.py b/tests/data/cases/prefer_rhs_split_reformatted.py similarity index 100% rename from tests/data/simple_cases/prefer_rhs_split_reformatted.py rename to tests/data/cases/prefer_rhs_split_reformatted.py diff --git a/tests/data/preview/async_stmts.py b/tests/data/cases/preview_async_stmts.py similarity index 93% rename from tests/data/preview/async_stmts.py rename to tests/data/cases/preview_async_stmts.py index fe9594b2164..0a7671be5a6 100644 --- a/tests/data/preview/async_stmts.py +++ b/tests/data/cases/preview_async_stmts.py @@ -1,3 +1,4 @@ +# flags: --preview async def func() -> (int): return 0 diff --git a/tests/data/preview/cantfit.py b/tests/data/cases/preview_cantfit.py similarity index 99% rename from tests/data/preview/cantfit.py rename to tests/data/cases/preview_cantfit.py index 0849374f776..d5da6654f0c 100644 --- a/tests/data/preview/cantfit.py +++ b/tests/data/cases/preview_cantfit.py @@ -1,3 +1,4 @@ +# flags: --preview # long variable name this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it = 0 this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it = 1 # with a comment diff --git a/tests/data/preview/comments7.py b/tests/data/cases/preview_comments7.py similarity index 99% rename from tests/data/preview/comments7.py rename to tests/data/cases/preview_comments7.py index 0655de999ec..006d4f7266f 100644 --- a/tests/data/preview/comments7.py +++ b/tests/data/cases/preview_comments7.py @@ -1,3 +1,4 @@ +# flags: --preview from .config import ( Any, Bool, diff --git a/tests/data/preview_context_managers/targeting_py38.py b/tests/data/cases/preview_context_managers_38.py similarity index 96% rename from tests/data/preview_context_managers/targeting_py38.py rename to tests/data/cases/preview_context_managers_38.py index f125cdffb8a..719d94fdcc5 100644 --- a/tests/data/preview_context_managers/targeting_py38.py +++ b/tests/data/cases/preview_context_managers_38.py @@ -1,3 +1,4 @@ +# flags: --preview --minimum-version=3.8 with \ make_context_manager1() as cm1, \ make_context_manager2() as cm2, \ diff --git a/tests/data/preview_context_managers/targeting_py39.py b/tests/data/cases/preview_context_managers_39.py similarity index 98% rename from tests/data/preview_context_managers/targeting_py39.py rename to tests/data/cases/preview_context_managers_39.py index c9fcf9c8ba2..589e00ad187 100644 --- a/tests/data/preview_context_managers/targeting_py39.py +++ b/tests/data/cases/preview_context_managers_39.py @@ -1,3 +1,4 @@ +# flags: --preview --minimum-version=3.9 with \ make_context_manager1() as cm1, \ make_context_manager2() as cm2, \ diff --git a/tests/data/preview_context_managers/auto_detect/features_3_10.py b/tests/data/cases/preview_context_managers_autodetect_310.py similarity index 93% rename from tests/data/preview_context_managers/auto_detect/features_3_10.py rename to tests/data/cases/preview_context_managers_autodetect_310.py index 1458df1cb41..a9e31076f03 100644 --- a/tests/data/preview_context_managers/auto_detect/features_3_10.py +++ b/tests/data/cases/preview_context_managers_autodetect_310.py @@ -1,3 +1,4 @@ +# flags: --preview --minimum-version=3.10 # This file uses pattern matching introduced in Python 3.10. diff --git a/tests/data/preview_context_managers/auto_detect/features_3_11.py b/tests/data/cases/preview_context_managers_autodetect_311.py similarity index 92% rename from tests/data/preview_context_managers/auto_detect/features_3_11.py rename to tests/data/cases/preview_context_managers_autodetect_311.py index f83c5330ab3..af1e83fe74c 100644 --- a/tests/data/preview_context_managers/auto_detect/features_3_11.py +++ b/tests/data/cases/preview_context_managers_autodetect_311.py @@ -1,3 +1,4 @@ +# flags: --preview --minimum-version=3.11 # This file uses except* clause in Python 3.11. diff --git a/tests/data/preview_context_managers/auto_detect/features_3_8.py b/tests/data/cases/preview_context_managers_autodetect_38.py similarity index 98% rename from tests/data/preview_context_managers/auto_detect/features_3_8.py rename to tests/data/cases/preview_context_managers_autodetect_38.py index 79e438b995e..25217a40604 100644 --- a/tests/data/preview_context_managers/auto_detect/features_3_8.py +++ b/tests/data/cases/preview_context_managers_autodetect_38.py @@ -1,3 +1,4 @@ +# flags: --preview # This file doesn't use any Python 3.9+ only grammars. diff --git a/tests/data/preview_context_managers/auto_detect/features_3_9.py b/tests/data/cases/preview_context_managers_autodetect_39.py similarity index 93% rename from tests/data/preview_context_managers/auto_detect/features_3_9.py rename to tests/data/cases/preview_context_managers_autodetect_39.py index 0d28f993108..3f72e48db9d 100644 --- a/tests/data/preview_context_managers/auto_detect/features_3_9.py +++ b/tests/data/cases/preview_context_managers_autodetect_39.py @@ -1,3 +1,4 @@ +# flags: --preview --minimum-version=3.9 # This file uses parenthesized context managers introduced in Python 3.9. diff --git a/tests/data/preview/dummy_implementations.py b/tests/data/cases/preview_dummy_implementations.py similarity index 98% rename from tests/data/preview/dummy_implementations.py rename to tests/data/cases/preview_dummy_implementations.py index e07c25ed129..98b69bf87b2 100644 --- a/tests/data/preview/dummy_implementations.py +++ b/tests/data/cases/preview_dummy_implementations.py @@ -1,3 +1,4 @@ +# flags: --preview from typing import NoReturn, Protocol, Union, overload diff --git a/tests/data/preview/format_unicode_escape_seq.py b/tests/data/cases/preview_format_unicode_escape_seq.py similarity index 96% rename from tests/data/preview/format_unicode_escape_seq.py rename to tests/data/cases/preview_format_unicode_escape_seq.py index 3440696c303..65c3d8d166e 100644 --- a/tests/data/preview/format_unicode_escape_seq.py +++ b/tests/data/cases/preview_format_unicode_escape_seq.py @@ -1,3 +1,4 @@ +# flags: --preview x = "\x1F" x = "\\x1B" x = "\\\x1B" diff --git a/tests/data/preview/long_dict_values.py b/tests/data/cases/preview_long_dict_values.py similarity index 99% rename from tests/data/preview/long_dict_values.py rename to tests/data/cases/preview_long_dict_values.py index 4c515180028..fbbacd13d1d 100644 --- a/tests/data/preview/long_dict_values.py +++ b/tests/data/cases/preview_long_dict_values.py @@ -1,3 +1,4 @@ +# flags: --preview my_dict = { "something_something": r"Lorem ipsum dolor sit amet, an sed convenire eloquentiam \t" diff --git a/tests/data/preview/long_strings.py b/tests/data/cases/preview_long_strings.py similarity index 99% rename from tests/data/preview/long_strings.py rename to tests/data/cases/preview_long_strings.py index 059148729d5..5519f098774 100644 --- a/tests/data/preview/long_strings.py +++ b/tests/data/cases/preview_long_strings.py @@ -1,3 +1,4 @@ +# flags: --preview x = "This is a really long string that can't possibly be expected to fit all together on one line. In fact it may even take up three or more lines... like four or five... but probably just three." x += "This is a really long string that can't possibly be expected to fit all together on one line. In fact it may even take up three or more lines... like four or five... but probably just three." diff --git a/tests/data/preview/long_strings__east_asian_width.py b/tests/data/cases/preview_long_strings__east_asian_width.py similarity index 96% rename from tests/data/preview/long_strings__east_asian_width.py rename to tests/data/cases/preview_long_strings__east_asian_width.py index fb66a78ed8b..d190f422a60 100644 --- a/tests/data/preview/long_strings__east_asian_width.py +++ b/tests/data/cases/preview_long_strings__east_asian_width.py @@ -1,3 +1,4 @@ +# flags: --preview # The following strings do not have not-so-many chars, but are long enough # when these are rendered in a monospace font (if the renderer respects # Unicode East Asian Width properties). diff --git a/tests/data/preview/long_strings__edge_case.py b/tests/data/cases/preview_long_strings__edge_case.py similarity index 99% rename from tests/data/preview/long_strings__edge_case.py rename to tests/data/cases/preview_long_strings__edge_case.py index 2bc0b6ed328..a8e8971968c 100644 --- a/tests/data/preview/long_strings__edge_case.py +++ b/tests/data/cases/preview_long_strings__edge_case.py @@ -1,3 +1,4 @@ +# flags: --preview some_variable = "This string is long but not so long that it needs to be split just yet" some_variable = 'This string is long but not so long that it needs to be split just yet' some_variable = "This string is long, just long enough that it needs to be split, u get?" diff --git a/tests/data/preview/long_strings__regression.py b/tests/data/cases/preview_long_strings__regression.py similarity index 99% rename from tests/data/preview/long_strings__regression.py rename to tests/data/cases/preview_long_strings__regression.py index 5f0646e6029..40d5e745cc8 100644 --- a/tests/data/preview/long_strings__regression.py +++ b/tests/data/cases/preview_long_strings__regression.py @@ -1,3 +1,4 @@ +# flags: --preview class A: def foo(): result = type(message)("") diff --git a/tests/data/preview/long_strings__type_annotations.py b/tests/data/cases/preview_long_strings__type_annotations.py similarity index 98% rename from tests/data/preview/long_strings__type_annotations.py rename to tests/data/cases/preview_long_strings__type_annotations.py index 45de882d02c..8beb877bdd1 100644 --- a/tests/data/preview/long_strings__type_annotations.py +++ b/tests/data/cases/preview_long_strings__type_annotations.py @@ -1,3 +1,4 @@ +# flags: --preview def func( arg1, arg2, diff --git a/tests/data/preview/multiline_strings.py b/tests/data/cases/preview_multiline_strings.py similarity index 99% rename from tests/data/preview/multiline_strings.py rename to tests/data/cases/preview_multiline_strings.py index bb517d128e2..dec4ef2e548 100644 --- a/tests/data/preview/multiline_strings.py +++ b/tests/data/cases/preview_multiline_strings.py @@ -1,3 +1,4 @@ +# flags: --preview """cow say""", call(3, "dogsay", textwrap.dedent("""dove diff --git a/tests/data/preview/no_blank_line_before_docstring.py b/tests/data/cases/preview_no_blank_line_before_docstring.py similarity index 97% rename from tests/data/preview/no_blank_line_before_docstring.py rename to tests/data/cases/preview_no_blank_line_before_docstring.py index a37362de100..303035a7efb 100644 --- a/tests/data/preview/no_blank_line_before_docstring.py +++ b/tests/data/cases/preview_no_blank_line_before_docstring.py @@ -1,3 +1,4 @@ +# flags: --preview def line_before_docstring(): """Please move me up""" diff --git a/tests/data/preview/pep_572.py b/tests/data/cases/preview_pep_572.py similarity index 75% rename from tests/data/preview/pep_572.py rename to tests/data/cases/preview_pep_572.py index a50e130ad9c..8e801ff6cdc 100644 --- a/tests/data/preview/pep_572.py +++ b/tests/data/cases/preview_pep_572.py @@ -1,3 +1,4 @@ +# flags: --preview x[(a:=0):] x[:(a:=0)] diff --git a/tests/data/preview/percent_precedence.py b/tests/data/cases/preview_percent_precedence.py similarity index 96% rename from tests/data/preview/percent_precedence.py rename to tests/data/cases/preview_percent_precedence.py index b895443fb46..aeaf450ff5e 100644 --- a/tests/data/preview/percent_precedence.py +++ b/tests/data/cases/preview_percent_precedence.py @@ -1,3 +1,4 @@ +# flags: --preview ("" % a) ** 2 ("" % a)[0] ("" % a)() diff --git a/tests/data/preview/prefer_rhs_split.py b/tests/data/cases/preview_prefer_rhs_split.py similarity index 99% rename from tests/data/preview/prefer_rhs_split.py rename to tests/data/cases/preview_prefer_rhs_split.py index a809eacc773..c732c33b53a 100644 --- a/tests/data/preview/prefer_rhs_split.py +++ b/tests/data/cases/preview_prefer_rhs_split.py @@ -1,3 +1,4 @@ +# flags: --preview first_item, second_item = ( some_looooooooong_module.some_looooooooooooooong_function_name( first_argument, second_argument, third_argument diff --git a/tests/data/preview/return_annotation_brackets_string.py b/tests/data/cases/preview_return_annotation_brackets_string.py similarity index 97% rename from tests/data/preview/return_annotation_brackets_string.py rename to tests/data/cases/preview_return_annotation_brackets_string.py index 9148bd045bc..fea0ea6839a 100644 --- a/tests/data/preview/return_annotation_brackets_string.py +++ b/tests/data/cases/preview_return_annotation_brackets_string.py @@ -1,3 +1,4 @@ +# flags: --preview # Long string example def frobnicate() -> "ThisIsTrulyUnreasonablyExtremelyLongClassName | list[ThisIsTrulyUnreasonablyExtremelyLongClassName]": pass diff --git a/tests/data/preview/trailing_comma.py b/tests/data/cases/preview_trailing_comma.py similarity index 97% rename from tests/data/preview/trailing_comma.py rename to tests/data/cases/preview_trailing_comma.py index 5b09c664606..bba7e7ad16d 100644 --- a/tests/data/preview/trailing_comma.py +++ b/tests/data/cases/preview_trailing_comma.py @@ -1,3 +1,4 @@ +# flags: --preview e = { "a": fun(msg, "ts"), "longggggggggggggggid": ..., diff --git a/tests/data/preview_py_310/pep_572.py b/tests/data/cases/py310_pep572.py similarity index 77% rename from tests/data/preview_py_310/pep_572.py rename to tests/data/cases/py310_pep572.py index 78d4e9e4506..172be3898d6 100644 --- a/tests/data/preview_py_310/pep_572.py +++ b/tests/data/cases/py310_pep572.py @@ -1,3 +1,4 @@ +# flags: --preview --minimum-version=3.10 x[a:=0] x[a := 0] x[a := 0, b := 1] diff --git a/tests/data/py_37/python37.py b/tests/data/cases/python37.py similarity index 95% rename from tests/data/py_37/python37.py rename to tests/data/cases/python37.py index dab8b404a73..3f61106c45d 100644 --- a/tests/data/py_37/python37.py +++ b/tests/data/cases/python37.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.7 +# flags: --minimum-version=3.7 def f(): @@ -33,9 +33,6 @@ def make_arange(n): # output -#!/usr/bin/env python3.7 - - def f(): return (i * 2 async for i in arange(42)) diff --git a/tests/data/py_38/python38.py b/tests/data/cases/python38.py similarity index 93% rename from tests/data/py_38/python38.py rename to tests/data/cases/python38.py index 63b0588bc27..919ea6aeed4 100644 --- a/tests/data/py_38/python38.py +++ b/tests/data/cases/python38.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.8 +# flags: --minimum-version=3.8 def starred_return(): @@ -22,9 +22,6 @@ def t(): # output -#!/usr/bin/env python3.8 - - def starred_return(): my_list = ["value2", "value3"] return "value1", *my_list diff --git a/tests/data/py_39/python39.py b/tests/data/cases/python39.py similarity index 92% rename from tests/data/py_39/python39.py rename to tests/data/cases/python39.py index ae67c2257eb..1b9536c1529 100644 --- a/tests/data/py_39/python39.py +++ b/tests/data/cases/python39.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.9 +# flags: --minimum-version=3.9 @relaxed_decorator[0] def f(): @@ -14,10 +14,6 @@ def f(): # output - -#!/usr/bin/env python3.9 - - @relaxed_decorator[0] def f(): ... diff --git a/tests/data/simple_cases/remove_await_parens.py b/tests/data/cases/remove_await_parens.py similarity index 100% rename from tests/data/simple_cases/remove_await_parens.py rename to tests/data/cases/remove_await_parens.py diff --git a/tests/data/simple_cases/remove_except_parens.py b/tests/data/cases/remove_except_parens.py similarity index 100% rename from tests/data/simple_cases/remove_except_parens.py rename to tests/data/cases/remove_except_parens.py diff --git a/tests/data/simple_cases/remove_for_brackets.py b/tests/data/cases/remove_for_brackets.py similarity index 100% rename from tests/data/simple_cases/remove_for_brackets.py rename to tests/data/cases/remove_for_brackets.py diff --git a/tests/data/simple_cases/remove_newline_after_code_block_open.py b/tests/data/cases/remove_newline_after_code_block_open.py similarity index 100% rename from tests/data/simple_cases/remove_newline_after_code_block_open.py rename to tests/data/cases/remove_newline_after_code_block_open.py diff --git a/tests/data/py_310/remove_newline_after_match.py b/tests/data/cases/remove_newline_after_match.py similarity index 94% rename from tests/data/py_310/remove_newline_after_match.py rename to tests/data/cases/remove_newline_after_match.py index f7bcfbf27a2..fe6592b664d 100644 --- a/tests/data/py_310/remove_newline_after_match.py +++ b/tests/data/cases/remove_newline_after_match.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.10 def http_status(status): match status: diff --git a/tests/data/simple_cases/remove_parens.py b/tests/data/cases/remove_parens.py similarity index 100% rename from tests/data/simple_cases/remove_parens.py rename to tests/data/cases/remove_parens.py diff --git a/tests/data/py_39/remove_with_brackets.py b/tests/data/cases/remove_with_brackets.py similarity index 98% rename from tests/data/py_39/remove_with_brackets.py rename to tests/data/cases/remove_with_brackets.py index ea58ab93a16..3ee64902a30 100644 --- a/tests/data/py_39/remove_with_brackets.py +++ b/tests/data/cases/remove_with_brackets.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.9 with (open("bla.txt")): pass diff --git a/tests/data/simple_cases/return_annotation_brackets.py b/tests/data/cases/return_annotation_brackets.py similarity index 100% rename from tests/data/simple_cases/return_annotation_brackets.py rename to tests/data/cases/return_annotation_brackets.py diff --git a/tests/data/simple_cases/skip_magic_trailing_comma.py b/tests/data/cases/skip_magic_trailing_comma.py similarity index 97% rename from tests/data/simple_cases/skip_magic_trailing_comma.py rename to tests/data/cases/skip_magic_trailing_comma.py index c020db79864..4dda5df40f0 100644 --- a/tests/data/simple_cases/skip_magic_trailing_comma.py +++ b/tests/data/cases/skip_magic_trailing_comma.py @@ -1,3 +1,4 @@ +# flags: --skip-magic-trailing-comma # We should not remove the trailing comma in a single-element subscript. a: tuple[int,] b = tuple[int,] diff --git a/tests/data/simple_cases/slices.py b/tests/data/cases/slices.py similarity index 100% rename from tests/data/simple_cases/slices.py rename to tests/data/cases/slices.py diff --git a/tests/data/py_310/starred_for_target.py b/tests/data/cases/starred_for_target.py similarity index 92% rename from tests/data/py_310/starred_for_target.py rename to tests/data/cases/starred_for_target.py index 8fc8e059ed3..13e517816d6 100644 --- a/tests/data/py_310/starred_for_target.py +++ b/tests/data/cases/starred_for_target.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.10 for x in *a, *b: print(x) diff --git a/tests/data/simple_cases/string_prefixes.py b/tests/data/cases/string_prefixes.py similarity index 100% rename from tests/data/simple_cases/string_prefixes.py rename to tests/data/cases/string_prefixes.py diff --git a/tests/data/miscellaneous/stub.pyi b/tests/data/cases/stub.py similarity index 99% rename from tests/data/miscellaneous/stub.pyi rename to tests/data/cases/stub.py index af2cd2c2c02..f3828d55ba2 100644 --- a/tests/data/miscellaneous/stub.pyi +++ b/tests/data/cases/stub.py @@ -1,3 +1,4 @@ +# flags: --pyi X: int def f(): ... diff --git a/tests/data/simple_cases/torture.py b/tests/data/cases/torture.py similarity index 100% rename from tests/data/simple_cases/torture.py rename to tests/data/cases/torture.py diff --git a/tests/data/simple_cases/trailing_comma_optional_parens1.py b/tests/data/cases/trailing_comma_optional_parens1.py similarity index 100% rename from tests/data/simple_cases/trailing_comma_optional_parens1.py rename to tests/data/cases/trailing_comma_optional_parens1.py diff --git a/tests/data/simple_cases/trailing_comma_optional_parens2.py b/tests/data/cases/trailing_comma_optional_parens2.py similarity index 100% rename from tests/data/simple_cases/trailing_comma_optional_parens2.py rename to tests/data/cases/trailing_comma_optional_parens2.py diff --git a/tests/data/simple_cases/trailing_comma_optional_parens3.py b/tests/data/cases/trailing_comma_optional_parens3.py similarity index 100% rename from tests/data/simple_cases/trailing_comma_optional_parens3.py rename to tests/data/cases/trailing_comma_optional_parens3.py diff --git a/tests/data/simple_cases/trailing_commas_in_leading_parts.py b/tests/data/cases/trailing_commas_in_leading_parts.py similarity index 100% rename from tests/data/simple_cases/trailing_commas_in_leading_parts.py rename to tests/data/cases/trailing_commas_in_leading_parts.py diff --git a/tests/data/simple_cases/tricky_unicode_symbols.py b/tests/data/cases/tricky_unicode_symbols.py similarity index 100% rename from tests/data/simple_cases/tricky_unicode_symbols.py rename to tests/data/cases/tricky_unicode_symbols.py diff --git a/tests/data/simple_cases/tupleassign.py b/tests/data/cases/tupleassign.py similarity index 100% rename from tests/data/simple_cases/tupleassign.py rename to tests/data/cases/tupleassign.py diff --git a/tests/data/py_312/type_aliases.py b/tests/data/cases/type_aliases.py similarity index 81% rename from tests/data/py_312/type_aliases.py rename to tests/data/cases/type_aliases.py index 84e07e50fe2..a3c1931c9fc 100644 --- a/tests/data/py_312/type_aliases.py +++ b/tests/data/cases/type_aliases.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.12 type A=int type Gen[T]=list[T] diff --git a/tests/data/type_comments/type_comment_syntax_error.py b/tests/data/cases/type_comment_syntax_error.py similarity index 100% rename from tests/data/type_comments/type_comment_syntax_error.py rename to tests/data/cases/type_comment_syntax_error.py diff --git a/tests/data/py_312/type_params.py b/tests/data/cases/type_params.py similarity index 97% rename from tests/data/py_312/type_params.py rename to tests/data/cases/type_params.py index 5f8ec43267c..720a775ef31 100644 --- a/tests/data/py_312/type_params.py +++ b/tests/data/cases/type_params.py @@ -1,3 +1,4 @@ +# flags: --minimum-version=3.12 def func [T ](): pass async def func [ T ] (): pass class C[ T ] : pass diff --git a/tests/data/simple_cases/whitespace.py b/tests/data/cases/whitespace.py similarity index 100% rename from tests/data/simple_cases/whitespace.py rename to tests/data/cases/whitespace.py diff --git a/tests/data/miscellaneous/force_pyi.py b/tests/data/miscellaneous/force_pyi.py index 07ed93c6879..40caf30a983 100644 --- a/tests/data/miscellaneous/force_pyi.py +++ b/tests/data/miscellaneous/force_pyi.py @@ -1,3 +1,4 @@ +# flags: --pyi from typing import Union @bird diff --git a/tests/test_black.py b/tests/test_black.py index c665eee3a6c..bb5cc1e08c7 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -187,7 +187,9 @@ def test_experimental_string_processing_warns(self) -> None: ) def test_piping(self) -> None: - source, expected = read_data_from_file(PROJECT_ROOT / "src/black/__init__.py") + _, source, expected = read_data_from_file( + PROJECT_ROOT / "src/black/__init__.py" + ) result = BlackRunner().invoke( black.main, [ @@ -209,8 +211,8 @@ def test_piping_diff(self) -> None: r"(STDIN|STDOUT)\t\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d\.\d\d\d\d\d\d" r"\+\d\d:\d\d" ) - source, _ = read_data("simple_cases", "expression.py") - expected, _ = read_data("simple_cases", "expression.diff") + source, _ = read_data("cases", "expression.py") + expected, _ = read_data("cases", "expression.diff") args = [ "-", "--fast", @@ -227,7 +229,7 @@ def test_piping_diff(self) -> None: self.assertEqual(expected, actual) def test_piping_diff_with_color(self) -> None: - source, _ = read_data("simple_cases", "expression.py") + source, _ = read_data("cases", "expression.py") args = [ "-", "--fast", @@ -263,7 +265,7 @@ def _test_wip(self) -> None: black.assert_stable(source, actual, black.FileMode()) def test_pep_572_version_detection(self) -> None: - source, _ = read_data("py_38", "pep_572") + source, _ = read_data("cases", "pep_572") root = black.lib2to3_parse(source) features = black.get_features_used(root) self.assertIn(black.Feature.ASSIGNMENT_EXPRESSIONS, features) @@ -272,7 +274,7 @@ def test_pep_572_version_detection(self) -> None: def test_pep_695_version_detection(self) -> None: for file in ("type_aliases", "type_params"): - source, _ = read_data("py_312", file) + source, _ = read_data("cases", file) root = black.lib2to3_parse(source) features = black.get_features_used(root) self.assertIn(black.Feature.TYPE_PARAMS, features) @@ -280,7 +282,7 @@ def test_pep_695_version_detection(self) -> None: self.assertIn(black.TargetVersion.PY312, versions) def test_expression_ff(self) -> None: - source, expected = read_data("simple_cases", "expression.py") + source, expected = read_data("cases", "expression.py") tmp_file = Path(black.dump_to_file(source)) try: self.assertTrue(ff(tmp_file, write_back=black.WriteBack.YES)) @@ -293,8 +295,8 @@ def test_expression_ff(self) -> None: black.assert_stable(source, actual, DEFAULT_MODE) def test_expression_diff(self) -> None: - source, _ = read_data("simple_cases", "expression.py") - expected, _ = read_data("simple_cases", "expression.diff") + source, _ = read_data("cases", "expression.py") + expected, _ = read_data("cases", "expression.diff") tmp_file = Path(black.dump_to_file(source)) diff_header = re.compile( rf"{re.escape(str(tmp_file))}\t\d\d\d\d-\d\d-\d\d " @@ -319,8 +321,8 @@ def test_expression_diff(self) -> None: self.assertEqual(expected, actual, msg) def test_expression_diff_with_color(self) -> None: - source, _ = read_data("simple_cases", "expression.py") - expected, _ = read_data("simple_cases", "expression.diff") + source, _ = read_data("cases", "expression.py") + expected, _ = read_data("cases", "expression.diff") tmp_file = Path(black.dump_to_file(source)) try: result = BlackRunner().invoke( @@ -339,7 +341,7 @@ def test_expression_diff_with_color(self) -> None: self.assertIn("\033[0m", actual) def test_detect_pos_only_arguments(self) -> None: - source, _ = read_data("py_38", "pep_570") + source, _ = read_data("cases", "pep_570") root = black.lib2to3_parse(source) features = black.get_features_used(root) self.assertIn(black.Feature.POS_ONLY_ARGUMENTS, features) @@ -401,7 +403,7 @@ def test_skip_source_first_line_when_mixing_newlines(self) -> None: self.assertEqual(test_file.read_bytes(), expected) def test_skip_magic_trailing_comma(self) -> None: - source, _ = read_data("simple_cases", "expression") + source, _ = read_data("cases", "expression") expected, _ = read_data( "miscellaneous", "expression_skip_magic_trailing_comma.diff" ) @@ -433,7 +435,7 @@ def test_skip_magic_trailing_comma(self) -> None: @patch("black.dump_to_file", dump_to_stderr) def test_async_as_identifier(self) -> None: source_path = get_case_path("miscellaneous", "async_as_identifier") - source, expected = read_data_from_file(source_path) + _, source, expected = read_data_from_file(source_path) actual = fs(source) self.assertFormatEqual(expected, actual) major, minor = sys.version_info[:2] @@ -447,8 +449,8 @@ def test_async_as_identifier(self) -> None: @patch("black.dump_to_file", dump_to_stderr) def test_python37(self) -> None: - source_path = get_case_path("py_37", "python37") - source, expected = read_data_from_file(source_path) + source_path = get_case_path("cases", "python37") + _, source, expected = read_data_from_file(source_path) actual = fs(source) self.assertFormatEqual(expected, actual) major, minor = sys.version_info[:2] @@ -884,7 +886,7 @@ def test_get_features_used(self) -> None: self.assertEqual(black.get_features_used(node), {Feature.NUMERIC_UNDERSCORES}) node = black.lib2to3_parse("123456\n") self.assertEqual(black.get_features_used(node), set()) - source, expected = read_data("simple_cases", "function") + source, expected = read_data("cases", "function") node = black.lib2to3_parse(source) expected_features = { Feature.TRAILING_COMMA_IN_CALL, @@ -894,7 +896,7 @@ def test_get_features_used(self) -> None: self.assertEqual(black.get_features_used(node), expected_features) node = black.lib2to3_parse(expected) self.assertEqual(black.get_features_used(node), expected_features) - source, expected = read_data("simple_cases", "expression") + source, expected = read_data("cases", "expression") node = black.lib2to3_parse(source) self.assertEqual(black.get_features_used(node), set()) node = black.lib2to3_parse(expected) @@ -1109,7 +1111,7 @@ def test_check_diff_use_together(self) -> None: src1 = get_case_path("miscellaneous", "string_quotes") self.invokeBlack([str(src1), "--diff", "--check"], exit_code=1) # Files which will not be reformatted. - src2 = get_case_path("simple_cases", "composition") + src2 = get_case_path("cases", "composition") self.invokeBlack([str(src2), "--diff", "--check"]) # Multi file command. self.invokeBlack([str(src1), str(src2), "--diff", "--check"], exit_code=1) @@ -1330,7 +1332,7 @@ def test_reformat_one_with_stdin_and_existing_path(self) -> None: report = MagicMock() # Even with an existing file, since we are forcing stdin, black # should output to stdout and not modify the file inplace - p = THIS_DIR / "data" / "simple_cases" / "collections.py" + p = THIS_DIR / "data" / "cases" / "collections.py" # Make sure is_file actually returns True self.assertTrue(p.is_file()) path = Path(f"__BLACK_STDIN_FILENAME__{p}") diff --git a/tests/test_blackd.py b/tests/test_blackd.py index dd2126e6bc2..c0152de73e6 100644 --- a/tests/test_blackd.py +++ b/tests/test_blackd.py @@ -104,7 +104,7 @@ async def check(header_value: str, expected_status: int = 400) -> None: @unittest_run_loop async def test_blackd_pyi(self) -> None: - source, expected = read_data("miscellaneous", "stub.pyi") + source, expected = read_data("cases", "stub.py") response = await self.client.post( "/", data=source, headers={blackd.PYTHON_VARIANT_HEADER: "pyi"} ) diff --git a/tests/test_format.py b/tests/test_format.py index ff358d59c94..4e863c6c54b 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -1,4 +1,3 @@ -import re from dataclasses import replace from typing import Any, Iterator from unittest.mock import patch @@ -6,13 +5,13 @@ import pytest import black +from black.mode import TargetVersion from tests.util import ( - DEFAULT_MODE, - PY36_VERSIONS, all_data_cases, assert_format, dump_to_stderr, read_data, + read_data_with_mode, ) @@ -22,61 +21,33 @@ def patch_dump_to_file(request: Any) -> Iterator[None]: yield -def check_file( - subdir: str, filename: str, mode: black.Mode, *, data: bool = True -) -> None: - source, expected = read_data(subdir, filename, data=data) - assert_format(source, expected, mode, fast=False) +def check_file(subdir: str, filename: str, *, data: bool = True) -> None: + args, source, expected = read_data_with_mode(subdir, filename, data=data) + assert_format( + source, + expected, + args.mode, + fast=args.fast, + minimum_version=args.minimum_version, + ) + if args.minimum_version is not None: + major, minor = args.minimum_version + target_version = TargetVersion[f"PY{major}{minor}"] + mode = replace(args.mode, target_versions={target_version}) + assert_format( + source, expected, mode, fast=args.fast, minimum_version=args.minimum_version + ) @pytest.mark.filterwarnings("ignore:invalid escape sequence.*:DeprecationWarning") -@pytest.mark.parametrize("filename", all_data_cases("simple_cases")) +@pytest.mark.parametrize("filename", all_data_cases("cases")) def test_simple_format(filename: str) -> None: - magic_trailing_comma = filename != "skip_magic_trailing_comma" - mode = black.Mode( - magic_trailing_comma=magic_trailing_comma, is_pyi=filename.endswith("_pyi") - ) - check_file("simple_cases", filename, mode) - - -@pytest.mark.parametrize("filename", all_data_cases("preview")) -def test_preview_format(filename: str) -> None: - check_file("preview", filename, black.Mode(preview=True)) - - -def test_preview_context_managers_targeting_py38() -> None: - source, expected = read_data("preview_context_managers", "targeting_py38.py") - mode = black.Mode(preview=True, target_versions={black.TargetVersion.PY38}) - assert_format(source, expected, mode, minimum_version=(3, 8)) - - -def test_preview_context_managers_targeting_py39() -> None: - source, expected = read_data("preview_context_managers", "targeting_py39.py") - mode = black.Mode(preview=True, target_versions={black.TargetVersion.PY39}) - assert_format(source, expected, mode, minimum_version=(3, 9)) - - -@pytest.mark.parametrize("filename", all_data_cases("preview_py_310")) -def test_preview_python_310(filename: str) -> None: - source, expected = read_data("preview_py_310", filename) - mode = black.Mode(target_versions={black.TargetVersion.PY310}, preview=True) - assert_format(source, expected, mode, minimum_version=(3, 10)) - - -@pytest.mark.parametrize( - "filename", all_data_cases("preview_context_managers/auto_detect") -) -def test_preview_context_managers_auto_detect(filename: str) -> None: - match = re.match(r"features_3_(\d+)", filename) - assert match is not None, "Unexpected filename format: %s" % filename - source, expected = read_data("preview_context_managers/auto_detect", filename) - mode = black.Mode(preview=True) - assert_format(source, expected, mode, minimum_version=(3, int(match.group(1)))) + check_file("cases", filename) # =============== # -# Complex cases -# ============= # +# Unusual cases +# =============== # def test_empty() -> None: @@ -84,48 +55,6 @@ def test_empty() -> None: assert_format(source, expected) -@pytest.mark.parametrize("filename", all_data_cases("py_36")) -def test_python_36(filename: str) -> None: - source, expected = read_data("py_36", filename) - mode = black.Mode(target_versions=PY36_VERSIONS) - assert_format(source, expected, mode, minimum_version=(3, 6)) - - -@pytest.mark.parametrize("filename", all_data_cases("py_37")) -def test_python_37(filename: str) -> None: - source, expected = read_data("py_37", filename) - mode = black.Mode(target_versions={black.TargetVersion.PY37}) - assert_format(source, expected, mode, minimum_version=(3, 7)) - - -@pytest.mark.parametrize("filename", all_data_cases("py_38")) -def test_python_38(filename: str) -> None: - source, expected = read_data("py_38", filename) - mode = black.Mode(target_versions={black.TargetVersion.PY38}) - assert_format(source, expected, mode, minimum_version=(3, 8)) - - -@pytest.mark.parametrize("filename", all_data_cases("py_39")) -def test_python_39(filename: str) -> None: - source, expected = read_data("py_39", filename) - mode = black.Mode(target_versions={black.TargetVersion.PY39}) - assert_format(source, expected, mode, minimum_version=(3, 9)) - - -@pytest.mark.parametrize("filename", all_data_cases("py_310")) -def test_python_310(filename: str) -> None: - source, expected = read_data("py_310", filename) - mode = black.Mode(target_versions={black.TargetVersion.PY310}) - assert_format(source, expected, mode, minimum_version=(3, 10)) - - -@pytest.mark.parametrize("filename", all_data_cases("py_310")) -def test_python_310_without_target_version(filename: str) -> None: - source, expected = read_data("py_310", filename) - mode = black.Mode() - assert_format(source, expected, mode, minimum_version=(3, 10)) - - def test_patma_invalid() -> None: source, expected = read_data("miscellaneous", "pattern_matching_invalid") mode = black.Mode(target_versions={black.TargetVersion.PY310}) @@ -133,82 +62,3 @@ def test_patma_invalid() -> None: assert_format(source, expected, mode, minimum_version=(3, 10)) exc_info.match("Cannot parse: 10:11") - - -@pytest.mark.parametrize("filename", all_data_cases("py_311")) -def test_python_311(filename: str) -> None: - source, expected = read_data("py_311", filename) - mode = black.Mode(target_versions={black.TargetVersion.PY311}) - assert_format(source, expected, mode, minimum_version=(3, 11)) - - -@pytest.mark.parametrize("filename", all_data_cases("py_312")) -def test_python_312(filename: str) -> None: - source, expected = read_data("py_312", filename) - mode = black.Mode(target_versions={black.TargetVersion.PY312}) - assert_format(source, expected, mode, minimum_version=(3, 12)) - - -@pytest.mark.parametrize("filename", all_data_cases("fast")) -def test_fast_cases(filename: str) -> None: - source, expected = read_data("fast", filename) - assert_format(source, expected, fast=True) - - -@pytest.mark.filterwarnings("ignore:invalid escape sequence.*:DeprecationWarning") -def test_docstring_no_string_normalization() -> None: - """Like test_docstring but with string normalization off.""" - source, expected = read_data("miscellaneous", "docstring_no_string_normalization") - mode = replace(DEFAULT_MODE, string_normalization=False) - assert_format(source, expected, mode) - - -def test_docstring_line_length_6() -> None: - """Like test_docstring but with line length set to 6.""" - source, expected = read_data("miscellaneous", "linelength6") - mode = black.Mode(line_length=6) - assert_format(source, expected, mode) - - -def test_preview_docstring_no_string_normalization() -> None: - """ - Like test_docstring but with string normalization off *and* the preview style - enabled. - """ - source, expected = read_data( - "miscellaneous", "docstring_preview_no_string_normalization" - ) - mode = replace(DEFAULT_MODE, string_normalization=False, preview=True) - assert_format(source, expected, mode) - - -def test_long_strings_flag_disabled() -> None: - """Tests for turning off the string processing logic.""" - source, expected = read_data("miscellaneous", "long_strings_flag_disabled") - mode = replace(DEFAULT_MODE, experimental_string_processing=False) - assert_format(source, expected, mode) - - -def test_stub() -> None: - mode = replace(DEFAULT_MODE, is_pyi=True) - source, expected = read_data("miscellaneous", "stub.pyi") - assert_format(source, expected, mode) - - -def test_nested_stub() -> None: - mode = replace(DEFAULT_MODE, is_pyi=True, preview=True) - source, expected = read_data("miscellaneous", "nested_stub.pyi") - assert_format(source, expected, mode) - - -def test_power_op_newline() -> None: - # requires line_length=0 - source, expected = read_data("miscellaneous", "power_op_newline") - assert_format(source, expected, mode=black.Mode(line_length=0)) - - -def test_type_comment_syntax_error() -> None: - """Test that black is able to format python code with type comment syntax errors.""" - source, expected = read_data("type_comments", "type_comment_syntax_error") - assert_format(source, expected) - black.assert_equivalent(source, expected) diff --git a/tests/util.py b/tests/util.py index 541d21da4df..a31ae0992c2 100644 --- a/tests/util.py +++ b/tests/util.py @@ -1,13 +1,17 @@ +import argparse +import functools import os +import shlex import sys import unittest from contextlib import contextmanager -from dataclasses import replace +from dataclasses import dataclass, field, replace from functools import partial from pathlib import Path from typing import Any, Iterator, List, Optional, Tuple import black +from black.const import DEFAULT_LINE_LENGTH from black.debug import DebugVisitor from black.mode import TargetVersion from black.output import diff, err, out @@ -35,6 +39,13 @@ fs = partial(black.format_str, mode=DEFAULT_MODE) +@dataclass +class TestCaseArgs: + mode: black.Mode = field(default_factory=black.Mode) + fast: bool = False + minimum_version: Optional[Tuple[int, int]] = None + + def _assert_format_equal(expected: str, actual: str) -> None: if actual != expected and (conftest.PRINT_FULL_TREE or conftest.PRINT_TREE_DIFF): bdv: DebugVisitor[Any] @@ -178,18 +189,85 @@ def get_case_path( return case_path +def read_data_with_mode( + subdir_name: str, name: str, data: bool = True +) -> Tuple[TestCaseArgs, str, str]: + """read_data_with_mode('test_name') -> Mode(), 'input', 'output'""" + return read_data_from_file(get_case_path(subdir_name, name, data)) + + def read_data(subdir_name: str, name: str, data: bool = True) -> Tuple[str, str]: """read_data('test_name') -> 'input', 'output'""" - return read_data_from_file(get_case_path(subdir_name, name, data)) + _, input, output = read_data_with_mode(subdir_name, name, data) + return input, output + + +def _parse_minimum_version(version: str) -> Tuple[int, int]: + major, minor = version.split(".") + return int(major), int(minor) -def read_data_from_file(file_name: Path) -> Tuple[str, str]: +@functools.lru_cache() +def get_flags_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + parser.add_argument( + "--target-version", + action="append", + type=lambda val: TargetVersion[val.upper()], + default=(), + ) + parser.add_argument("--line-length", default=DEFAULT_LINE_LENGTH, type=int) + parser.add_argument( + "--skip-string-normalization", default=False, action="store_true" + ) + parser.add_argument("--pyi", default=False, action="store_true") + parser.add_argument("--ipynb", default=False, action="store_true") + parser.add_argument( + "--skip-magic-trailing-comma", default=False, action="store_true" + ) + parser.add_argument("--preview", default=False, action="store_true") + parser.add_argument("--fast", default=False, action="store_true") + parser.add_argument( + "--minimum-version", + type=_parse_minimum_version, + default=None, + help=( + "Minimum version of Python where this test case is parseable. If this is" + " set, the test case will be run twice: once with the specified" + " --target-version, and once with --target-version set to exactly the" + " specified version. This ensures that Black's autodetection of the target" + " version works correctly." + ), + ) + return parser + + +def parse_mode(flags_line: str) -> TestCaseArgs: + parser = get_flags_parser() + args = parser.parse_args(shlex.split(flags_line)) + mode = black.Mode( + target_versions=set(args.target_version), + line_length=args.line_length, + string_normalization=not args.skip_string_normalization, + is_pyi=args.pyi, + is_ipynb=args.ipynb, + magic_trailing_comma=not args.skip_magic_trailing_comma, + preview=args.preview, + ) + return TestCaseArgs(mode=mode, fast=args.fast, minimum_version=args.minimum_version) + + +def read_data_from_file(file_name: Path) -> Tuple[TestCaseArgs, str, str]: with open(file_name, "r", encoding="utf8") as test: lines = test.readlines() _input: List[str] = [] _output: List[str] = [] result = _input + mode = TestCaseArgs() for line in lines: + if not _input and line.startswith("# flags: "): + mode = parse_mode(line[len("# flags: ") :]) + continue line = line.replace(EMPTY_LINE, "") if line.rstrip() == "# output": result = _output @@ -199,7 +277,7 @@ def read_data_from_file(file_name: Path) -> Tuple[str, str]: if _input and not _output: # If there's no output marker, treat the entire file as already pre-formatted. _output = _input[:] - return "".join(_input).strip() + "\n", "".join(_output).strip() + "\n" + return mode, "".join(_input).strip() + "\n", "".join(_output).strip() + "\n" def read_jupyter_notebook(subdir_name: str, name: str, data: bool = True) -> str: From 5d5bf6e0878539baeef797b87636235b8c02be3f Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 9 Oct 2023 18:44:36 -0700 Subject: [PATCH 138/279] Fix cache versioning when BLACK_CACHE_DIR is set (#3937) --- CHANGES.md | 2 ++ src/black/cache.py | 3 ++- tests/test_black.py | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ffc63b3287d..fe4b621a3e5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -24,6 +24,8 @@ +- Fix cache versioning logic when `BLACK_CACHE_DIR` is set (#3937) + ### Packaging diff --git a/src/black/cache.py b/src/black/cache.py index 77f66cc34a9..f7dc64c0bca 100644 --- a/src/black/cache.py +++ b/src/black/cache.py @@ -36,8 +36,9 @@ def get_cache_dir() -> Path: repeated calls. """ # NOTE: Function mostly exists as a clean way to test getting the cache directory. - default_cache_dir = user_cache_dir("black", version=__version__) + default_cache_dir = user_cache_dir("black") cache_dir = Path(os.environ.get("BLACK_CACHE_DIR", default_cache_dir)) + cache_dir = cache_dir / __version__ return cache_dir diff --git a/tests/test_black.py b/tests/test_black.py index bb5cc1e08c7..537ca80d432 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1963,11 +1963,11 @@ def test_get_cache_dir( # If BLACK_CACHE_DIR is not set, use user_cache_dir monkeypatch.delenv("BLACK_CACHE_DIR", raising=False) with patch_user_cache_dir: - assert get_cache_dir() == workspace1 + assert get_cache_dir().parent == workspace1 # If it is set, use the path provided in the env var. monkeypatch.setenv("BLACK_CACHE_DIR", str(workspace2)) - assert get_cache_dir() == workspace2 + assert get_cache_dir().parent == workspace2 def test_cache_broken_file(self) -> None: mode = DEFAULT_MODE From 7aa37ea0adf864baf3ef3dfbcfaf5ff1ff780250 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 9 Oct 2023 19:15:51 -0700 Subject: [PATCH 139/279] Report all stacktraces in verbose mode (#3938) Previously these were swallowed (unlike the ones in black/__init__.py) --- CHANGES.md | 2 ++ src/black/concurrency.py | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fe4b621a3e5..6ad6308945c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -44,6 +44,8 @@ - Black no longer attempts to provide special errors for attempting to format Python 2 code (#3933) +- Black will more consistently print stacktraces on internal errors in verbose mode + (#3938) ### _Blackd_ diff --git a/src/black/concurrency.py b/src/black/concurrency.py index ce016578399..55c96b66c86 100644 --- a/src/black/concurrency.py +++ b/src/black/concurrency.py @@ -9,6 +9,7 @@ import os import signal import sys +import traceback from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor from multiprocessing import Manager from pathlib import Path @@ -170,8 +171,10 @@ async def schedule_formatting( src = tasks.pop(task) if task.cancelled(): cancelled.append(task) - elif task.exception(): - report.failed(src, str(task.exception())) + elif exc := task.exception(): + if report.verbose: + traceback.print_exception(type(exc), exc, exc.__traceback__) + report.failed(src, str(exc)) else: changed = Changed.YES if task.result() else Changed.NO # If the file was written back or was successfully checked as From b7717c3f1e73d6b847e2534a2cebbb657b96caf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Tue, 10 Oct 2023 04:34:26 +0200 Subject: [PATCH 140/279] Standardise newlines after module-level docstrings (#3932) Co-authored-by: jpy-git Co-authored-by: Jelle Zijlstra --- CHANGES.md | 1 + scripts/make_width_table.py | 1 + src/black/cache.py | 1 + src/black/linegen.py | 1 + src/black/lines.py | 9 +++ src/black/mode.py | 1 + src/black/numerics.py | 1 + src/black/parsing.py | 1 + src/black/report.py | 1 + src/black/rusty.py | 1 + src/black/trans.py | 1 + tests/data/cases/module_docstring_1.py | 26 +++++++++ tests/data/cases/module_docstring_2.py | 68 +++++++++++++++++++++++ tests/data/cases/module_docstring_3.py | 8 +++ tests/data/cases/module_docstring_4.py | 9 +++ tests/data/miscellaneous/string_quotes.py | 2 + 16 files changed, 132 insertions(+) create mode 100644 tests/data/cases/module_docstring_1.py create mode 100644 tests/data/cases/module_docstring_2.py create mode 100644 tests/data/cases/module_docstring_3.py create mode 100644 tests/data/cases/module_docstring_4.py diff --git a/CHANGES.md b/CHANGES.md index 6ad6308945c..a608551815f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,6 +19,7 @@ - Long type hints are now wrapped in parentheses and properly indented when split across multiple lines (#3899) - Magic trailing commas are now respected in return types. (#3916) +- Require one empty line after module-level docstrings. (#3932) ### Configuration diff --git a/scripts/make_width_table.py b/scripts/make_width_table.py index 30fd32c34b0..061fdc8d95d 100644 --- a/scripts/make_width_table.py +++ b/scripts/make_width_table.py @@ -15,6 +15,7 @@ pip install -U wcwidth """ + import sys from os.path import basename, dirname, join from typing import Iterable, Tuple diff --git a/src/black/cache.py b/src/black/cache.py index f7dc64c0bca..6baa096baca 100644 --- a/src/black/cache.py +++ b/src/black/cache.py @@ -1,4 +1,5 @@ """Caching of formatted files with feature-based invalidation.""" + import hashlib import os import pickle diff --git a/src/black/linegen.py b/src/black/linegen.py index bdc4ee54ab2..faeb3ba664c 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -1,6 +1,7 @@ """ Generating lines of code. """ + import sys from dataclasses import replace from enum import Enum, auto diff --git a/src/black/lines.py b/src/black/lines.py index 71b657a0654..14754d7532f 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -550,6 +550,15 @@ def maybe_empty_lines(self, current_line: Line) -> LinesBlock: if self.previous_line is None else before - previous_after ) + if ( + Preview.module_docstring_newlines in current_line.mode + and self.previous_block + and self.previous_block.previous_block is None + and len(self.previous_block.original_line.leaves) == 1 + and self.previous_block.original_line.is_triple_quoted_string + ): + before = 1 + block = LinesBlock( mode=self.mode, previous_block=self.previous_block, diff --git a/src/black/mode.py b/src/black/mode.py index 30c5d2f1b2f..baf886abb7f 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -187,6 +187,7 @@ class Preview(Enum): wrap_multiple_context_managers_in_parens = auto() dummy_implementations = auto() walrus_subscript = auto() + module_docstring_newlines = auto() class Deprecated(UserWarning): diff --git a/src/black/numerics.py b/src/black/numerics.py index 879e5b2cf36..67ac8595fcc 100644 --- a/src/black/numerics.py +++ b/src/black/numerics.py @@ -1,6 +1,7 @@ """ Formatting numeric literals. """ + from blib2to3.pytree import Leaf diff --git a/src/black/parsing.py b/src/black/parsing.py index 03e767a333b..ea282d1805c 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -1,6 +1,7 @@ """ Parse Python code and perform AST validation. """ + import ast import sys from typing import Iterable, Iterator, List, Set, Tuple diff --git a/src/black/report.py b/src/black/report.py index a507671e4c0..89899f2f389 100644 --- a/src/black/report.py +++ b/src/black/report.py @@ -1,6 +1,7 @@ """ Summarize Black runs to users. """ + from dataclasses import dataclass from enum import Enum from pathlib import Path diff --git a/src/black/rusty.py b/src/black/rusty.py index 84a80b5a2c2..ebd4c052d1f 100644 --- a/src/black/rusty.py +++ b/src/black/rusty.py @@ -2,6 +2,7 @@ See https://doc.rust-lang.org/book/ch09-00-error-handling.html. """ + from typing import Generic, TypeVar, Union T = TypeVar("T") diff --git a/src/black/trans.py b/src/black/trans.py index c0cc92613ac..a2bff7f227a 100644 --- a/src/black/trans.py +++ b/src/black/trans.py @@ -1,6 +1,7 @@ """ String transformers that can split and merge strings. """ + import re from abc import ABC, abstractmethod from collections import defaultdict diff --git a/tests/data/cases/module_docstring_1.py b/tests/data/cases/module_docstring_1.py new file mode 100644 index 00000000000..d5897b4db60 --- /dev/null +++ b/tests/data/cases/module_docstring_1.py @@ -0,0 +1,26 @@ +# flags: --preview +"""Single line module-level docstring should be followed by single newline.""" + + + + +a = 1 + + +"""I'm just a string so should be followed by 2 newlines.""" + + + + +b = 2 + +# output +"""Single line module-level docstring should be followed by single newline.""" + +a = 1 + + +"""I'm just a string so should be followed by 2 newlines.""" + + +b = 2 diff --git a/tests/data/cases/module_docstring_2.py b/tests/data/cases/module_docstring_2.py new file mode 100644 index 00000000000..e1f81b4d76b --- /dev/null +++ b/tests/data/cases/module_docstring_2.py @@ -0,0 +1,68 @@ +# flags: --preview +"""I am a very helpful module docstring. + +Lorem ipsum dolor sit amet, consectetur adipiscing elit, +sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. +Ut enim ad minim veniam, +quis nostrud exercitation ullamco laboris +nisi ut aliquip ex ea commodo consequat. +Duis aute irure dolor in reprehenderit in voluptate +velit esse cillum dolore eu fugiat nulla pariatur. +Excepteur sint occaecat cupidatat non proident, +sunt in culpa qui officia deserunt mollit anim id est laborum. +""" + + + + +a = 1 + + +"""Look at me I'm a docstring... + +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +........................................................NOT! +""" + + + + +b = 2 + +# output +"""I am a very helpful module docstring. + +Lorem ipsum dolor sit amet, consectetur adipiscing elit, +sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. +Ut enim ad minim veniam, +quis nostrud exercitation ullamco laboris +nisi ut aliquip ex ea commodo consequat. +Duis aute irure dolor in reprehenderit in voluptate +velit esse cillum dolore eu fugiat nulla pariatur. +Excepteur sint occaecat cupidatat non proident, +sunt in culpa qui officia deserunt mollit anim id est laborum. +""" + +a = 1 + + +"""Look at me I'm a docstring... + +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +............................................................ +........................................................NOT! +""" + + +b = 2 diff --git a/tests/data/cases/module_docstring_3.py b/tests/data/cases/module_docstring_3.py new file mode 100644 index 00000000000..0631e136a3d --- /dev/null +++ b/tests/data/cases/module_docstring_3.py @@ -0,0 +1,8 @@ +# flags: --preview +"""Single line module-level docstring should be followed by single newline.""" +a = 1 + +# output +"""Single line module-level docstring should be followed by single newline.""" + +a = 1 diff --git a/tests/data/cases/module_docstring_4.py b/tests/data/cases/module_docstring_4.py new file mode 100644 index 00000000000..515174dcc04 --- /dev/null +++ b/tests/data/cases/module_docstring_4.py @@ -0,0 +1,9 @@ +# flags: --preview +"""Single line module-level docstring should be followed by single newline.""" + +a = 1 + +# output +"""Single line module-level docstring should be followed by single newline.""" + +a = 1 diff --git a/tests/data/miscellaneous/string_quotes.py b/tests/data/miscellaneous/string_quotes.py index 3384241f4ad..6ec088ac79b 100644 --- a/tests/data/miscellaneous/string_quotes.py +++ b/tests/data/miscellaneous/string_quotes.py @@ -1,4 +1,5 @@ '''''' + '\'' '"' "'" @@ -59,6 +60,7 @@ # output """""" + "'" '"' "'" From 935f303a0a7b794e722c7df00c906be285884874 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 9 Oct 2023 20:02:27 -0700 Subject: [PATCH 141/279] Fix test that was not being run (#3939) --- tests/data/{ => cases}/conditional_expression.py | 1 + 1 file changed, 1 insertion(+) rename tests/data/{ => cases}/conditional_expression.py (99%) diff --git a/tests/data/conditional_expression.py b/tests/data/cases/conditional_expression.py similarity index 99% rename from tests/data/conditional_expression.py rename to tests/data/cases/conditional_expression.py index 620a12dc986..c30cd76c791 100644 --- a/tests/data/conditional_expression.py +++ b/tests/data/cases/conditional_expression.py @@ -1,3 +1,4 @@ +# flags: --preview long_kwargs_single_line = my_function( foo="test, this is a sample value", bar=some_long_value_name_foo_bar_baz if some_boolean_variable else some_fallback_value_foo_bar_baz, From 3bb92146f59804a6ead47d5f2d0fdc47daa6b698 Mon Sep 17 00:00:00 2001 From: rdrll <13176405+rdrll@users.noreply.github.com> Date: Mon, 16 Oct 2023 05:13:53 -0700 Subject: [PATCH 142/279] CI Test: Deprecating 'Healthcheck.all()' from Hypothesis in fuzz.py (#3945) --- scripts/fuzz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/fuzz.py b/scripts/fuzz.py index 25362c927d4..0c507381d92 100644 --- a/scripts/fuzz.py +++ b/scripts/fuzz.py @@ -21,7 +21,7 @@ max_examples=1000, # roughly 1k tests/minute, or half that under coverage derandomize=True, # deterministic mode to avoid CI flakiness deadline=None, # ignore Hypothesis' health checks; we already know that - suppress_health_check=HealthCheck.all(), # this is slow and filter-heavy. + suppress_health_check=list(HealthCheck), # this is slow and filter-heavy. ) @given( # Note that while Hypothesmith might generate code unlike that written by From 6f84f652850dca8a1b578581e2fbb2cb95e791cc Mon Sep 17 00:00:00 2001 From: Charles Patel <17268094+acharles7@users.noreply.github.com> Date: Mon, 16 Oct 2023 07:24:16 -0500 Subject: [PATCH 143/279] Migrate mypy config to pyproject.toml (#3936) Co-authored-by: Charles Patel --- .gitignore | 1 + .pre-commit-config.yaml | 5 +++ mypy.ini | 46 ---------------------- pyproject.toml | 24 +++++++++++ scripts/check_pre_commit_rev_in_example.py | 2 +- scripts/check_version_in_basics_example.py | 2 +- scripts/diff_shades_gha_helper.py | 2 +- scripts/fuzz.py | 2 +- scripts/make_width_table.py | 2 +- src/blackd/__init__.py | 4 +- tests/optional.py | 2 +- tests/test_blackd.py | 2 +- 12 files changed, 40 insertions(+), 54 deletions(-) delete mode 100644 mypy.ini diff --git a/.gitignore b/.gitignore index 249499b135e..4a4f1b738ad 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ _build .DS_Store .vscode +.python-version docs/_static/pypi.svg .tox __pycache__ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 99b3565ed0e..623e661ac07 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,6 +43,7 @@ repos: hooks: - id: mypy exclude: ^docs/conf.py + args: ["--config-file", "pyproject.toml"] additional_dependencies: - types-PyYAML - tomli >= 0.2.6, < 2.0.0 @@ -51,6 +52,10 @@ repos: - platformdirs >= 2.1.0 - pytest - hypothesis + - aiohttp >= 3.7.4 + - types-commonmark + - urllib3 + - hypothesmith - repo: https://github.com/pre-commit/mirrors-prettier rev: v3.0.3 diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index ad916185bce..00000000000 --- a/mypy.ini +++ /dev/null @@ -1,46 +0,0 @@ -[mypy] -# Specify the target platform details in config, so your developers are -# free to run mypy on Windows, Linux, or macOS and get consistent -# results. -python_version=3.8 - -mypy_path=src - -show_column_numbers=True -show_error_codes=True - -# be strict -strict=True - -# except for... -no_implicit_reexport = False - -# Unreachable blocks have been an issue when compiling mypyc, let's try -# to avoid 'em in the first place. -warn_unreachable=True - -[mypy-blib2to3.driver.*] -ignore_missing_imports = True - -[mypy-IPython.*] -ignore_missing_imports = True - -[mypy-colorama.*] -ignore_missing_imports = True - -[mypy-pathspec.*] -ignore_missing_imports = True - -[mypy-tokenize_rt.*] -ignore_missing_imports = True - -[mypy-uvloop.*] -ignore_missing_imports = True - -[mypy-_black_version.*] -ignore_missing_imports = True - -# CI only checks src/, but in case users are running LSP or similar we explicitly ignore -# errors in test data files. -[mypy-tests.data.*] -ignore_errors = True diff --git a/pyproject.toml b/pyproject.toml index d246eb0b272..8c55076e4c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -137,6 +137,7 @@ exclude = [ # Compiled modules can't be run directly and that's a problem here: "/src/black/__main__.py", ] +mypy-args = ["--ignore-missing-imports"] options = { debug_level = "0" } [tool.cibuildwheel] @@ -223,3 +224,26 @@ omit = [ ] [tool.coverage.run] relative_files = true + +[tool.mypy] +# Specify the target platform details in config, so your developers are +# free to run mypy on Windows, Linux, or macOS and get consistent +# results. +python_version = "3.8" +mypy_path = "src" +strict = true +# Unreachable blocks have been an issue when compiling mypyc, let's try to avoid 'em in the first place. +warn_unreachable = true +implicit_reexport = true +show_error_codes = true +show_column_numbers = true + +[[tool.mypy.overrides]] +module = ["pathspec.*", "IPython.*", "colorama.*", "tokenize_rt.*", "uvloop.*", "_black_version.*"] +ignore_missing_imports = true + +# CI only checks src/, but in case users are running LSP or similar we explicitly ignore +# errors in test data files. +[[tool.mypy.overrides]] +module = ["tests.data.*"] +ignore_errors = true diff --git a/scripts/check_pre_commit_rev_in_example.py b/scripts/check_pre_commit_rev_in_example.py index 9560b3b8401..107c6444dca 100644 --- a/scripts/check_pre_commit_rev_in_example.py +++ b/scripts/check_pre_commit_rev_in_example.py @@ -14,7 +14,7 @@ import commonmark import yaml -from bs4 import BeautifulSoup +from bs4 import BeautifulSoup # type: ignore[import] def main(changes: str, source_version_control: str) -> None: diff --git a/scripts/check_version_in_basics_example.py b/scripts/check_version_in_basics_example.py index 7f559b3aee1..0f42bafe334 100644 --- a/scripts/check_version_in_basics_example.py +++ b/scripts/check_version_in_basics_example.py @@ -8,7 +8,7 @@ import sys import commonmark -from bs4 import BeautifulSoup +from bs4 import BeautifulSoup # type: ignore[import] def main(changes: str, the_basics: str) -> None: diff --git a/scripts/diff_shades_gha_helper.py b/scripts/diff_shades_gha_helper.py index 7a58fbe9b28..895516deb51 100644 --- a/scripts/diff_shades_gha_helper.py +++ b/scripts/diff_shades_gha_helper.py @@ -119,7 +119,7 @@ def main() -> None: @main.command("config", help="Acquire run configuration and metadata.") @click.argument("event", type=click.Choice(["push", "pull_request"])) def config(event: Literal["push", "pull_request"]) -> None: - import diff_shades + import diff_shades # type: ignore[import] if event == "push": jobs = [{"mode": "preview-changes", "force-flag": "--force-preview-style"}] diff --git a/scripts/fuzz.py b/scripts/fuzz.py index 0c507381d92..929d3eac4f5 100644 --- a/scripts/fuzz.py +++ b/scripts/fuzz.py @@ -80,7 +80,7 @@ def test_idempotent_any_syntatically_valid_python( try: import sys - import atheris + import atheris # type: ignore[import] except ImportError: pass else: diff --git a/scripts/make_width_table.py b/scripts/make_width_table.py index 061fdc8d95d..3c7cae60f7f 100644 --- a/scripts/make_width_table.py +++ b/scripts/make_width_table.py @@ -20,7 +20,7 @@ from os.path import basename, dirname, join from typing import Iterable, Tuple -import wcwidth +import wcwidth # type: ignore[import] def make_width_table() -> Iterable[Tuple[int, int, int]]: diff --git a/src/blackd/__init__.py b/src/blackd/__init__.py index 6b0f3d33295..972f24181cb 100644 --- a/src/blackd/__init__.py +++ b/src/blackd/__init__.py @@ -74,7 +74,9 @@ def main(bind_host: str, bind_port: int) -> None: app = make_app() ver = black.__version__ black.out(f"blackd version {ver} listening on {bind_host} port {bind_port}") - web.run_app(app, host=bind_host, port=bind_port, handle_signals=True, print=None) + # TODO: aiohttp had an incorrect annotation for `print` argument, + # It'll be fixed once aiohttp releases that code + web.run_app(app, host=bind_host, port=bind_port, handle_signals=True, print=None) # type: ignore[arg-type] def make_app() -> web.Application: diff --git a/tests/optional.py b/tests/optional.py index 8a39cc440a6..3f5277b6b03 100644 --- a/tests/optional.py +++ b/tests/optional.py @@ -26,7 +26,7 @@ from pytest import StashKey except ImportError: # pytest < 7 - from _pytest.store import StoreKey as StashKey # type: ignore[no-redef] + from _pytest.store import StoreKey as StashKey # type: ignore[import, no-redef] log = logging.getLogger(__name__) diff --git a/tests/test_blackd.py b/tests/test_blackd.py index c0152de73e6..59703036dc0 100644 --- a/tests/test_blackd.py +++ b/tests/test_blackd.py @@ -31,7 +31,7 @@ def unittest_run_loop(func, *args, **kwargs): @pytest.mark.blackd -class BlackDTestCase(AioHTTPTestCase): # type: ignore[misc] +class BlackDTestCase(AioHTTPTestCase): def test_blackd_main(self) -> None: with patch("blackd.web.run_app"): result = CliRunner().invoke(blackd.main, []) From 1648ac51806d092c95cb9bb2e4a5bffda6095bc1 Mon Sep 17 00:00:00 2001 From: Henri Holopainen Date: Mon, 16 Oct 2023 17:08:21 +0300 Subject: [PATCH 144/279] Fix long lines with power operator(s) getting splitted before line length (#3942) Fixes #3889 --- CHANGES.md | 1 + src/black/linegen.py | 21 ++++- src/black/mode.py | 1 + tests/data/cases/power_op_spacing.py | 18 ++++ tests/data/cases/preview_power_op_spacing.py | 97 ++++++++++++++++++++ 5 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 tests/data/cases/preview_power_op_spacing.py diff --git a/CHANGES.md b/CHANGES.md index a608551815f..d1c4a075c32 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,6 +16,7 @@ +- Fix long lines with power operators getting splitted before the line length (#3942) - Long type hints are now wrapped in parentheses and properly indented when split across multiple lines (#3899) - Magic trailing commas are now respected in return types. (#3916) diff --git a/src/black/linegen.py b/src/black/linegen.py index faeb3ba664c..d12ca39d037 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -536,6 +536,17 @@ def __post_init__(self) -> None: self.visit_case_block = self.visit_match_case +def _hugging_power_ops_line_to_string( + line: Line, + features: Collection[Feature], + mode: Mode, +) -> Optional[str]: + try: + return line_to_string(next(hug_power_op(line, features, mode))) + except CannotTransform: + return None + + def transform_line( line: Line, mode: Mode, features: Collection[Feature] = () ) -> Iterator[Line]: @@ -551,6 +562,14 @@ def transform_line( line_str = line_to_string(line) + # We need the line string when power operators are hugging to determine if we should + # split the line. Default to line_str, if no power operator are present on the line. + line_str_hugging_power_ops = ( + (_hugging_power_ops_line_to_string(line, features, mode) or line_str) + if Preview.fix_power_op_line_length in mode + else line_str + ) + ll = mode.line_length sn = mode.string_normalization string_merge = StringMerger(ll, sn) @@ -564,7 +583,7 @@ def transform_line( and not line.should_split_rhs and not line.magic_trailing_comma and ( - is_line_short_enough(line, mode=mode, line_str=line_str) + is_line_short_enough(line, mode=mode, line_str=line_str_hugging_power_ops) or line.contains_unsplittable_type_ignore() ) and not (line.inside_brackets and line.contains_standalone_comments()) diff --git a/src/black/mode.py b/src/black/mode.py index baf886abb7f..a57fa373568 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -188,6 +188,7 @@ class Preview(Enum): dummy_implementations = auto() walrus_subscript = auto() module_docstring_newlines = auto() + fix_power_op_line_length = auto() class Deprecated(UserWarning): diff --git a/tests/data/cases/power_op_spacing.py b/tests/data/cases/power_op_spacing.py index c95fa788fc3..b3ef0aae084 100644 --- a/tests/data/cases/power_op_spacing.py +++ b/tests/data/cases/power_op_spacing.py @@ -29,6 +29,13 @@ def function_dont_replace_spaces(): p = {(k, k**2): v**2 for k, v in pairs} q = [10**i for i in range(6)] r = x**y +s = 1 ** 1 +t = ( + 1 + ** 1 + **1 + ** 1 +) a = 5.0**~4.0 b = 5.0 ** f() @@ -47,6 +54,13 @@ def function_dont_replace_spaces(): o = settings(max_examples=10**6.0) p = {(k, k**2): v**2.0 for k, v in pairs} q = [10.5**i for i in range(6)] +s = 1.0 ** 1.0 +t = ( + 1.0 + ** 1.0 + **1.0 + ** 1.0 +) # WE SHOULD DEFINITELY NOT EAT THESE COMMENTS (https://github.com/psf/black/issues/2873) @@ -97,6 +111,8 @@ def function_dont_replace_spaces(): p = {(k, k**2): v**2 for k, v in pairs} q = [10**i for i in range(6)] r = x**y +s = 1**1 +t = 1**1**1**1 a = 5.0**~4.0 b = 5.0 ** f() @@ -115,6 +131,8 @@ def function_dont_replace_spaces(): o = settings(max_examples=10**6.0) p = {(k, k**2): v**2.0 for k, v in pairs} q = [10.5**i for i in range(6)] +s = 1.0**1.0 +t = 1.0**1.0**1.0**1.0 # WE SHOULD DEFINITELY NOT EAT THESE COMMENTS (https://github.com/psf/black/issues/2873) diff --git a/tests/data/cases/preview_power_op_spacing.py b/tests/data/cases/preview_power_op_spacing.py new file mode 100644 index 00000000000..650c6fecb20 --- /dev/null +++ b/tests/data/cases/preview_power_op_spacing.py @@ -0,0 +1,97 @@ +# flags: --preview +a = 1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1 +b = 1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1 +c = 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 +d = 1**1 ** 1**1 ** 1**1 ** 1**1 ** 1**1**1 ** 1 ** 1**1 ** 1**1**1**1**1 ** 1 ** 1**1**1 **1**1** 1 ** 1 ** 1 +e = 𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟 +f = 𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟 + +a = 1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0 +b = 1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0 +c = 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 ** 1.0 +d = 1.0**1.0 ** 1.0**1.0 ** 1.0**1.0 ** 1.0**1.0 ** 1.0**1.0**1.0 ** 1.0 ** 1.0**1.0 ** 1.0**1.0**1.0 + +# output +a = 1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1 +b = ( + 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 + ** 1 +) +c = 1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1 +d = 1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1 +e = 𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟**𨉟 +f = ( + 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 + ** 𨉟 +) + +a = 1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0 +b = ( + 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 + ** 1.0 +) +c = 1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0 +d = 1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0**1.0 From abe57e3d92727f1b26c717fad3978159b987fe79 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 16 Oct 2023 10:51:51 -0700 Subject: [PATCH 145/279] Treat raw strings like other docstrings (#3947) Fixes #3944 --- CHANGES.md | 1 + src/black/lines.py | 15 ++++++++++----- src/black/mode.py | 1 + tests/data/raw_docstring.py | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 tests/data/raw_docstring.py diff --git a/CHANGES.md b/CHANGES.md index d1c4a075c32..1f6a008d643 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -21,6 +21,7 @@ multiple lines (#3899) - Magic trailing commas are now respected in return types. (#3916) - Require one empty line after module-level docstrings. (#3932) +- Treat raw triple-quoted strings as docstrings (#3947) ### Configuration diff --git a/src/black/lines.py b/src/black/lines.py index 14754d7532f..48fde888208 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -193,11 +193,16 @@ def is_class_paren_empty(self) -> bool: @property def is_triple_quoted_string(self) -> bool: """Is the line a triple quoted string?""" - return ( - bool(self) - and self.leaves[0].type == token.STRING - and self.leaves[0].value.startswith(('"""', "'''")) - ) + if not self or self.leaves[0].type != token.STRING: + return False + value = self.leaves[0].value + if value.startswith(('"""', "'''")): + return True + if Preview.accept_raw_docstrings in self.mode and value.startswith( + ("r'''", 'r"""', "R'''", 'R"""') + ): + return True + return False @property def opens_block(self) -> bool: diff --git a/src/black/mode.py b/src/black/mode.py index a57fa373568..309f22dae94 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -188,6 +188,7 @@ class Preview(Enum): dummy_implementations = auto() walrus_subscript = auto() module_docstring_newlines = auto() + accept_raw_docstrings = auto() fix_power_op_line_length = auto() diff --git a/tests/data/raw_docstring.py b/tests/data/raw_docstring.py new file mode 100644 index 00000000000..751fd3201df --- /dev/null +++ b/tests/data/raw_docstring.py @@ -0,0 +1,32 @@ +# flags: --preview --skip-string-normalization +class C: + + r"""Raw""" + +def f(): + + r"""Raw""" + +class SingleQuotes: + + + r'''Raw''' + +class UpperCaseR: + R"""Raw""" + +# output +class C: + r"""Raw""" + + +def f(): + r"""Raw""" + + +class SingleQuotes: + r'''Raw''' + + +class UpperCaseR: + R"""Raw""" From 722735d20ebdc66c0da0e0df7658293455694500 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 16 Oct 2023 10:53:38 -0700 Subject: [PATCH 146/279] Fix grammar for type alias support (#3949) Fixes #3948 --- CHANGES.md | 3 +++ src/blib2to3/Grammar.txt | 2 +- tests/data/cases/type_aliases.py | 7 +++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 1f6a008d643..610a9de0e43 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -37,6 +37,9 @@ +- Add support for PEP 695 type aliases containing lambdas and other unusual expressions + (#3949) + ### Performance diff --git a/src/blib2to3/Grammar.txt b/src/blib2to3/Grammar.txt index be91df75740..5db78723cec 100644 --- a/src/blib2to3/Grammar.txt +++ b/src/blib2to3/Grammar.txt @@ -108,7 +108,7 @@ dotted_as_names: dotted_as_name (',' dotted_as_name)* dotted_name: NAME ('.' NAME)* global_stmt: ('global' | 'nonlocal') NAME (',' NAME)* assert_stmt: 'assert' test [',' test] -type_stmt: "type" NAME [typeparams] '=' expr +type_stmt: "type" NAME [typeparams] '=' test compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt | match_stmt async_stmt: ASYNC (funcdef | with_stmt | for_stmt) diff --git a/tests/data/cases/type_aliases.py b/tests/data/cases/type_aliases.py index a3c1931c9fc..9631bfd5ccc 100644 --- a/tests/data/cases/type_aliases.py +++ b/tests/data/cases/type_aliases.py @@ -1,6 +1,10 @@ # flags: --minimum-version=3.12 + type A=int type Gen[T]=list[T] +type Alias[T]=lambda: T +type And[T]=T and T +type IfElse[T]=T if T else T type = aliased print(type(42)) @@ -9,6 +13,9 @@ type A = int type Gen[T] = list[T] +type Alias[T] = lambda: T +type And[T] = T and T +type IfElse[T] = T if T else T type = aliased print(type(42)) From bb588073ab286a9f1f8d839ab2cebe13011dd22c Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 17 Oct 2023 00:59:15 -0700 Subject: [PATCH 147/279] Fix parser bug where "type" was misinterpreted as a keyword inside a match (#3950) Fixes #3790 Slightly hacky, but I think this is correct and it should also improve performance somewhat. --- CHANGES.md | 2 ++ src/blib2to3/pgen2/parse.py | 19 ++++++++++++++++++- tests/data/cases/pattern_matching_complex.py | 4 ++++ tests/data/cases/type_aliases.py | 9 +++++++++ 4 files changed, 33 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 610a9de0e43..f89b1b9df0a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -37,6 +37,8 @@ +- Fix bug where attributes named `type` were not acccepted inside `match` statements + (#3950) - Add support for PEP 695 type aliases containing lambdas and other unusual expressions (#3949) diff --git a/src/blib2to3/pgen2/parse.py b/src/blib2to3/pgen2/parse.py index 299cc24a15f..ad51a3dad08 100644 --- a/src/blib2to3/pgen2/parse.py +++ b/src/blib2to3/pgen2/parse.py @@ -211,6 +211,7 @@ def __init__(self, grammar: Grammar, convert: Optional[Convert] = None) -> None: # See note in docstring above. TL;DR this is ignored. self.convert = convert or lam_sub self.is_backtracking = False + self.last_token: Optional[int] = None def setup(self, proxy: "TokenProxy", start: Optional[int] = None) -> None: """Prepare for parsing. @@ -236,6 +237,7 @@ def setup(self, proxy: "TokenProxy", start: Optional[int] = None) -> None: self.rootnode: Optional[NL] = None self.used_names: Set[str] = set() self.proxy = proxy + self.last_token = None def addtoken(self, type: int, value: str, context: Context) -> bool: """Add a token; return True iff this is the end of the program.""" @@ -317,6 +319,7 @@ def _addtoken(self, ilabel: int, type: int, value: str, context: Context) -> boo dfa, state, node = self.stack[-1] states, first = dfa # Done with this token + self.last_token = type return False else: @@ -343,9 +346,23 @@ def classify(self, type: int, value: str, context: Context) -> List[int]: return [self.grammar.keywords[value]] elif value in self.grammar.soft_keywords: assert type in self.grammar.tokens + # Current soft keywords (match, case, type) can only appear at the + # beginning of a statement. So as a shortcut, don't try to treat them + # like keywords in any other context. + # ('_' is also a soft keyword in the real grammar, but for our grammar + # it's just an expression, so we don't need to treat it specially.) + if self.last_token not in ( + None, + token.INDENT, + token.DEDENT, + token.NEWLINE, + token.SEMI, + token.COLON, + ): + return [self.grammar.tokens[type]] return [ - self.grammar.soft_keywords[value], self.grammar.tokens[type], + self.grammar.soft_keywords[value], ] ilabel = self.grammar.tokens.get(type) diff --git a/tests/data/cases/pattern_matching_complex.py b/tests/data/cases/pattern_matching_complex.py index b4355c7333a..10b4d26e289 100644 --- a/tests/data/cases/pattern_matching_complex.py +++ b/tests/data/cases/pattern_matching_complex.py @@ -143,3 +143,7 @@ y = 1 case []: y = 2 +# issue 3790 +match (X.type, Y): + case _: + pass diff --git a/tests/data/cases/type_aliases.py b/tests/data/cases/type_aliases.py index 9631bfd5ccc..7c2009e8202 100644 --- a/tests/data/cases/type_aliases.py +++ b/tests/data/cases/type_aliases.py @@ -5,6 +5,8 @@ type Alias[T]=lambda: T type And[T]=T and T type IfElse[T]=T if T else T +type One = int; type Another = str +class X: type InClass = int type = aliased print(type(42)) @@ -16,6 +18,13 @@ type Alias[T] = lambda: T type And[T] = T and T type IfElse[T] = T if T else T +type One = int +type Another = str + + +class X: + type InClass = int + type = aliased print(type(42)) From 9edba85f71d50d12996ef7bda576426362016171 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 17 Oct 2023 07:22:24 -0700 Subject: [PATCH 148/279] Prepare release 23.10.0 (#3951) --- CHANGES.md | 60 +++++++++++++-------- docs/integrations/source_version_control.md | 4 +- docs/usage_and_configuration/the_basics.md | 6 +-- 3 files changed, 42 insertions(+), 28 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f89b1b9df0a..2a50e456550 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,25 +10,14 @@ -- Fix comments getting removed from inside parenthesized strings (#3909) - ### Preview style -- Fix long lines with power operators getting splitted before the line length (#3942) -- Long type hints are now wrapped in parentheses and properly indented when split across - multiple lines (#3899) -- Magic trailing commas are now respected in return types. (#3916) -- Require one empty line after module-level docstrings. (#3932) -- Treat raw triple-quoted strings as docstrings (#3947) - ### Configuration -- Fix cache versioning logic when `BLACK_CACHE_DIR` is set (#3937) - ### Packaging @@ -37,11 +26,6 @@ -- Fix bug where attributes named `type` were not acccepted inside `match` statements - (#3950) -- Add support for PEP 695 type aliases containing lambdas and other unusual expressions - (#3949) - ### Performance @@ -50,11 +34,6 @@ -- Black no longer attempts to provide special errors for attempting to format Python 2 - code (#3933) -- Black will more consistently print stacktraces on internal errors in verbose mode - (#3938) - ### _Blackd_ @@ -63,13 +42,48 @@ -- The action output displayed in the job summary is now wrapped in Markdown (#3914) - ### Documentation +## 23.10.0 + +### Stable style + +- Fix comments getting removed from inside parenthesized strings (#3909) + +### Preview style + +- Fix long lines with power operators getting split before the line length (#3942) +- Long type hints are now wrapped in parentheses and properly indented when split across + multiple lines (#3899) +- Magic trailing commas are now respected in return types. (#3916) +- Require one empty line after module-level docstrings. (#3932) +- Treat raw triple-quoted strings as docstrings (#3947) + +### Configuration + +- Fix cache versioning logic when `BLACK_CACHE_DIR` is set (#3937) + +### Parser + +- Fix bug where attributes named `type` were not acccepted inside `match` statements + (#3950) +- Add support for PEP 695 type aliases containing lambdas and other unusual expressions + (#3949) + +### Output + +- Black no longer attempts to provide special errors for attempting to format Python 2 + code (#3933) +- Black will more consistently print stacktraces on internal errors in verbose mode + (#3938) + +### Integrations + +- The action output displayed in the job summary is now wrapped in Markdown (#3914) + ## 23.9.1 Due to various issues, the previous release (23.9.0) did not include compiled mypyc diff --git a/docs/integrations/source_version_control.md b/docs/integrations/source_version_control.md index 2afcc02f3cd..16354f849ba 100644 --- a/docs/integrations/source_version_control.md +++ b/docs/integrations/source_version_control.md @@ -8,7 +8,7 @@ Use [pre-commit](https://pre-commit.com/). Once you repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.9.1 + rev: 23.10.0 hooks: - id: black # It is recommended to specify the latest version of Python @@ -35,7 +35,7 @@ include Jupyter Notebooks. To use this hook, simply replace the hook's `id: blac repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.9.1 + rev: 23.10.0 hooks: - id: black-jupyter # It is recommended to specify the latest version of Python diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 36119f225e6..5b132a95eae 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -194,8 +194,8 @@ configuration file for consistent results across environments. ```console $ black --version -black, 23.9.1 (compiled: yes) -$ black --required-version 23.9.1 -c "format = 'this'" +black, 23.10.0 (compiled: yes) +$ black --required-version 23.10.0 -c "format = 'this'" format = "this" $ black --required-version 31.5b2 -c "still = 'beta?!'" Oh no! 💥 💔 💥 The required version does not match the running version! @@ -286,7 +286,7 @@ You can check the version of _Black_ you have installed using the `--version` fl ```console $ black --version -black, 23.9.1 +black, 23.10.0 ``` #### `--config` From 882d8795c6ff65c02f2657e596391748d1b6b7f5 Mon Sep 17 00:00:00 2001 From: Henri Holopainen Date: Fri, 20 Oct 2023 06:09:33 +0300 Subject: [PATCH 149/279] Fix merging implicit multiline strings that have inline comments (#3956) * Fix test behaviour * Add new test cases * Skip merging strings that have inline comments * Don't merge lines with multiline strings with inline comments * Changelog entry * Document implicit multiline string merging rules * Fix PR number --- CHANGES.md | 2 +- docs/the_black_code_style/future_style.md | 64 +++++++++++++++++++ src/black/linegen.py | 1 + src/black/lines.py | 15 +++++ src/black/trans.py | 14 +++- .../cases/preview_long_strings__regression.py | 6 +- tests/data/cases/preview_multiline_strings.py | 28 ++++++++ 7 files changed, 125 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2a50e456550..79b5c6034e8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,7 +12,7 @@ ### Preview style - +- Fix merging implicit multiline strings that have inline comments (#3956) ### Configuration diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index 861bb64bff4..367ff98537c 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -160,3 +160,67 @@ MULTILINE = """ foobar """.replace("\n", "") ``` + +Implicit multiline strings are special, because they can have inline comments. Strings +without comments are merged, for example + +```python +s = ( + "An " + "implicit " + "multiline " + "string" +) +``` + +becomes + +```python +s = "An implicit multiline string" +``` + +A comment on any line of the string (or between two string lines) will block the +merging, so + +```python +s = ( + "An " # Important comment concerning just this line + "implicit " + "multiline " + "string" +) +``` + +and + +```python +s = ( + "An " + "implicit " + # Comment in between + "multiline " + "string" +) +``` + +will not be merged. Having the comment after or before the string lines (but still +inside the parens) will merge the string. For example + +```python +s = ( # Top comment + "An " + "implicit " + "multiline " + "string" + # Bottom comment +) +``` + +becomes + +```python +s = ( # Top comment + "An implicit multiline string" + # Bottom comment +) +``` diff --git a/src/black/linegen.py b/src/black/linegen.py index d12ca39d037..2bfe587fa0e 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -587,6 +587,7 @@ def transform_line( or line.contains_unsplittable_type_ignore() ) and not (line.inside_brackets and line.contains_standalone_comments()) + and not line.contains_implicit_multiline_string_with_comments() ): # Only apply basic string preprocessing, since lines shouldn't be split here. if Preview.string_processing in mode: diff --git a/src/black/lines.py b/src/black/lines.py index 48fde888208..6acc95e7a7e 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -239,6 +239,21 @@ def contains_standalone_comments(self, depth_limit: int = sys.maxsize) -> bool: return False + def contains_implicit_multiline_string_with_comments(self) -> bool: + """Chck if we have an implicit multiline string with comments on the line""" + for leaf_type, leaf_group_iterator in itertools.groupby( + self.leaves, lambda leaf: leaf.type + ): + if leaf_type != token.STRING: + continue + leaf_list = list(leaf_group_iterator) + if len(leaf_list) == 1: + continue + for leaf in leaf_list: + if self.comments_after(leaf): + return True + return False + def contains_uncollapsable_type_comments(self) -> bool: ignored_ids = set() try: diff --git a/src/black/trans.py b/src/black/trans.py index a2bff7f227a..a3f6467cc9e 100644 --- a/src/black/trans.py +++ b/src/black/trans.py @@ -390,7 +390,19 @@ def do_match(self, line: Line) -> TMatchResult: and is_valid_index(idx + 1) and LL[idx + 1].type == token.STRING ): - if not is_part_of_annotation(leaf): + # Let's check if the string group contains an inline comment + # If we have a comment inline, we don't merge the strings + contains_comment = False + i = idx + while is_valid_index(i): + if LL[i].type != token.STRING: + break + if line.comments_after(LL[i]): + contains_comment = True + break + i += 1 + + if not is_part_of_annotation(leaf) and not contains_comment: string_indices.append(idx) # Advance to the next non-STRING leaf. diff --git a/tests/data/cases/preview_long_strings__regression.py b/tests/data/cases/preview_long_strings__regression.py index 40d5e745cc8..436157f4e05 100644 --- a/tests/data/cases/preview_long_strings__regression.py +++ b/tests/data/cases/preview_long_strings__regression.py @@ -210,8 +210,8 @@ def foo(): some_tuple = ("some string", "some string" " which should be joined") -some_commented_string = ( - "This string is long but not so long that it needs hahahah toooooo be so greatttt" # This comment gets thrown to the top. +some_commented_string = ( # This comment stays at the top. + "This string is long but not so long that it needs hahahah toooooo be so greatttt" " {} that I just can't think of any more good words to say about it at" " allllllllllll".format("ha") # comments here are fine ) @@ -834,7 +834,7 @@ def foo(): some_tuple = ("some string", "some string which should be joined") -some_commented_string = ( # This comment gets thrown to the top. +some_commented_string = ( # This comment stays at the top. "This string is long but not so long that it needs hahahah toooooo be so greatttt" " {} that I just can't think of any more good words to say about it at" " allllllllllll".format("ha") # comments here are fine diff --git a/tests/data/cases/preview_multiline_strings.py b/tests/data/cases/preview_multiline_strings.py index dec4ef2e548..3ff643610b7 100644 --- a/tests/data/cases/preview_multiline_strings.py +++ b/tests/data/cases/preview_multiline_strings.py @@ -157,6 +157,24 @@ def dastardly_default_value( `--global-option` is reserved to flags like `--verbose` or `--quiet`. """ +this_will_become_one_line = ( + "a" + "b" + "c" +) + +this_will_stay_on_three_lines = ( + "a" # comment + "b" + "c" +) + +this_will_also_become_one_line = ( # comment + "a" + "b" + "c" +) + # output """cow say""", @@ -357,3 +375,13 @@ def dastardly_default_value( Please use `--build-option` instead, `--global-option` is reserved to flags like `--verbose` or `--quiet`. """ + +this_will_become_one_line = "abc" + +this_will_stay_on_three_lines = ( + "a" # comment + "b" + "c" +) + +this_will_also_become_one_line = "abc" # comment From 0a37888e79059018eef9293a724b14da59d3377a Mon Sep 17 00:00:00 2001 From: Aniket Patil <128228805+AniketP04@users.noreply.github.com> Date: Mon, 23 Oct 2023 02:46:43 +0530 Subject: [PATCH 150/279] Fix typos in CHANGES.md (#3963) --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 79b5c6034e8..a75b54d8d81 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -68,7 +68,7 @@ ### Parser -- Fix bug where attributes named `type` were not acccepted inside `match` statements +- Fix bug where attributes named `type` were not accepted inside `match` statements (#3950) - Add support for PEP 695 type aliases containing lambdas and other unusual expressions (#3949) @@ -926,7 +926,7 @@ and the first release covered by our new [`master`](https://github.com/psf/black/tree/main) branch with the [`main`](https://github.com/psf/black/tree/main) branch. Some additional changes in the source code were also made. (#2210) -- Sigificantly reorganized the documentation to make much more sense. Check them out by +- Significantly reorganized the documentation to make much more sense. Check them out by heading over to [the stable docs on RTD](https://black.readthedocs.io/en/stable/). (#2174) From 2db5ab0a7b3b321e4cec70964239ee88087cd810 Mon Sep 17 00:00:00 2001 From: Henri Holopainen Date: Mon, 23 Oct 2023 17:38:36 +0300 Subject: [PATCH 151/279] Allow empty line after block open before a comment or compound statement (#3967) --- CHANGES.md | 1 + src/black/lines.py | 27 ++++- src/black/mode.py | 1 + src/black/nodes.py | 4 + ...allow_empty_first_line_in_special_cases.py | 106 ++++++++++++++++++ 5 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 tests/data/cases/preview_allow_empty_first_line_in_special_cases.py diff --git a/CHANGES.md b/CHANGES.md index a75b54d8d81..86e820a6fd9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,6 +13,7 @@ ### Preview style - Fix merging implicit multiline strings that have inline comments (#3956) +- Allow empty first line after block open before a comment or compound statement (#3967) ### Configuration diff --git a/src/black/lines.py b/src/black/lines.py index 6acc95e7a7e..a73c429e3d9 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -24,6 +24,8 @@ STANDALONE_COMMENT, TEST_DESCENDANTS, child_towards, + is_docstring, + is_funcdef, is_import, is_multiline_string, is_one_sequence_between, @@ -686,7 +688,30 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: return 0, 1 return before, 1 - if self.previous_line and self.previous_line.opens_block: + is_empty_first_line_ok = ( + Preview.allow_empty_first_line_before_new_block_or_comment + in current_line.mode + and ( + # If it's a standalone comment + current_line.leaves[0].type == STANDALONE_COMMENT + # If it opens a new block + or current_line.opens_block + # If it's a triple quote comment (but not at the start of a funcdef) + or ( + is_docstring(current_line.leaves[0]) + and self.previous_line + and self.previous_line.leaves[0] + and self.previous_line.leaves[0].parent + and not is_funcdef(self.previous_line.leaves[0].parent) + ) + ) + ) + + if ( + self.previous_line + and self.previous_line.opens_block + and not is_empty_first_line_ok + ): return 0, 0 return before, 0 diff --git a/src/black/mode.py b/src/black/mode.py index 309f22dae94..4effeef3e7c 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -190,6 +190,7 @@ class Preview(Enum): module_docstring_newlines = auto() accept_raw_docstrings = auto() fix_power_op_line_length = auto() + allow_empty_first_line_before_new_block_or_comment = auto() class Deprecated(UserWarning): diff --git a/src/black/nodes.py b/src/black/nodes.py index edd201a21e9..b2e96cb9edf 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -718,6 +718,10 @@ def is_multiline_string(leaf: Leaf) -> bool: return has_triple_quotes(leaf.value) and "\n" in leaf.value +def is_funcdef(node: Node) -> bool: + return node.type == syms.funcdef + + def is_stub_suite(node: Node) -> bool: """Return True if `node` is a suite with a stub body.""" diff --git a/tests/data/cases/preview_allow_empty_first_line_in_special_cases.py b/tests/data/cases/preview_allow_empty_first_line_in_special_cases.py new file mode 100644 index 00000000000..96c1433c110 --- /dev/null +++ b/tests/data/cases/preview_allow_empty_first_line_in_special_cases.py @@ -0,0 +1,106 @@ +# flags: --preview +def foo(): + """ + Docstring + """ + + # Here we go + if x: + + # This is also now fine + a = 123 + + else: + # But not necessary + a = 123 + + if y: + + while True: + + """ + Long comment here + """ + a = 123 + + if z: + + for _ in range(100): + a = 123 + else: + + try: + + # this should be ok + a = 123 + except: + + """also this""" + a = 123 + + +def bar(): + + if x: + a = 123 + + +def baz(): + + # OK + if x: + a = 123 + +# output + +def foo(): + """ + Docstring + """ + + # Here we go + if x: + + # This is also now fine + a = 123 + + else: + # But not necessary + a = 123 + + if y: + + while True: + + """ + Long comment here + """ + a = 123 + + if z: + + for _ in range(100): + a = 123 + else: + + try: + + # this should be ok + a = 123 + except: + + """also this""" + a = 123 + + +def bar(): + + if x: + a = 123 + + +def baz(): + + # OK + if x: + a = 123 From 7f1c578b89b2c256a6ce3b70fc1b970b3ffa3349 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Oct 2023 07:42:49 -0700 Subject: [PATCH 152/279] Bump peter-evans/create-or-update-comment from 3.0.2 to 3.1.0 (#3966) Bumps [peter-evans/create-or-update-comment](https://github.com/peter-evans/create-or-update-comment) from 3.0.2 to 3.1.0. - [Release notes](https://github.com/peter-evans/create-or-update-comment/releases) - [Commits](https://github.com/peter-evans/create-or-update-comment/compare/c6c9a1a66007646a28c153e2a8580a5bad27bcfa...23ff15729ef2fc348714a3bb66d2f655ca9066f2) --- updated-dependencies: - dependency-name: peter-evans/create-or-update-comment dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/diff_shades_comment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/diff_shades_comment.yml b/.github/workflows/diff_shades_comment.yml index b86bd93410e..49fd376d85e 100644 --- a/.github/workflows/diff_shades_comment.yml +++ b/.github/workflows/diff_shades_comment.yml @@ -41,7 +41,7 @@ jobs: - name: Create or update PR comment if: steps.metadata.outputs.needs-comment == 'true' - uses: peter-evans/create-or-update-comment@c6c9a1a66007646a28c153e2a8580a5bad27bcfa + uses: peter-evans/create-or-update-comment@23ff15729ef2fc348714a3bb66d2f655ca9066f2 with: comment-id: ${{ steps.find-comment.outputs.comment-id }} issue-number: ${{ steps.metadata.outputs.pr-number }} From d291c2338c3c1feee4f3f26985c0964ec1b7eb9f Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Mon, 23 Oct 2023 08:36:47 -0700 Subject: [PATCH 153/279] Move Docker image to hatch + compile (#3965) --- CHANGES.md | 2 ++ Dockerfile | 12 +++++++----- docs/usage_and_configuration/black_docker_image.md | 2 ++ 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 86e820a6fd9..fe0b2ebb9d8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -23,6 +23,8 @@ +- Change Dockerfile to hatch + compile black (#3965) + ### Parser diff --git a/Dockerfile b/Dockerfile index a9e0ea5081e..bfd9acccb99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,12 +3,14 @@ FROM python:3.11-slim AS builder RUN mkdir /src COPY . /src/ ENV VIRTUAL_ENV=/opt/venv +ENV HATCH_BUILD_HOOKS_ENABLE=1 +# Install build tools to compile black + dependencies +RUN apt update && apt install -y build-essential git python3-dev RUN python -m venv $VIRTUAL_ENV -RUN . /opt/venv/bin/activate && pip install --no-cache-dir --upgrade pip setuptools wheel \ - # Install build tools to compile dependencies that don't have prebuilt wheels - && apt update && apt install -y git build-essential \ - && cd /src \ - && pip install --no-cache-dir .[colorama,d] +RUN python -m pip install --no-cache-dir hatch hatch-fancy-pypi-readme hatch-vcs +RUN . /opt/venv/bin/activate && pip install --no-cache-dir --upgrade pip setuptools \ + && cd /src && hatch build -t wheel \ + && pip install --no-cache-dir dist/*-cp*[colorama,d,uvloop] FROM python:3.11-slim diff --git a/docs/usage_and_configuration/black_docker_image.md b/docs/usage_and_configuration/black_docker_image.md index 85aec91ef1c..c97c25af328 100644 --- a/docs/usage_and_configuration/black_docker_image.md +++ b/docs/usage_and_configuration/black_docker_image.md @@ -24,6 +24,8 @@ created for all unreleased [commits on the `main` branch](https://github.com/psf/black/commits/main). This tag is not meant to be used by external users. +From version 23.11.0 the Docker image installs a compiled black into the image. + ## Usage A permanent container doesn't have to be created to use _Black_ as a Docker image. It's From a7643fac8d97c15640a2b1a79f68b3dc771aebfb Mon Sep 17 00:00:00 2001 From: Dario Curreri <48800335+dariocurr@users.noreply.github.com> Date: Mon, 23 Oct 2023 18:40:09 +0200 Subject: [PATCH 154/279] Add summary parameter to action (#3958) --- CHANGES.md | 3 ++- action.yml | 14 ++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fe0b2ebb9d8..89837c8f545 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -43,7 +43,8 @@ ### Integrations - +- The summary output for GitHub workflows is now suppressible using the `summary` + parameter. (#3958) ### Documentation diff --git a/action.yml b/action.yml index 8b698ae3c80..a22005ac243 100644 --- a/action.yml +++ b/action.yml @@ -27,6 +27,10 @@ inputs: description: 'Python Version specifier (PEP440) - e.g. "21.5b1"' required: false default: "" + summary: + description: "Whether to add the output to the workflow summary" + required: false + default: true branding: color: "black" icon: "check-circle" @@ -47,10 +51,12 @@ runs: # Display the raw output in the step echo "${out}" - # Display the Markdown output in the job summary - echo "\`\`\`python" >> $GITHUB_STEP_SUMMARY - echo "${out}" >> $GITHUB_STEP_SUMMARY - echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + if [ "${{ inputs.summary }}" == "true" ]; then + # Display the Markdown output in the job summary + echo "\`\`\`python" >> $GITHUB_STEP_SUMMARY + echo "${out}" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + fi # Exit with the exit-code returned by Black exit ${exit_code} From c0adca321dc0d97a704de8ed0353e5b894a6a912 Mon Sep 17 00:00:00 2001 From: William Moreno Date: Mon, 23 Oct 2023 11:21:58 -0600 Subject: [PATCH 155/279] docs: specifies the use of the .git-blame-ignore-revs file (#3961) --- docs/guides/introducing_black_to_your_project.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/guides/introducing_black_to_your_project.md b/docs/guides/introducing_black_to_your_project.md index 71a566fbda1..3927eb1a386 100644 --- a/docs/guides/introducing_black_to_your_project.md +++ b/docs/guides/introducing_black_to_your_project.md @@ -18,7 +18,8 @@ previous revision that modified those lines. So when migrating your project's code style to _Black_, reformat everything and commit the changes (preferably in one massive commit). Then put the full 40 characters commit -identifier(s) into a file. +identifier(s) into a file usually called `.git-blame-ignore-revs` at the root of your +project directory. ```text # Migrate code style to Black From 8de4be516879302afce542ac80a6a43ced807759 Mon Sep 17 00:00:00 2001 From: Kiyoon Kim Date: Tue, 24 Oct 2023 02:37:14 +0900 Subject: [PATCH 156/279] Fix CI failing (#3957) * Fix CI failing * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * docs: update CHANGES.md * docs: fix changelog location to unreleased --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- CHANGES.md | 2 ++ action.yml | 3 +++ 2 files changed, 5 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 89837c8f545..7e1ed79cd48 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -46,6 +46,8 @@ - The summary output for GitHub workflows is now suppressible using the `summary` parameter. (#3958) +- Fix the action failing when Black check doesn't pass (#3957) + ### Documentation - -### Stable style - - +- Maintanence release to get a fix out for GitHub Action edge case (#3957) ### Preview style - Fix merging implicit multiline strings that have inline comments (#3956) - Allow empty first line after block open before a comment or compound statement (#3967) -### Configuration - - - ### Packaging - - - Change Dockerfile to hatch + compile black (#3965) -### Parser - - - -### Performance - - - -### Output - - - -### _Blackd_ - - - ### Integrations - The summary output for GitHub workflows is now suppressible using the `summary` parameter. (#3958) - - Fix the action failing when Black check doesn't pass (#3957) ### Documentation - +- It is known Windows documentation CI is broken + https://github.com/psf/black/issues/3968 ## 23.10.0 diff --git a/docs/integrations/source_version_control.md b/docs/integrations/source_version_control.md index 16354f849ba..597a6b993c7 100644 --- a/docs/integrations/source_version_control.md +++ b/docs/integrations/source_version_control.md @@ -8,7 +8,7 @@ Use [pre-commit](https://pre-commit.com/). Once you repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.10.0 + rev: 23.10.1 hooks: - id: black # It is recommended to specify the latest version of Python @@ -35,7 +35,7 @@ include Jupyter Notebooks. To use this hook, simply replace the hook's `id: blac repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.10.0 + rev: 23.10.1 hooks: - id: black-jupyter # It is recommended to specify the latest version of Python diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 5b132a95eae..f25dbb13d4d 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -194,8 +194,8 @@ configuration file for consistent results across environments. ```console $ black --version -black, 23.10.0 (compiled: yes) -$ black --required-version 23.10.0 -c "format = 'this'" +black, 23.10.1 (compiled: yes) +$ black --required-version 23.10.1 -c "format = 'this'" format = "this" $ black --required-version 31.5b2 -c "still = 'beta?!'" Oh no! 💥 💔 💥 The required version does not match the running version! @@ -286,7 +286,7 @@ You can check the version of _Black_ you have installed using the `--version` fl ```console $ black --version -black, 23.10.0 +black, 23.10.1 ``` #### `--config` From ef1048d5f8205cb03358a6a373710c2a71d047b4 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Mon, 23 Oct 2023 23:26:40 -0700 Subject: [PATCH 158/279] Add Unreleased template to CHANGES.md (#3973) Add Unreleased template to CHANGES.md - Did this via tool working on in another branch --- CHANGES.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 1e90c12b4fb..c4ae056b1b9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,52 @@ # Change Log +## Unreleased + +### Highlights + + + +### Stable style + + + +### Preview style + + + +### Configuration + + + +### Packaging + + + +### Parser + + + +### Performance + + + +### Output + + + +### _Blackd_ + + + +### Integrations + + + +### Documentation + + + ## 23.10.1 ### Highlights From 1d4c31aa589dc0c8633af7531f8cc09192917b38 Mon Sep 17 00:00:00 2001 From: Henri Holopainen Date: Wed, 25 Oct 2023 18:35:37 +0300 Subject: [PATCH 159/279] [925] Improve multiline dictionary and list indentation for sole function parameter (#3964) --- CHANGES.md | 3 +- docs/the_black_code_style/future_style.md | 26 ++ src/black/linegen.py | 13 + src/black/mode.py | 1 + ..._parens_with_braces_and_square_brackets.py | 273 ++++++++++++++++++ .../cases/preview_long_strings__regression.py | 22 +- 6 files changed, 325 insertions(+), 13 deletions(-) create mode 100644 tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py diff --git a/CHANGES.md b/CHANGES.md index c4ae056b1b9..f7d02af187d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,7 +12,8 @@ ### Preview style - +- Multiline dictionaries and lists that are the sole argument to a function are now + indented less (#3964) ### Configuration diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index 367ff98537c..e73c16ba26e 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -113,6 +113,32 @@ my_dict = { } ``` +### Improved multiline dictionary and list indentation for sole function parameter + +For better readability and less verticality, _Black_ now pairs parantheses ("(", ")") +with braces ("{", "}") and square brackets ("[", "]") on the same line for single +parameter function calls. For example: + +```python +foo( + [ + 1, + 2, + 3, + ] +) +``` + +will be changed to: + +```python +foo([ + 1, + 2, + 3, +]) +``` + ### Improved multiline string handling _Black_ is smarter when formatting multiline strings, especially in function arguments, diff --git a/src/black/linegen.py b/src/black/linegen.py index 2bfe587fa0e..5f5a69152d5 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -815,6 +815,19 @@ def _first_right_hand_split( tail_leaves.reverse() body_leaves.reverse() head_leaves.reverse() + + if Preview.hug_parens_with_braces_and_square_brackets in line.mode: + if ( + tail_leaves[0].type == token.RPAR + and tail_leaves[0].value + and tail_leaves[0].opening_bracket is head_leaves[-1] + and body_leaves[-1].type in [token.RBRACE, token.RSQB] + and body_leaves[-1].opening_bracket is body_leaves[0] + ): + head_leaves = head_leaves + body_leaves[:1] + tail_leaves = body_leaves[-1:] + tail_leaves + body_leaves = body_leaves[1:-1] + head = bracket_split_build_line( head_leaves, line, opening_bracket, component=_BracketSplitComponent.head ) diff --git a/src/black/mode.py b/src/black/mode.py index 4effeef3e7c..99b2a84a63d 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -190,6 +190,7 @@ class Preview(Enum): module_docstring_newlines = auto() accept_raw_docstrings = auto() fix_power_op_line_length = auto() + hug_parens_with_braces_and_square_brackets = auto() allow_empty_first_line_before_new_block_or_comment = auto() diff --git a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py new file mode 100644 index 00000000000..98ed342fcbc --- /dev/null +++ b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py @@ -0,0 +1,273 @@ +# flags: --preview +def foo_brackets(request): + return JsonResponse( + { + "var_1": foo, + "var_2": bar, + } + ) + +def foo_square_brackets(request): + return JsonResponse( + [ + "var_1", + "var_2", + ] + ) + +func({"a": 37, "b": 42, "c": 927, "aaaaaaaaaaaaaaaaaaaaaaaaa": 11111111111111111111111111111111111111111}) + +func(["random_string_number_one","random_string_number_two","random_string_number_three","random_string_number_four"]) + +func( + { + # expand me + 'a':37, + 'b':42, + 'c':927 + } +) + +func( + [ + 'a', + 'b', + 'c', + ] +) + +func( # a + [ # b + "c", # c + "d", # d + "e", # e + ] # f +) # g + +func( # a + { # b + "c": 1, # c + "d": 2, # d + "e": 3, # e + } # f +) # g + +func( + # preserve me + [ + "c", + "d", + "e", + ] +) + +func( + [ # preserve me but hug brackets + "c", + "d", + "e", + ] +) + +func( + [ + # preserve me but hug brackets + "c", + "d", + "e", + ] +) + +func( + [ + "c", + # preserve me but hug brackets + "d", + "e", + ] +) + +func( + [ + "c", + "d", + "e", + # preserve me but hug brackets + ] +) + +func( + [ + "c", + "d", + "e", + ] # preserve me but hug brackets +) + +func( + [ + "c", + "d", + "e", + ] + # preserve me +) + +func([x for x in "short line"]) +func([x for x in "long line long line long line long line long line long line long line"]) +func([x for x in [x for x in "long line long line long line long line long line long line long line"]]) + +func({"short line"}) +func({"long line", "long long line", "long long long line", "long long long long line", "long long long long long line"}) +func({{"long line", "long long line", "long long long line", "long long long long line", "long long long long long line"}}) + +foooooooooooooooooooo( + [{c: n + 1 for c in range(256)} for n in range(100)] + [{}], {size} +) + +baaaaaaaaaaaaar( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], {x}, "a string", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +) + +# output +def foo_brackets(request): + return JsonResponse({ + "var_1": foo, + "var_2": bar, + }) + + +def foo_square_brackets(request): + return JsonResponse([ + "var_1", + "var_2", + ]) + + +func({ + "a": 37, + "b": 42, + "c": 927, + "aaaaaaaaaaaaaaaaaaaaaaaaa": 11111111111111111111111111111111111111111, +}) + +func([ + "random_string_number_one", + "random_string_number_two", + "random_string_number_three", + "random_string_number_four", +]) + +func({ + # expand me + "a": 37, + "b": 42, + "c": 927, +}) + +func([ + "a", + "b", + "c", +]) + +func([ # a # b + "c", # c + "d", # d + "e", # e +]) # f # g + +func({ # a # b + "c": 1, # c + "d": 2, # d + "e": 3, # e +}) # f # g + +func( + # preserve me + [ + "c", + "d", + "e", + ] +) + +func([ # preserve me but hug brackets + "c", + "d", + "e", +]) + +func([ + # preserve me but hug brackets + "c", + "d", + "e", +]) + +func([ + "c", + # preserve me but hug brackets + "d", + "e", +]) + +func([ + "c", + "d", + "e", + # preserve me but hug brackets +]) + +func([ + "c", + "d", + "e", +]) # preserve me but hug brackets + +func( + [ + "c", + "d", + "e", + ] + # preserve me +) + +func([x for x in "short line"]) +func([ + x for x in "long line long line long line long line long line long line long line" +]) +func([ + x + for x in [ + x + for x in "long line long line long line long line long line long line long line" + ] +]) + +func({"short line"}) +func({ + "long line", + "long long line", + "long long long line", + "long long long long line", + "long long long long long line", +}) +func({ + { + "long line", + "long long line", + "long long long line", + "long long long long line", + "long long long long long line", + } +}) + +foooooooooooooooooooo( + [{c: n + 1 for c in range(256)} for n in range(100)] + [{}], {size} +) + +baaaaaaaaaaaaar( + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], {x}, "a string", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +) diff --git a/tests/data/cases/preview_long_strings__regression.py b/tests/data/cases/preview_long_strings__regression.py index 436157f4e05..313d898cd83 100644 --- a/tests/data/cases/preview_long_strings__regression.py +++ b/tests/data/cases/preview_long_strings__regression.py @@ -962,19 +962,17 @@ def who(self): ) -xxxxxxx_xxxxxx_xxxxxxx = xxx( - [ - xxxxxxxxxxxx( - xxxxxx_xxxxxxx=( - '((x.aaaaaaaaa = "xxxxxx.xxxxxxxxxxxxxxxxxxxxx") || (x.xxxxxxxxx =' - ' "xxxxxxxxxxxx")) && ' - # xxxxx xxxxxxxxxxxx xxxx xxx (xxxxxxxxxxxxxxxx) xx x xxxxxxxxx xx xxxxxx. - "(x.bbbbbbbbbbbb.xxx != " - '"xxx:xxx:xxx::cccccccccccc:xxxxxxx-xxxx/xxxxxxxxxxx/xxxxxxxxxxxxxxxxx") && ' - ) +xxxxxxx_xxxxxx_xxxxxxx = xxx([ + xxxxxxxxxxxx( + xxxxxx_xxxxxxx=( + '((x.aaaaaaaaa = "xxxxxx.xxxxxxxxxxxxxxxxxxxxx") || (x.xxxxxxxxx =' + ' "xxxxxxxxxxxx")) && ' + # xxxxx xxxxxxxxxxxx xxxx xxx (xxxxxxxxxxxxxxxx) xx x xxxxxxxxx xx xxxxxx. + "(x.bbbbbbbbbbbb.xxx != " + '"xxx:xxx:xxx::cccccccccccc:xxxxxxx-xxxx/xxxxxxxxxxx/xxxxxxxxxxxxxxxxx") && ' ) - ] -) + ) +]) if __name__ == "__main__": for i in range(4, 8): From 878937bcc3282319081057e2f1dbee5e24d69d68 Mon Sep 17 00:00:00 2001 From: Henri Holopainen Date: Wed, 25 Oct 2023 19:47:21 +0300 Subject: [PATCH 160/279] [2213] Add support for single line format skip with other comments on the same line (#3959) --- CHANGES.md | 2 +- docs/the_black_code_style/current_style.md | 12 ++-- src/black/__init__.py | 2 +- src/black/comments.py | 63 +++++++++++++++---- src/black/mode.py | 1 + ...line_format_skip_with_multiple_comments.py | 20 ++++++ 6 files changed, 81 insertions(+), 19 deletions(-) create mode 100644 tests/data/cases/preview_single_line_format_skip_with_multiple_comments.py diff --git a/CHANGES.md b/CHANGES.md index f7d02af187d..c96186c93cc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,7 +17,7 @@ ### Configuration - +- Add support for single line format skip with other comments on the same line (#3959) ### Packaging diff --git a/docs/the_black_code_style/current_style.md b/docs/the_black_code_style/current_style.md index ff757a8276b..f59c1853f72 100644 --- a/docs/the_black_code_style/current_style.md +++ b/docs/the_black_code_style/current_style.md @@ -8,12 +8,14 @@ deliberately limited and rarely added. Previous formatting is taken into account little as possible, with rare exceptions like the magic trailing comma. The coding style used by _Black_ can be viewed as a strict subset of PEP 8. -_Black_ reformats entire files in place. It doesn't reformat lines that end with +_Black_ reformats entire files in place. It doesn't reformat lines that contain `# fmt: skip` or blocks that start with `# fmt: off` and end with `# fmt: on`. -`# fmt: on/off` must be on the same level of indentation and in the same block, meaning -no unindents beyond the initial indentation level between them. It also recognizes -[YAPF](https://github.com/google/yapf)'s block comments to the same effect, as a -courtesy for straddling code. +`# fmt: skip` can be mixed with other pragmas/comments either with multiple comments +(e.g. `# fmt: skip # pylint # noqa`) or as a semicolon separeted list (e.g. +`# fmt: skip; pylint; noqa`). `# fmt: on/off` must be on the same level of indentation +and in the same block, meaning no unindents beyond the initial indentation level between +them. It also recognizes [YAPF](https://github.com/google/yapf)'s block comments to the +same effect, as a courtesy for straddling code. The rest of this document describes the current formatting style. If you're interested in trying out where the style is heading, see [future style](./future_style.md) and try diff --git a/src/black/__init__.py b/src/black/__init__.py index 188a4f79f0e..7cf93b89e42 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1099,7 +1099,7 @@ def _format_str_once(src_contents: str, *, mode: Mode) -> str: for feature in {Feature.PARENTHESIZED_CONTEXT_MANAGERS} if supports_feature(versions, feature) } - normalize_fmt_off(src_node) + normalize_fmt_off(src_node, mode) lines = LineGenerator(mode=mode, features=context_manager_features) elt = EmptyLineTracker(mode=mode) split_line_features = { diff --git a/src/black/comments.py b/src/black/comments.py index 226968bff98..862fc7607cc 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -3,6 +3,7 @@ from functools import lru_cache from typing import Final, Iterator, List, Optional, Union +from black.mode import Mode, Preview from black.nodes import ( CLOSING_BRACKETS, STANDALONE_COMMENT, @@ -20,10 +21,11 @@ FMT_OFF: Final = {"# fmt: off", "# fmt:off", "# yapf: disable"} FMT_SKIP: Final = {"# fmt: skip", "# fmt:skip"} -FMT_PASS: Final = {*FMT_OFF, *FMT_SKIP} FMT_ON: Final = {"# fmt: on", "# fmt:on", "# yapf: enable"} COMMENT_EXCEPTIONS = " !:#'" +_COMMENT_PREFIX = "# " +_COMMENT_LIST_SEPARATOR = ";" @dataclass @@ -130,14 +132,14 @@ def make_comment(content: str) -> str: return "#" + content -def normalize_fmt_off(node: Node) -> None: +def normalize_fmt_off(node: Node, mode: Mode) -> None: """Convert content between `# fmt: off`/`# fmt: on` into standalone comments.""" try_again = True while try_again: - try_again = convert_one_fmt_off_pair(node) + try_again = convert_one_fmt_off_pair(node, mode) -def convert_one_fmt_off_pair(node: Node) -> bool: +def convert_one_fmt_off_pair(node: Node, mode: Mode) -> bool: """Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment. Returns True if a pair was converted. @@ -145,21 +147,27 @@ def convert_one_fmt_off_pair(node: Node) -> bool: for leaf in node.leaves(): previous_consumed = 0 for comment in list_comments(leaf.prefix, is_endmarker=False): - if comment.value not in FMT_PASS: + should_pass_fmt = comment.value in FMT_OFF or _contains_fmt_skip_comment( + comment.value, mode + ) + if not should_pass_fmt: previous_consumed = comment.consumed continue # We only want standalone comments. If there's no previous leaf or # the previous leaf is indentation, it's a standalone comment in # disguise. - if comment.value in FMT_PASS and comment.type != STANDALONE_COMMENT: + if should_pass_fmt and comment.type != STANDALONE_COMMENT: prev = preceding_leaf(leaf) if prev: if comment.value in FMT_OFF and prev.type not in WHITESPACE: continue - if comment.value in FMT_SKIP and prev.type in WHITESPACE: + if ( + _contains_fmt_skip_comment(comment.value, mode) + and prev.type in WHITESPACE + ): continue - ignored_nodes = list(generate_ignored_nodes(leaf, comment)) + ignored_nodes = list(generate_ignored_nodes(leaf, comment, mode)) if not ignored_nodes: continue @@ -168,7 +176,7 @@ def convert_one_fmt_off_pair(node: Node) -> bool: prefix = first.prefix if comment.value in FMT_OFF: first.prefix = prefix[comment.consumed :] - if comment.value in FMT_SKIP: + if _contains_fmt_skip_comment(comment.value, mode): first.prefix = "" standalone_comment_prefix = prefix else: @@ -178,7 +186,7 @@ def convert_one_fmt_off_pair(node: Node) -> bool: hidden_value = "".join(str(n) for n in ignored_nodes) if comment.value in FMT_OFF: hidden_value = comment.value + "\n" + hidden_value - if comment.value in FMT_SKIP: + if _contains_fmt_skip_comment(comment.value, mode): hidden_value += " " + comment.value if hidden_value.endswith("\n"): # That happens when one of the `ignored_nodes` ended with a NEWLINE @@ -205,13 +213,15 @@ def convert_one_fmt_off_pair(node: Node) -> bool: return False -def generate_ignored_nodes(leaf: Leaf, comment: ProtoComment) -> Iterator[LN]: +def generate_ignored_nodes( + leaf: Leaf, comment: ProtoComment, mode: Mode +) -> Iterator[LN]: """Starting from the container of `leaf`, generate all leaves until `# fmt: on`. If comment is skip, returns leaf only. Stops at the end of the block. """ - if comment.value in FMT_SKIP: + if _contains_fmt_skip_comment(comment.value, mode): yield from _generate_ignored_nodes_from_fmt_skip(leaf, comment) return container: Optional[LN] = container_of(leaf) @@ -327,3 +337,32 @@ def contains_pragma_comment(comment_list: List[Leaf]) -> bool: return True return False + + +def _contains_fmt_skip_comment(comment_line: str, mode: Mode) -> bool: + """ + Checks if the given comment contains FMT_SKIP alone or paired with other comments. + Matching styles: + # fmt:skip <-- single comment + # noqa:XXX # fmt:skip # a nice line <-- multiple comments (Preview) + # pylint:XXX; fmt:skip <-- list of comments (; separated, Preview) + """ + semantic_comment_blocks = ( + [ + comment_line, + *[ + _COMMENT_PREFIX + comment.strip() + for comment in comment_line.split(_COMMENT_PREFIX)[1:] + ], + *[ + _COMMENT_PREFIX + comment.strip() + for comment in comment_line.strip(_COMMENT_PREFIX).split( + _COMMENT_LIST_SEPARATOR + ) + ], + ] + if Preview.single_line_format_skip_with_multiple_comments in mode + else [comment_line] + ) + + return any(comment in FMT_SKIP for comment in semantic_comment_blocks) diff --git a/src/black/mode.py b/src/black/mode.py index 99b2a84a63d..4e4effffb86 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -192,6 +192,7 @@ class Preview(Enum): fix_power_op_line_length = auto() hug_parens_with_braces_and_square_brackets = auto() allow_empty_first_line_before_new_block_or_comment = auto() + single_line_format_skip_with_multiple_comments = auto() class Deprecated(UserWarning): diff --git a/tests/data/cases/preview_single_line_format_skip_with_multiple_comments.py b/tests/data/cases/preview_single_line_format_skip_with_multiple_comments.py new file mode 100644 index 00000000000..efde662baa8 --- /dev/null +++ b/tests/data/cases/preview_single_line_format_skip_with_multiple_comments.py @@ -0,0 +1,20 @@ +# flags: --preview +foo = 123 # fmt: skip # noqa: E501 # pylint +bar = ( + 123 , + ( 1 + 5 ) # pylint # fmt:skip +) +baz = "a" + "b" # pylint; fmt: skip; noqa: E501 +skip_will_not_work = "a" + "b" # pylint fmt:skip +skip_will_not_work2 = "a" + "b" # some text; fmt:skip happens to be part of it + +# output + +foo = 123 # fmt: skip # noqa: E501 # pylint +bar = ( + 123 , + ( 1 + 5 ) # pylint # fmt:skip +) +baz = "a" + "b" # pylint; fmt: skip; noqa: E501 +skip_will_not_work = "a" + "b" # pylint fmt:skip +skip_will_not_work2 = "a" + "b" # some text; fmt:skip happens to be part of it From f7174bfc431e22f38b502579d1234989c3c5ce15 Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Fri, 27 Oct 2023 01:43:42 +0900 Subject: [PATCH 161/279] Fix typo in future_style.md (#3979) parantheses -> parentheses --- docs/the_black_code_style/future_style.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index e73c16ba26e..f2534b0f0d0 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -115,7 +115,7 @@ my_dict = { ### Improved multiline dictionary and list indentation for sole function parameter -For better readability and less verticality, _Black_ now pairs parantheses ("(", ")") +For better readability and less verticality, _Black_ now pairs parentheses ("(", ")") with braces ("{", "}") and square brackets ("[", "]") on the same line for single parameter function calls. For example: From de701fe6aa0d61526b806dd31610da5cf8b67ab9 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 26 Oct 2023 21:13:25 -0700 Subject: [PATCH 162/279] Fix CI by running on Python 3.11 (#3984) aiohttp doesn't yet support 3.12 --- .github/workflows/diff_shades.yml | 4 ++-- .github/workflows/doc.yml | 2 +- .github/workflows/lint.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/diff_shades.yml b/.github/workflows/diff_shades.yml index 97db907abc8..6bfc6ca9ed8 100644 --- a/.github/workflows/diff_shades.yml +++ b/.github/workflows/diff_shades.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: - python-version: "*" + python-version: "3.11" - name: Install diff-shades and support dependencies run: | @@ -59,7 +59,7 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: "*" + python-version: "3.11" - name: Install diff-shades and support dependencies run: | diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index fa3d87c70f5..9a23e19cadd 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -26,7 +26,7 @@ jobs: - name: Set up latest Python uses: actions/setup-python@v4 with: - python-version: "*" + python-version: "3.11" - name: Install dependencies run: | diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 3eaf5785f5a..7fe1b04eb02 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -26,7 +26,7 @@ jobs: - name: Set up latest Python uses: actions/setup-python@v4 with: - python-version: "*" + python-version: "3.11" - name: Install dependencies run: | From 7bfa35cca88a2a6b875fb8564c19164143a46f1d Mon Sep 17 00:00:00 2001 From: Surav Shrestha <148626286+shresthasurav@users.noreply.github.com> Date: Fri, 27 Oct 2023 10:11:47 +0545 Subject: [PATCH 163/279] docs: fix typos in change log and documentations (#3985) --- CHANGES.md | 2 +- docs/the_black_code_style/current_style.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c96186c93cc..7703223a119 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -52,7 +52,7 @@ ### Highlights -- Maintanence release to get a fix out for GitHub Action edge case (#3957) +- Maintenance release to get a fix out for GitHub Action edge case (#3957) ### Preview style diff --git a/docs/the_black_code_style/current_style.md b/docs/the_black_code_style/current_style.md index f59c1853f72..431bae525f6 100644 --- a/docs/the_black_code_style/current_style.md +++ b/docs/the_black_code_style/current_style.md @@ -11,7 +11,7 @@ used by _Black_ can be viewed as a strict subset of PEP 8. _Black_ reformats entire files in place. It doesn't reformat lines that contain `# fmt: skip` or blocks that start with `# fmt: off` and end with `# fmt: on`. `# fmt: skip` can be mixed with other pragmas/comments either with multiple comments -(e.g. `# fmt: skip # pylint # noqa`) or as a semicolon separeted list (e.g. +(e.g. `# fmt: skip # pylint # noqa`) or as a semicolon separated list (e.g. `# fmt: skip; pylint; noqa`). `# fmt: on/off` must be on the same level of indentation and in the same block, meaning no unindents beyond the initial indentation level between them. It also recognizes [YAPF](https://github.com/google/yapf)'s block comments to the From c369e446f9dbff313ebb555bf461b4e7778ca78d Mon Sep 17 00:00:00 2001 From: sth Date: Fri, 27 Oct 2023 09:43:51 +0200 Subject: [PATCH 164/279] Fix matching of absolute paths in `--include` (#3976) --- CHANGES.md | 2 ++ src/black/files.py | 2 +- tests/test_black.py | 59 +++++++++++++++++++++++++++++++++++++++------ 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 7703223a119..71f62d0e11f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,6 +19,8 @@ - Add support for single line format skip with other comments on the same line (#3959) +- Fix a bug in the matching of absolute path names in `--include` (#3976) + ### Packaging diff --git a/src/black/files.py b/src/black/files.py index 362898dc0fd..1eed7eda828 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -389,7 +389,7 @@ def gen_python_files( warn=verbose or not quiet ): continue - include_match = include.search(normalized_path) if include else True + include_match = include.search(root_relative_path) if include else True if include_match: yield child diff --git a/tests/test_black.py b/tests/test_black.py index 537ca80d432..56c20243020 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -2388,6 +2388,27 @@ def test_empty_include(self) -> None: # Setting exclude explicitly to an empty string to block .gitignore usage. assert_collected_sources(src, expected, include="", exclude="") + def test_include_absolute_path(self) -> None: + path = DATA_DIR / "include_exclude_tests" + src = [path] + expected = [ + Path(path / "b/dont_exclude/a.pie"), + ] + assert_collected_sources( + src, expected, root=path, include=r"^/b/dont_exclude/a\.pie$", exclude="" + ) + + def test_exclude_absolute_path(self) -> None: + path = DATA_DIR / "include_exclude_tests" + src = [path] + expected = [ + Path(path / "b/dont_exclude/a.py"), + Path(path / "b/.definitely_exclude/a.py"), + ] + assert_collected_sources( + src, expected, root=path, include=r"\.py$", exclude=r"^/b/exclude/a\.py$" + ) + def test_extend_exclude(self) -> None: path = DATA_DIR / "include_exclude_tests" src = [path] @@ -2401,7 +2422,6 @@ def test_extend_exclude(self) -> None: @pytest.mark.incompatible_with_mypyc def test_symlinks(self) -> None: - path = MagicMock() root = THIS_DIR.resolve() include = re.compile(black.DEFAULT_INCLUDES) exclude = re.compile(black.DEFAULT_EXCLUDES) @@ -2409,19 +2429,44 @@ def test_symlinks(self) -> None: gitignore = PathSpec.from_lines("gitwildmatch", []) regular = MagicMock() - outside_root_symlink = MagicMock() - ignored_symlink = MagicMock() - - path.iterdir.return_value = [regular, outside_root_symlink, ignored_symlink] - regular.absolute.return_value = root / "regular.py" regular.resolve.return_value = root / "regular.py" regular.is_dir.return_value = False + regular.is_file.return_value = True + outside_root_symlink = MagicMock() outside_root_symlink.absolute.return_value = root / "symlink.py" outside_root_symlink.resolve.return_value = Path("/nowhere") + outside_root_symlink.is_dir.return_value = False + outside_root_symlink.is_file.return_value = True + ignored_symlink = MagicMock() ignored_symlink.absolute.return_value = root / ".mypy_cache" / "symlink.py" + ignored_symlink.is_dir.return_value = False + ignored_symlink.is_file.return_value = True + + # A symlink that has an excluded name, but points to an included name + symlink_excluded_name = MagicMock() + symlink_excluded_name.absolute.return_value = root / "excluded_name" + symlink_excluded_name.resolve.return_value = root / "included_name.py" + symlink_excluded_name.is_dir.return_value = False + symlink_excluded_name.is_file.return_value = True + + # A symlink that has an included name, but points to an excluded name + symlink_included_name = MagicMock() + symlink_included_name.absolute.return_value = root / "included_name.py" + symlink_included_name.resolve.return_value = root / "excluded_name" + symlink_included_name.is_dir.return_value = False + symlink_included_name.is_file.return_value = True + + path = MagicMock() + path.iterdir.return_value = [ + regular, + outside_root_symlink, + ignored_symlink, + symlink_excluded_name, + symlink_included_name, + ] files = list( black.gen_python_files( @@ -2437,7 +2482,7 @@ def test_symlinks(self) -> None: quiet=False, ) ) - assert files == [regular] + assert files == [regular, symlink_included_name] path.iterdir.assert_called_once() outside_root_symlink.resolve.assert_called_once() From caef19689b153f3a7baea1764a5adccae8bf1f1e Mon Sep 17 00:00:00 2001 From: Gabriel Perren Date: Fri, 27 Oct 2023 15:54:31 -0300 Subject: [PATCH 165/279] Update current_style.md (#3990) Fix small typo --- docs/the_black_code_style/current_style.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/the_black_code_style/current_style.md b/docs/the_black_code_style/current_style.md index 431bae525f6..2a5e10162f2 100644 --- a/docs/the_black_code_style/current_style.md +++ b/docs/the_black_code_style/current_style.md @@ -178,7 +178,7 @@ If you use Flake8, you have a few options: extend-ignore = E203, E501, E704 ``` - The rationale for E950 is explained in + The rationale for B950 is explained in [Bugbear's documentation](https://github.com/PyCQA/flake8-bugbear#opinionated-warnings). 2. For a minimally compatible config: From c712d57ca9e30ba0db61c2fd7e4a2bf67f58bcc2 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 27 Oct 2023 12:17:54 -0700 Subject: [PATCH 166/279] Add trailing comma test case for hugging parens (#3991) --- docs/the_black_code_style/future_style.md | 13 +++++++++++++ ...hug_parens_with_braces_and_square_brackets.py | 16 ++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index f2534b0f0d0..c744902577d 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -139,6 +139,19 @@ foo([ ]) ``` +You can use a magic trailing comma to avoid this compacting behavior; by default, +_Black_ will not reformat the following code: + +```python +foo( + [ + 1, + 2, + 3, + ], +) +``` + ### Improved multiline string handling _Black_ is smarter when formatting multiline strings, especially in function arguments, diff --git a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py index 98ed342fcbc..6d10518133c 100644 --- a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py +++ b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py @@ -36,6 +36,14 @@ def foo_square_brackets(request): ] ) +func( + [ + 'a', + 'b', + 'c', + ], +) + func( # a [ # b "c", # c @@ -171,6 +179,14 @@ def foo_square_brackets(request): "c", ]) +func( + [ + "a", + "b", + "c", + ], +) + func([ # a # b "c", # c "d", # d From 53c4278a4c9b81baa86630ffda5f680f33968d1e Mon Sep 17 00:00:00 2001 From: Satyam Namdev <111422209+Spyrosigma@users.noreply.github.com> Date: Sat, 28 Oct 2023 01:57:19 +0530 Subject: [PATCH 167/279] Update CHANGES.md (#3988) Fixed a grammatical mistake --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 71f62d0e11f..84d9061135a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,7 +17,7 @@ ### Configuration -- Add support for single line format skip with other comments on the same line (#3959) +- Add support for single-line format skip with other comments on the same line (#3959) - Fix a bug in the matching of absolute path names in `--include` (#3976) From 7686989fc89aad5ea235a34977ebf8c81c26c4eb Mon Sep 17 00:00:00 2001 From: David Culley <6276049+davidculley@users.noreply.github.com> Date: Sat, 28 Oct 2023 00:43:34 +0200 Subject: [PATCH 168/279] confine pre-commit to stages (#3940) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://pre-commit.com/#confining-hooks-to-run-at-certain-stages > If you are authoring a tool, it is usually a good idea to provide an appropriate `stages` property. For example a reasonable setting for a linter or code formatter would be `stages: [pre-commit, pre-merge-commit, pre-push, manual]`. Co-authored-by: Jelle Zijlstra --- .pre-commit-hooks.yaml | 2 ++ CHANGES.md | 3 +++ 2 files changed, 5 insertions(+) diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index a1ff41fded8..54a03efe7a1 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -4,6 +4,7 @@ name: black description: "Black: The uncompromising Python code formatter" entry: black + stages: [pre-commit, pre-merge-commit, pre-push, manual] language: python minimum_pre_commit_version: 2.9.2 require_serial: true @@ -13,6 +14,7 @@ description: "Black: The uncompromising Python code formatter (with Jupyter Notebook support)" entry: black + stages: [pre-commit, pre-merge-commit, pre-push, manual] language: python minimum_pre_commit_version: 2.9.2 require_serial: true diff --git a/CHANGES.md b/CHANGES.md index 84d9061135a..60231468bdf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -45,6 +45,9 @@ +- Black's pre-commit integration will now run only on git hooks appropriate for a code + formatter (#3940) + ### Documentation in CHANGES.md to delete ... - Update ci to run out of scripts dir too - Update test_tuple_calver --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Jelle Zijlstra --- .github/workflows/release_tests.yml | 56 ++++++ docs/contributing/release_process.md | 70 ++------ scripts/release.py | 243 +++++++++++++++++++++++++++ scripts/release_tests.py | 69 ++++++++ 4 files changed, 383 insertions(+), 55 deletions(-) create mode 100644 .github/workflows/release_tests.yml create mode 100755 scripts/release.py create mode 100644 scripts/release_tests.py diff --git a/.github/workflows/release_tests.yml b/.github/workflows/release_tests.yml new file mode 100644 index 00000000000..74729445052 --- /dev/null +++ b/.github/workflows/release_tests.yml @@ -0,0 +1,56 @@ +name: Release tool CI + +on: + push: + paths: + - .github/workflows/release_tests.yml + - release.py + - release_tests.py + pull_request: + paths: + - .github/workflows/release_tests.yml + - release.py + - release_tests.py + +jobs: + build: + # We want to run on external PRs, but not on our own internal PRs as they'll be run + # by the push to the branch. Without this if check, checks are duplicated since + # internal PRs match both the push and pull_request events. + if: + github.event_name == 'push' || github.event.pull_request.head.repo.full_name != + github.repository + + name: Running python ${{ matrix.python-version }} on ${{matrix.os}} + runs-on: ${{ matrix.os }} + strategy: + matrix: + python-version: ["3.12"] + os: [macOS-latest, ubuntu-latest, windows-latest] + + steps: + - uses: actions/checkout@v4 + with: + # Give us all history, branches and tags + fetch-depth: 0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + + - name: Print Python Version + run: python --version --version && which python + + - name: Print Git Version + run: git --version && which git + + - name: Update pip, setuptools + wheels + run: | + python -m pip install --upgrade pip setuptools wheel + + - name: Run unit tests via coverage + print report + run: | + python -m pip install coverage + coverage run scripts/release_tests.py + coverage report --show-missing diff --git a/docs/contributing/release_process.md b/docs/contributing/release_process.md index 02865d6f4bd..c66ffae8ace 100644 --- a/docs/contributing/release_process.md +++ b/docs/contributing/release_process.md @@ -32,21 +32,29 @@ The 10,000 foot view of the release process is that you prepare a release PR and publish a [GitHub Release]. This triggers [release automation](#release-workflows) that builds all release artifacts and publishes them to the various platforms we publish to. +We now have a `scripts/release.py` script to help with cutting the release PRs. + +- `python3 scripts/release.py --help` is your friend. + - `release.py` has only been tested in Python 3.12 (so get with the times :D) + To cut a release: 1. Determine the release's version number - **_Black_ follows the [CalVer] versioning standard using the `YY.M.N` format** - So unless there already has been a release during this month, `N` should be `0` - Example: the first release in January, 2022 → `22.1.0` + - `release.py` will calculate this and log to stderr for you copy paste pleasure 1. File a PR editing `CHANGES.md` and the docs to version the latest changes + - Run `python3 scripts/release.py [--debug]` to generate most changes + - Sub headings in the template, if they have no bullet points need manual removal + _PR welcome to improve :D_ +1. If `release.py` fail manually edit; otherwise, yay, skip this step! 1. Replace the `## Unreleased` header with the version number 1. Remove any empty sections for the current release 1. (_optional_) Read through and copy-edit the changelog (eg. by moving entries, fixing typos, or rephrasing entries) 1. Double-check that no changelog entries since the last release were put in the wrong section (e.g., run `git diff CHANGES.md`) - 1. Add a new empty template for the next release above - ([template below](#changelog-template)) 1. Update references to the latest version in {doc}`/integrations/source_version_control` and {doc}`/usage_and_configuration/the_basics` @@ -63,6 +71,11 @@ To cut a release: description box 1. Publish the GitHub Release, triggering [release automation](#release-workflows) that will handle the rest +1. Once CI is done add + commit (git push - No review) a new empty template for the next + release to CHANGES.md _(Template is able to be copy pasted from release.py should we + fail)_ + 1. `python3 scripts/release.py --add-changes-template|-a [--debug]` + 1. Should that fail, please return to copy + paste 1. At this point, you're basically done. It's good practice to go and [watch and verify that all the release workflows pass][black-actions], although you will receive a GitHub notification should something fail. @@ -81,59 +94,6 @@ release is probably unnecessary. In the end, use your best judgement and ask other maintainers for their thoughts. ``` -### Changelog template - -Use the following template for a clean changelog after the release: - -``` -## Unreleased - -### Highlights - - - -### Stable style - - - -### Preview style - - - -### Configuration - - - -### Packaging - - - -### Parser - - - -### Performance - - - -### Output - - - -### _Blackd_ - - - -### Integrations - - - -### Documentation - - -``` - ## Release workflows All of _Black_'s release automation uses [GitHub Actions]. All workflows are therefore diff --git a/scripts/release.py b/scripts/release.py new file mode 100755 index 00000000000..d588429c2d3 --- /dev/null +++ b/scripts/release.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +""" +Tool to help automate changes needed in commits during and after releases +""" + +import argparse +import logging +import sys +from datetime import datetime +from pathlib import Path +from subprocess import PIPE, run +from typing import List + +LOG = logging.getLogger(__name__) +NEW_VERSION_CHANGELOG_TEMPLATE = """\ +## Unreleased + +### Highlights + + + +### Stable style + + + +### Preview style + + + +### Configuration + + + +### Packaging + + + +### Parser + + + +### Performance + + + +### Output + + + +### _Blackd_ + + + +### Integrations + + + +### Documentation + + +""" + + +class NoGitTagsError(Exception): ... # noqa: E701,E761 + + +# TODO: Do better with alpha + beta releases +# Maybe we vendor packaging library +def get_git_tags(versions_only: bool = True) -> List[str]: + """Pull out all tags or calvers only""" + cp = run(["git", "tag"], stdout=PIPE, stderr=PIPE, check=True, encoding="utf8") + if not cp.stdout: + LOG.error(f"Returned no git tags stdout: {cp.stderr}") + raise NoGitTagsError + git_tags = cp.stdout.splitlines() + if versions_only: + return [t for t in git_tags if t[0].isdigit()] + return git_tags + + +# TODO: Support sorting alhpa/beta releases correctly +def tuple_calver(calver: str) -> tuple[int, ...]: # mypy can't notice maxsplit below + """Convert a calver string into a tuple of ints for sorting""" + try: + return tuple(map(int, calver.split(".", maxsplit=2))) + except ValueError: + return (0, 0, 0) + + +class SourceFiles: + def __init__(self, black_repo_dir: Path): + # File path fun all pathlib to be platform agnostic + self.black_repo_path = black_repo_dir + self.changes_path = self.black_repo_path / "CHANGES.md" + self.docs_path = self.black_repo_path / "docs" + self.version_doc_paths = ( + self.docs_path / "integrations" / "source_version_control.md", + self.docs_path / "usage_and_configuration" / "the_basics.md", + ) + self.current_version = self.get_current_version() + self.next_version = self.get_next_version() + + def __str__(self) -> str: + return f"""\ +> SourceFiles ENV: + Repo path: {self.black_repo_path} + CHANGES.md path: {self.changes_path} + docs path: {self.docs_path} + Current version: {self.current_version} + Next version: {self.next_version} +""" + + def add_template_to_changes(self) -> int: + """Add the template to CHANGES.md if it does not exist""" + LOG.info(f"Adding template to {self.changes_path}") + + with self.changes_path.open("r") as cfp: + changes_string = cfp.read() + + if "## Unreleased" in changes_string: + LOG.error(f"{self.changes_path} already has unreleased template") + return 1 + + templated_changes_string = changes_string.replace( + "# Change Log\n", + f"# Change Log\n\n{NEW_VERSION_CHANGELOG_TEMPLATE}", + ) + + with self.changes_path.open("w") as cfp: + cfp.write(templated_changes_string) + + LOG.info(f"Added template to {self.changes_path}") + return 0 + + def cleanup_changes_template_for_release(self) -> None: + LOG.info(f"Cleaning up {self.changes_path}") + + with self.changes_path.open("r") as cfp: + changes_string = cfp.read() + + # Change Unreleased to next version + versioned_changes = changes_string.replace( + "## Unreleased", f"## {self.next_version}" + ) + + # Remove all comments (subheadings are harder - Human required still) + no_comments_changes = [] + for line in versioned_changes.splitlines(): + if line.startswith(""): + continue + no_comments_changes.append(line) + + with self.changes_path.open("w") as cfp: + cfp.write("\n".join(no_comments_changes) + "\n") + + LOG.debug(f"Finished Cleaning up {self.changes_path}") + + def get_current_version(self) -> str: + """Get the latest git (version) tag as latest version""" + return sorted(get_git_tags(), key=lambda k: tuple_calver(k))[-1] + + def get_next_version(self) -> str: + """Workout the year and month + version number we need to move to""" + base_calver = datetime.today().strftime("%y.%m") + calver_parts = base_calver.split(".") + base_calver = f"{calver_parts[0]}.{int(calver_parts[1])}" # Remove leading 0 + git_tags = get_git_tags() + same_month_releases = [t for t in git_tags if t.startswith(base_calver)] + if len(same_month_releases) < 1: + return f"{base_calver}.0" + same_month_version = same_month_releases[-1].split(".", 2)[-1] + return f"{base_calver}.{int(same_month_version) + 1}" + + def update_repo_for_release(self) -> int: + """Update CHANGES.md + doc files ready for release""" + self.cleanup_changes_template_for_release() + self.update_version_in_docs() + return 0 # return 0 if no exceptions hit + + def update_version_in_docs(self) -> None: + for doc_path in self.version_doc_paths: + LOG.info(f"Updating black version to {self.next_version} in {doc_path}") + + with doc_path.open("r") as dfp: + doc_string = dfp.read() + + next_version_doc = doc_string.replace( + self.current_version, self.next_version + ) + + with doc_path.open("w") as dfp: + dfp.write(next_version_doc) + + LOG.debug( + f"Finished updating black version to {self.next_version} in {doc_path}" + ) + + +def _handle_debug(debug: bool) -> None: + """Turn on debugging if asked otherwise INFO default""" + log_level = logging.DEBUG if debug else logging.INFO + logging.basicConfig( + format="[%(asctime)s] %(levelname)s: %(message)s (%(filename)s:%(lineno)d)", + level=log_level, + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument( + "-a", + "--add-changes-template", + action="store_true", + help="Add the Unreleased template to CHANGES.md", + ) + parser.add_argument( + "-d", "--debug", action="store_true", help="Verbose debug output" + ) + args = parser.parse_args() + _handle_debug(args.debug) + return args + + +def main() -> int: + args = parse_args() + + # Need parent.parent cause script is in scripts/ directory + sf = SourceFiles(Path(__file__).parent.parent) + + if args.add_changes_template: + return sf.add_template_to_changes() + + LOG.info(f"Current version detected to be {sf.current_version}") + LOG.info(f"Next version will be {sf.next_version}") + return sf.update_repo_for_release() + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/scripts/release_tests.py b/scripts/release_tests.py new file mode 100644 index 00000000000..bd72cb4b48a --- /dev/null +++ b/scripts/release_tests.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 + +import unittest +from pathlib import Path +from shutil import rmtree +from tempfile import TemporaryDirectory +from typing import Any +from unittest.mock import Mock, patch + +from release import SourceFiles, tuple_calver # type: ignore + + +class FakeDateTime: + """Used to mock the date to test generating next calver function""" + + def today(*args: Any, **kwargs: Any) -> "FakeDateTime": # noqa + return FakeDateTime() + + # Add leading 0 on purpose to ensure we remove it + def strftime(*args: Any, **kwargs: Any) -> str: # noqa + return "69.01" + + +class TestRelease(unittest.TestCase): + def setUp(self) -> None: + # We only test on >= 3.12 + self.tempdir = TemporaryDirectory(delete=False) # type: ignore + self.tempdir_path = Path(self.tempdir.name) + self.sf = SourceFiles(self.tempdir_path) + + def tearDown(self) -> None: + rmtree(self.tempdir.name) + return super().tearDown() + + @patch("release.get_git_tags") + def test_get_current_version(self, mocked_git_tags: Mock) -> None: + mocked_git_tags.return_value = ["1.1.0", "69.1.0", "69.1.1", "2.2.0"] + self.assertEqual("69.1.1", self.sf.get_current_version()) + + @patch("release.get_git_tags") + @patch("release.datetime", FakeDateTime) + def test_get_next_version(self, mocked_git_tags: Mock) -> None: + # test we handle no args + mocked_git_tags.return_value = [] + self.assertEqual( + "69.1.0", + self.sf.get_next_version(), + "Unable to get correct next version with no git tags", + ) + + # test we handle + mocked_git_tags.return_value = ["1.1.0", "69.1.0", "69.1.1", "2.2.0"] + self.assertEqual( + "69.1.2", + self.sf.get_next_version(), + "Unable to get correct version with 2 previous versions released this" + " month", + ) + + def test_tuple_calver(self) -> None: + first_month_release = tuple_calver("69.1.0") + second_month_release = tuple_calver("69.1.1") + self.assertEqual((69, 1, 0), first_month_release) + self.assertEqual((0, 0, 0), tuple_calver("69.1.1a0")) # Hack for alphas/betas + self.assertTrue(first_month_release < second_month_release) + + +if __name__ == "__main__": + unittest.main() From ddfecf06c13dd86205c851e340124e325ed82c5c Mon Sep 17 00:00:00 2001 From: Henri Holopainen Date: Mon, 30 Oct 2023 17:35:26 +0200 Subject: [PATCH 173/279] Hug parens also with multiline unpacking (#3992) --- CHANGES.md | 2 ++ docs/the_black_code_style/future_style.md | 20 +++++++++++ src/black/cache.py | 6 ++-- src/black/linegen.py | 7 ++-- ..._parens_with_braces_and_square_brackets.py | 36 +++++++++++++++++++ 5 files changed, 65 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 60231468bdf..dd5f52cf706 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,8 @@ - Multiline dictionaries and lists that are the sole argument to a function are now indented less (#3964) +- Multiline list and dict unpacking as the sole argument to a function is now also + indented less (#3992) ### Configuration diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index c744902577d..944ffad033e 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -139,6 +139,26 @@ foo([ ]) ``` +This also applies to list and dictionary unpacking: + +```python +foo( + *[ + a_long_function_name(a_long_variable_name) + for a_long_variable_name in some_generator + ] +) +``` + +will become: + +```python +foo(*[ + a_long_function_name(a_long_variable_name) + for a_long_variable_name in some_generator +]) +``` + You can use a magic trailing comma to avoid this compacting behavior; by default, _Black_ will not reformat the following code: diff --git a/src/black/cache.py b/src/black/cache.py index 6baa096baca..6a332304981 100644 --- a/src/black/cache.py +++ b/src/black/cache.py @@ -124,9 +124,9 @@ def filtered_cached(self, sources: Iterable[Path]) -> Tuple[Set[Path], Set[Path] def write(self, sources: Iterable[Path]) -> None: """Update the cache file data and write a new cache file.""" - self.file_data.update( - **{str(src.resolve()): Cache.get_file_data(src) for src in sources} - ) + self.file_data.update(**{ + str(src.resolve()): Cache.get_file_data(src) for src in sources + }) try: CACHE_DIR.mkdir(parents=True, exist_ok=True) with tempfile.NamedTemporaryFile( diff --git a/src/black/linegen.py b/src/black/linegen.py index 5f5a69152d5..43bc08efbbd 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -817,16 +817,17 @@ def _first_right_hand_split( head_leaves.reverse() if Preview.hug_parens_with_braces_and_square_brackets in line.mode: + is_unpacking = 1 if body_leaves[0].type in [token.STAR, token.DOUBLESTAR] else 0 if ( tail_leaves[0].type == token.RPAR and tail_leaves[0].value and tail_leaves[0].opening_bracket is head_leaves[-1] and body_leaves[-1].type in [token.RBRACE, token.RSQB] - and body_leaves[-1].opening_bracket is body_leaves[0] + and body_leaves[-1].opening_bracket is body_leaves[is_unpacking] ): - head_leaves = head_leaves + body_leaves[:1] + head_leaves = head_leaves + body_leaves[: 1 + is_unpacking] tail_leaves = body_leaves[-1:] + tail_leaves - body_leaves = body_leaves[1:-1] + body_leaves = body_leaves[1 + is_unpacking : -1] head = bracket_split_build_line( head_leaves, line, opening_bracket, component=_BracketSplitComponent.head diff --git a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py index 6d10518133c..51fe516add5 100644 --- a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py +++ b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py @@ -137,6 +137,21 @@ def foo_square_brackets(request): [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], {x}, "a string", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] ) +foo(*["long long long long long line", "long long long long long line", "long long long long long line"]) + +foo(*[str(i) for i in range(100000000000000000000000000000000000000000000000000000000000)]) + +foo( + **{ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": 1, + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb": 2, + "ccccccccccccccccccccccccccccccccc": 3, + **other, + } +) + +foo(**{x: y for x, y in enumerate(["long long long long line","long long long long line"])}) + # output def foo_brackets(request): return JsonResponse({ @@ -287,3 +302,24 @@ def foo_square_brackets(request): baaaaaaaaaaaaar( [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], {x}, "a string", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] ) + +foo(*[ + "long long long long long line", + "long long long long long line", + "long long long long long line", +]) + +foo(*[ + str(i) for i in range(100000000000000000000000000000000000000000000000000000000000) +]) + +foo(**{ + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": 1, + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb": 2, + "ccccccccccccccccccccccccccccccccc": 3, + **other, +}) + +foo(**{ + x: y for x, y in enumerate(["long long long long line", "long long long long line"]) +}) From e50110353ab81b539aaee686453c18c707b5f045 Mon Sep 17 00:00:00 2001 From: Henri Holopainen Date: Tue, 31 Oct 2023 17:27:11 +0200 Subject: [PATCH 174/279] Produce equivalent code for docstrings containing backslash followed by whitespace(s) before newline (#4008) Fixes #3727 --- CHANGES.md | 3 ++- src/black/linegen.py | 3 ++- tests/data/cases/docstring.py | 13 +++++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index dd5f52cf706..e910fbed162 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,7 +8,8 @@ ### Stable style - +- Fix a crash when whitespace(s) followed a backslash before newline in a docstring + (#4008) ### Preview style diff --git a/src/black/linegen.py b/src/black/linegen.py index 43bc08efbbd..121c6e314fe 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -2,6 +2,7 @@ Generating lines of code. """ +import re import sys from dataclasses import replace from enum import Enum, auto @@ -420,7 +421,7 @@ def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: if Preview.hex_codes_in_unicode_sequences in self.mode: normalize_unicode_escape_sequences(leaf) - if is_docstring(leaf) and "\\\n" not in leaf.value: + if is_docstring(leaf) and not re.search(r"\\\s*\n", leaf.value): # We're ignoring docstrings with backslash newline escapes because changing # indentation of those changes the AST representation of the code. if self.mode.string_normalization: diff --git a/tests/data/cases/docstring.py b/tests/data/cases/docstring.py index c31d6a68783..e983c5bd438 100644 --- a/tests/data/cases/docstring.py +++ b/tests/data/cases/docstring.py @@ -221,6 +221,12 @@ def stable_quote_normalization_with_immediate_inner_single_quote(self): ''' +def foo(): + """ + Docstring with a backslash followed by a space\ + and then another line + """ + # output class MyClass: @@ -442,3 +448,10 @@ def stable_quote_normalization_with_immediate_inner_single_quote(self): """ + + +def foo(): + """ + Docstring with a backslash followed by a space\ + and then another line + """ From 5758da6e3cda4ec037c5dbb7867373cf694edd03 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 31 Oct 2023 17:11:28 -0700 Subject: [PATCH 175/279] Fix bytes strings being treated as docstrings (#4003) Fixes #4002 --- CHANGES.md | 4 +-- src/black/nodes.py | 9 ++++++- tests/data/cases/bytes_docstring.py | 34 +++++++++++++++++++++++++ tests/data/{ => cases}/raw_docstring.py | 0 4 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 tests/data/cases/bytes_docstring.py rename tests/data/{ => cases}/raw_docstring.py (100%) diff --git a/CHANGES.md b/CHANGES.md index e910fbed162..f365f1c239b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,8 +8,8 @@ ### Stable style -- Fix a crash when whitespace(s) followed a backslash before newline in a docstring - (#4008) +- Fix crash on formatting bytes strings that look like docstrings (#4003) +- Fix crash when whitespace followed a backslash before newline in a docstring (#4008) ### Preview style diff --git a/src/black/nodes.py b/src/black/nodes.py index b2e96cb9edf..5f6b280c035 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -14,7 +14,7 @@ from black.cache import CACHE_DIR from black.mode import Mode, Preview -from black.strings import has_triple_quotes +from black.strings import get_string_prefix, has_triple_quotes from blib2to3 import pygram from blib2to3.pgen2 import token from blib2to3.pytree import NL, Leaf, Node, type_repr @@ -525,6 +525,13 @@ def is_arith_like(node: LN) -> bool: def is_docstring(leaf: Leaf) -> bool: + if leaf.type != token.STRING: + return False + + prefix = get_string_prefix(leaf.value) + if "b" in prefix or "B" in prefix: + return False + if prev_siblings_are( leaf.parent, [None, token.NEWLINE, token.INDENT, syms.simple_stmt] ): diff --git a/tests/data/cases/bytes_docstring.py b/tests/data/cases/bytes_docstring.py new file mode 100644 index 00000000000..2326e95293a --- /dev/null +++ b/tests/data/cases/bytes_docstring.py @@ -0,0 +1,34 @@ +def bitey(): + b" not a docstring" + +def bitey2(): + b' also not a docstring' + +def triple_quoted_bytes(): + b""" not a docstring""" + +def triple_quoted_bytes2(): + b''' also not a docstring''' + +def capitalized_bytes(): + B" NOT A DOCSTRING" + +# output +def bitey(): + b" not a docstring" + + +def bitey2(): + b" also not a docstring" + + +def triple_quoted_bytes(): + b""" not a docstring""" + + +def triple_quoted_bytes2(): + b""" also not a docstring""" + + +def capitalized_bytes(): + b" NOT A DOCSTRING" \ No newline at end of file diff --git a/tests/data/raw_docstring.py b/tests/data/cases/raw_docstring.py similarity index 100% rename from tests/data/raw_docstring.py rename to tests/data/cases/raw_docstring.py From e2f2bd076fbc19d4adb90b70b5a7be32b08d5dbe Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Wed, 1 Nov 2023 06:20:14 -0700 Subject: [PATCH 176/279] Minor refactoring in get_sources and gen_python_files (#4013) --- src/black/__init__.py | 39 ++++++++++++++++++--------------------- src/black/files.py | 7 ++----- tests/test_black.py | 5 +++-- 3 files changed, 23 insertions(+), 28 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 7cf93b89e42..c11a66b7bc8 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -50,6 +50,7 @@ get_gitignore, normalize_path_maybe_ignore, parse_pyproject_toml, + path_is_excluded, wrap_stream_for_windows, ) from black.handle_ipynb_magics import ( @@ -632,15 +633,15 @@ def get_sources( for s in src: if s == "-" and stdin_filename: - p = Path(stdin_filename) + path = Path(stdin_filename) is_stdin = True else: - p = Path(s) + path = Path(s) is_stdin = False - if is_stdin or p.is_file(): + if is_stdin or path.is_file(): normalized_path: Optional[str] = normalize_path_maybe_ignore( - p, root, report + path, root, report ) if normalized_path is None: if verbose: @@ -651,38 +652,34 @@ def get_sources( normalized_path = "/" + normalized_path # Hard-exclude any files that matches the `--force-exclude` regex. - if force_exclude: - force_exclude_match = force_exclude.search(normalized_path) - else: - force_exclude_match = None - if force_exclude_match and force_exclude_match.group(0): - report.path_ignored(p, "matches the --force-exclude regular expression") + if path_is_excluded(normalized_path, force_exclude): + report.path_ignored( + path, "matches the --force-exclude regular expression" + ) continue if is_stdin: - p = Path(f"{STDIN_PLACEHOLDER}{str(p)}") + path = Path(f"{STDIN_PLACEHOLDER}{str(path)}") - if p.suffix == ".ipynb" and not jupyter_dependencies_are_installed( + if path.suffix == ".ipynb" and not jupyter_dependencies_are_installed( warn=verbose or not quiet ): continue - sources.add(p) - elif p.is_dir(): - p_relative = normalize_path_maybe_ignore(p, root, report) - assert p_relative is not None - p = root / p_relative + sources.add(path) + elif path.is_dir(): + path = root / (path.resolve().relative_to(root)) if verbose: - out(f'Found input source directory: "{p}"', fg="blue") + out(f'Found input source directory: "{path}"', fg="blue") if using_default_exclude: gitignore = { root: root_gitignore, - p: get_gitignore(p), + path: get_gitignore(path), } sources.update( gen_python_files( - p.iterdir(), + path.iterdir(), root, include, exclude, @@ -697,7 +694,7 @@ def get_sources( elif s == "-": if verbose: out("Found input source stdin", fg="blue") - sources.add(p) + sources.add(path) else: err(f"invalid path: {s}") diff --git a/src/black/files.py b/src/black/files.py index 1eed7eda828..858303ca1a3 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -280,7 +280,6 @@ def _path_is_ignored( root_relative_path: str, root: Path, gitignore_dict: Dict[Path, PathSpec], - report: Report, ) -> bool: path = root / root_relative_path # Note that this logic is sensitive to the ordering of gitignore_dict. Callers must @@ -291,9 +290,6 @@ def _path_is_ignored( except ValueError: break if pattern.match_file(relative_path): - report.path_ignored( - path.relative_to(root), "matches a .gitignore file content" - ) return True return False @@ -334,8 +330,9 @@ def gen_python_files( # First ignore files matching .gitignore, if passed if gitignore_dict and _path_is_ignored( - root_relative_path, root, gitignore_dict, report + root_relative_path, root, gitignore_dict ): + report.path_ignored(child, "matches a .gitignore file content") continue # Then ignore with `--exclude` `--extend-exclude` and `--force-exclude` options. diff --git a/tests/test_black.py b/tests/test_black.py index 56c20243020..c7196098e14 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -504,7 +504,7 @@ def _mocked_calls() -> bool: return _mocked_calls with patch("pathlib.Path.iterdir", return_value=target_contents), patch( - "pathlib.Path.cwd", return_value=working_directory + "pathlib.Path.resolve", return_value=target_abspath ), patch("pathlib.Path.is_dir", side_effect=mock_n_calls([True])): # Note that the root folder (project_root) isn't the folder # named "root" (aka working_directory) @@ -526,7 +526,8 @@ def _mocked_calls() -> bool: for _, mock_args, _ in report.path_ignored.mock_calls ), "A symbolic link was reported." report.path_ignored.assert_called_once_with( - Path("root", "child", "b.py"), "matches a .gitignore file content" + Path(working_directory, "child", "b.py"), + "matches a .gitignore file content", ) def test_report_verbose(self) -> None: From c54c213d6a3132986feede0cf0525f5bae5b43d6 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 2 Nov 2023 20:42:11 -0700 Subject: [PATCH 177/279] Fix crash on await (a ** b) (#3994) --- CHANGES.md | 2 ++ src/black/linegen.py | 22 ++++++++++------------ tests/data/cases/remove_await_parens.py | 19 +++++++++++++++++++ 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f365f1c239b..5ce37943693 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,8 @@ - Fix crash on formatting bytes strings that look like docstrings (#4003) - Fix crash when whitespace followed a backslash before newline in a docstring (#4008) +- Fix crash on formatting code like `await (a ** b)` (#3994) + ### Preview style - Multiline dictionaries and lists that are the sole argument to a function are now diff --git a/src/black/linegen.py b/src/black/linegen.py index 121c6e314fe..b13b95d9b31 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -1352,18 +1352,16 @@ def remove_await_parens(node: Node) -> None: opening_bracket = cast(Leaf, node.children[1].children[0]) closing_bracket = cast(Leaf, node.children[1].children[-1]) bracket_contents = node.children[1].children[1] - if isinstance(bracket_contents, Node): - if bracket_contents.type != syms.power: - ensure_visible(opening_bracket) - ensure_visible(closing_bracket) - elif ( - bracket_contents.type == syms.power - and bracket_contents.children[0].type == token.AWAIT - ): - ensure_visible(opening_bracket) - ensure_visible(closing_bracket) - # If we are in a nested await then recurse down. - remove_await_parens(bracket_contents) + if isinstance(bracket_contents, Node) and ( + bracket_contents.type != syms.power + or bracket_contents.children[0].type == token.AWAIT + or any( + isinstance(child, Leaf) and child.type == token.DOUBLESTAR + for child in bracket_contents.children + ) + ): + ensure_visible(opening_bracket) + ensure_visible(closing_bracket) def _maybe_wrap_cms_in_parens( diff --git a/tests/data/cases/remove_await_parens.py b/tests/data/cases/remove_await_parens.py index 8c7223d2f39..073150c5f08 100644 --- a/tests/data/cases/remove_await_parens.py +++ b/tests/data/cases/remove_await_parens.py @@ -80,6 +80,15 @@ async def main(): async def main(): await (yield) +async def main(): + await (a ** b) + await (a[b] ** c) + await (a ** b[c]) + await ((a + b) ** (c + d)) + await (a + b) + await (a[b]) + await (a[b ** c]) + # output import asyncio @@ -174,3 +183,13 @@ async def main(): async def main(): await (yield) + + +async def main(): + await (a**b) + await (a[b] ** c) + await (a ** b[c]) + await ((a + b) ** (c + d)) + await (a + b) + await a[b] + await a[b**c] From 448324637d12514b540efb33b4df7bf8af10c6d5 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Sat, 4 Nov 2023 22:49:12 +0200 Subject: [PATCH 178/279] Enable branch coverage (#4022) When trying to understand the code logic, and looking at coverage reports, branch coverage is very helpful. --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 8c55076e4c9..f3689bfb746 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -224,6 +224,7 @@ omit = [ ] [tool.coverage.run] relative_files = true +branch = true [tool.mypy] # Specify the target platform details in config, so your developers are From 9e3daa1107a66f311a8367395a33ed5fc5d5e73d Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 5 Nov 2023 18:29:37 -0800 Subject: [PATCH 179/279] Fix arm wheels on macOS (#4017) --- .github/workflows/pypi_upload.yml | 7 ++++--- CHANGES.md | 2 ++ pyproject.toml | 13 ++++++++++++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index a57013d67c1..07273f09508 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -68,9 +68,10 @@ jobs: - name: generate matrix (PR) if: github.event_name == 'pull_request' run: | - cibuildwheel --print-build-identifiers --platform linux \ - | pyp 'json.dumps({"only": x, "os": "ubuntu-latest"})' \ - | pyp 'json.dumps(list(map(json.loads, lines)))' > /tmp/matrix + { + cibuildwheel --print-build-identifiers --platform linux \ + | pyp 'json.dumps({"only": x, "os": "ubuntu-latest"})' + } | pyp 'json.dumps(list(map(json.loads, lines)))' > /tmp/matrix env: CIBW_BUILD: "cp38-* cp311-*" CIBW_ARCHS_LINUX: x86_64 diff --git a/CHANGES.md b/CHANGES.md index 5ce37943693..97084a2bfc1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -38,6 +38,8 @@ +- Fix mypyc builds on arm64 on macOS (#4017) + ### Output diff --git a/pyproject.toml b/pyproject.toml index f3689bfb746..c0302d2302a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,6 +113,8 @@ exclude = ["/profiling"] [tool.hatch.build.targets.wheel] only-include = ["src"] sources = ["src"] +# Note that we change the behaviour of this flag below +macos-max-compat = true [tool.hatch.build.targets.wheel.hooks.mypyc] enable-by-default = false @@ -175,9 +177,18 @@ before-build = [ HATCH_BUILD_HOOKS_ENABLE = "1" MYPYC_OPT_LEVEL = "3" MYPYC_DEBUG_LEVEL = "0" +AIOHTTP_NO_EXTENSIONS = "1" + # Black needs Clang to compile successfully on Linux. CC = "clang" -AIOHTTP_NO_EXTENSIONS = "1" + +[tool.cibuildwheel.macos] +build-frontend = { name = "build", args = ["--no-isolation"] } +# Unfortunately, hatch doesn't respect MACOSX_DEPLOYMENT_TARGET +before-build = [ + "python -m pip install 'hatchling==1.18.0' hatch-vcs hatch-fancy-pypi-readme 'hatch-mypyc>=0.16.0' 'mypy==1.5.1' 'click==8.1.3'", + """sed -i '' -e "600,700s/'10_16'/os.environ['MACOSX_DEPLOYMENT_TARGET'].replace('.', '_')/" $(python -c 'import hatchling.builders.wheel as h; print(h.__file__)') """, +] [tool.isort] atomic = true From e808e61db8c7a8f9c7fd4b2fff2281141f6b2517 Mon Sep 17 00:00:00 2001 From: Yilei Yang Date: Mon, 6 Nov 2023 14:30:04 -0800 Subject: [PATCH 180/279] Preview: Keep requiring two empty lines between module-level docstring and first function or class definition (#4028) Fixes #4027. --- CHANGES.md | 2 ++ src/black/lines.py | 1 + .../data/cases/module_docstring_followed_by_class.py | 11 +++++++++++ .../cases/module_docstring_followed_by_function.py | 11 +++++++++++ 4 files changed, 25 insertions(+) create mode 100644 tests/data/cases/module_docstring_followed_by_class.py create mode 100644 tests/data/cases/module_docstring_followed_by_function.py diff --git a/CHANGES.md b/CHANGES.md index 97084a2bfc1..a68f87bfc12 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -19,6 +19,8 @@ indented less (#3964) - Multiline list and dict unpacking as the sole argument to a function is now also indented less (#3992) +- Keep requiring two empty lines between module-level docstring and first function or + class definition. (#4028) ### Configuration diff --git a/src/black/lines.py b/src/black/lines.py index a73c429e3d9..23c1a93d3d4 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -578,6 +578,7 @@ def maybe_empty_lines(self, current_line: Line) -> LinesBlock: and self.previous_block.previous_block is None and len(self.previous_block.original_line.leaves) == 1 and self.previous_block.original_line.is_triple_quoted_string + and not (current_line.is_class or current_line.is_def) ): before = 1 diff --git a/tests/data/cases/module_docstring_followed_by_class.py b/tests/data/cases/module_docstring_followed_by_class.py new file mode 100644 index 00000000000..6fdbfc8c240 --- /dev/null +++ b/tests/data/cases/module_docstring_followed_by_class.py @@ -0,0 +1,11 @@ +# flags: --preview +"""Two blank lines between module docstring and a class.""" +class MyClass: + pass + +# output +"""Two blank lines between module docstring and a class.""" + + +class MyClass: + pass diff --git a/tests/data/cases/module_docstring_followed_by_function.py b/tests/data/cases/module_docstring_followed_by_function.py new file mode 100644 index 00000000000..5913a59e1fe --- /dev/null +++ b/tests/data/cases/module_docstring_followed_by_function.py @@ -0,0 +1,11 @@ +# flags: --preview +"""Two blank lines between module docstring and a function def.""" +def function(): + pass + +# output +"""Two blank lines between module docstring and a function def.""" + + +def function(): + pass From ecbd9e8cf71f13068c7e6803a534e00363114c91 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 6 Nov 2023 16:58:43 -0800 Subject: [PATCH 181/279] Fix crash with f-string docstrings (#4019) Python does not consider f-strings to be docstrings, so we probably shouldn't be formatting them as such Fixes #4018 Co-authored-by: Alex Waygood --- CHANGES.md | 3 +++ src/black/nodes.py | 2 +- tests/data/cases/docstring_preview.py | 3 ++- tests/data/cases/f_docstring.py | 20 +++++++++++++++++++ ...view_docstring_no_string_normalization.py} | 0 5 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 tests/data/cases/f_docstring.py rename tests/data/cases/{docstring_preview_no_string_normalization.py => preview_docstring_no_string_normalization.py} (100%) diff --git a/CHANGES.md b/CHANGES.md index a68f87bfc12..b1fe25ef625 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,6 +13,9 @@ - Fix crash on formatting code like `await (a ** b)` (#3994) +- No longer treat leading f-strings as docstrings. This matches Python's behaviour and + fixes a crash (#4019) + ### Preview style - Multiline dictionaries and lists that are the sole argument to a function are now diff --git a/src/black/nodes.py b/src/black/nodes.py index 5f6b280c035..fff8e05a118 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -529,7 +529,7 @@ def is_docstring(leaf: Leaf) -> bool: return False prefix = get_string_prefix(leaf.value) - if "b" in prefix or "B" in prefix: + if set(prefix).intersection("bBfF"): return False if prev_siblings_are( diff --git a/tests/data/cases/docstring_preview.py b/tests/data/cases/docstring_preview.py index ff4819acb67..a3c656be2f8 100644 --- a/tests/data/cases/docstring_preview.py +++ b/tests/data/cases/docstring_preview.py @@ -58,7 +58,8 @@ def docstring_almost_at_line_limit(): def docstring_almost_at_line_limit_with_prefix(): - f"""long docstring................................................................""" + f"""long docstring................................................................ + """ def mulitline_docstring_almost_at_line_limit(): diff --git a/tests/data/cases/f_docstring.py b/tests/data/cases/f_docstring.py new file mode 100644 index 00000000000..667f550b353 --- /dev/null +++ b/tests/data/cases/f_docstring.py @@ -0,0 +1,20 @@ +def foo(e): + f""" {'.'.join(e)}""" + +def bar(e): + f"{'.'.join(e)}" + +def baz(e): + F""" {'.'.join(e)}""" + +# output +def foo(e): + f""" {'.'.join(e)}""" + + +def bar(e): + f"{'.'.join(e)}" + + +def baz(e): + f""" {'.'.join(e)}""" diff --git a/tests/data/cases/docstring_preview_no_string_normalization.py b/tests/data/cases/preview_docstring_no_string_normalization.py similarity index 100% rename from tests/data/cases/docstring_preview_no_string_normalization.py rename to tests/data/cases/preview_docstring_no_string_normalization.py From 46be1f8e54ac9a7d67723c0fa28c7bec13a0a2bf Mon Sep 17 00:00:00 2001 From: Yilei Yang Date: Mon, 6 Nov 2023 18:05:25 -0800 Subject: [PATCH 182/279] Support formatting specified lines (#4020) --- CHANGES.md | 3 + docs/usage_and_configuration/the_basics.md | 17 + src/black/__init__.py | 130 ++++- src/black/nodes.py | 28 + src/black/ranges.py | 496 ++++++++++++++++++ tests/data/cases/line_ranges_basic.py | 107 ++++ tests/data/cases/line_ranges_fmt_off.py | 49 ++ .../cases/line_ranges_fmt_off_decorator.py | 27 + .../data/cases/line_ranges_fmt_off_overlap.py | 37 ++ tests/data/cases/line_ranges_imports.py | 9 + tests/data/cases/line_ranges_indentation.py | 27 + tests/data/cases/line_ranges_two_passes.py | 27 + tests/data/cases/line_ranges_unwrapping.py | 25 + tests/data/invalid_line_ranges.toml | 2 + tests/data/line_ranges_formatted/basic.py | 50 ++ .../line_ranges_formatted/pattern_matching.py | 25 + tests/test_black.py | 87 ++- tests/test_format.py | 26 +- tests/test_ranges.py | 185 +++++++ tests/util.py | 29 +- 20 files changed, 1358 insertions(+), 28 deletions(-) create mode 100644 src/black/ranges.py create mode 100644 tests/data/cases/line_ranges_basic.py create mode 100644 tests/data/cases/line_ranges_fmt_off.py create mode 100644 tests/data/cases/line_ranges_fmt_off_decorator.py create mode 100644 tests/data/cases/line_ranges_fmt_off_overlap.py create mode 100644 tests/data/cases/line_ranges_imports.py create mode 100644 tests/data/cases/line_ranges_indentation.py create mode 100644 tests/data/cases/line_ranges_two_passes.py create mode 100644 tests/data/cases/line_ranges_unwrapping.py create mode 100644 tests/data/invalid_line_ranges.toml create mode 100644 tests/data/line_ranges_formatted/basic.py create mode 100644 tests/data/line_ranges_formatted/pattern_matching.py create mode 100644 tests/test_ranges.py diff --git a/CHANGES.md b/CHANGES.md index b1fe25ef625..780a00247ce 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,9 @@ +- Support formatting ranges of lines with the new `--line-ranges` command-line option + (#4020). + ### Stable style - Fix crash on formatting bytes strings that look like docstrings (#4003) diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index f25dbb13d4d..dbd8c7ba434 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -175,6 +175,23 @@ All done! ✨ 🍰 ✨ 1 file would be reformatted. ``` +### `--line-ranges` + +When specified, _Black_ will try its best to only format these lines. + +This option can be specified multiple times, and a union of the lines will be formatted. +Each range must be specified as two integers connected by a `-`: `-`. The +`` and `` integer indices are 1-based and inclusive on both ends. + +_Black_ may still format lines outside of the ranges for multi-line statements. +Formatting more than one file or any ipynb files with this option is not supported. This +option cannot be specified in the `pyproject.toml` config. + +Example: `black --line-ranges=1-10 --line-ranges=21-30 test.py` will format lines from +`1` to `10` and `21` to `30`. + +This option is mainly for editor integrations, such as "Format Selection". + #### `--color` / `--no-color` Show (or do not show) colored diff. Only applies when `--diff` is given. diff --git a/src/black/__init__.py b/src/black/__init__.py index c11a66b7bc8..5aca3316df0 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -13,6 +13,7 @@ from pathlib import Path from typing import ( Any, + Collection, Dict, Generator, Iterator, @@ -77,6 +78,7 @@ from black.output import color_diff, diff, dump_to_file, err, ipynb_diff, out from black.parsing import InvalidInput # noqa F401 from black.parsing import lib2to3_parse, parse_ast, stringify_ast +from black.ranges import adjusted_lines, convert_unchanged_lines, parse_line_ranges from black.report import Changed, NothingChanged, Report from black.trans import iter_fexpr_spans from blib2to3.pgen2 import token @@ -163,6 +165,12 @@ def read_pyproject_toml( "extend-exclude", "Config key extend-exclude must be a string" ) + line_ranges = config.get("line_ranges") + if line_ranges is not None: + raise click.BadOptionUsage( + "line-ranges", "Cannot use line-ranges in the pyproject.toml file." + ) + default_map: Dict[str, Any] = {} if ctx.default_map: default_map.update(ctx.default_map) @@ -304,6 +312,19 @@ def validate_regex( is_flag=True, help="Don't write the files back, just output a diff for each file on stdout.", ) +@click.option( + "--line-ranges", + multiple=True, + metavar="START-END", + help=( + "When specified, _Black_ will try its best to only format these lines. This" + " option can be specified multiple times, and a union of the lines will be" + " formatted. Each range must be specified as two integers connected by a `-`:" + " `-`. The `` and `` integer indices are 1-based and" + " inclusive on both ends." + ), + default=(), +) @click.option( "--color/--no-color", is_flag=True, @@ -443,6 +464,7 @@ def main( # noqa: C901 target_version: List[TargetVersion], check: bool, diff: bool, + line_ranges: Sequence[str], color: bool, fast: bool, pyi: bool, @@ -544,6 +566,18 @@ def main( # noqa: C901 python_cell_magics=set(python_cell_magics), ) + lines: List[Tuple[int, int]] = [] + if line_ranges: + if ipynb: + err("Cannot use --line-ranges with ipynb files.") + ctx.exit(1) + + try: + lines = parse_line_ranges(line_ranges) + except ValueError as e: + err(str(e)) + ctx.exit(1) + if code is not None: # Run in quiet mode by default with -c; the extra output isn't useful. # You can still pass -v to get verbose output. @@ -553,7 +587,12 @@ def main( # noqa: C901 if code is not None: reformat_code( - content=code, fast=fast, write_back=write_back, mode=mode, report=report + content=code, + fast=fast, + write_back=write_back, + mode=mode, + report=report, + lines=lines, ) else: assert root is not None # root is only None if code is not None @@ -588,10 +627,14 @@ def main( # noqa: C901 write_back=write_back, mode=mode, report=report, + lines=lines, ) else: from black.concurrency import reformat_many + if lines: + err("Cannot use --line-ranges to format multiple files.") + ctx.exit(1) reformat_many( sources=sources, fast=fast, @@ -714,7 +757,13 @@ def path_empty( def reformat_code( - content: str, fast: bool, write_back: WriteBack, mode: Mode, report: Report + content: str, + fast: bool, + write_back: WriteBack, + mode: Mode, + report: Report, + *, + lines: Collection[Tuple[int, int]] = (), ) -> None: """ Reformat and print out `content` without spawning child processes. @@ -727,7 +776,7 @@ def reformat_code( try: changed = Changed.NO if format_stdin_to_stdout( - content=content, fast=fast, write_back=write_back, mode=mode + content=content, fast=fast, write_back=write_back, mode=mode, lines=lines ): changed = Changed.YES report.done(path, changed) @@ -741,7 +790,13 @@ def reformat_code( # not ideal, but this shouldn't cause any issues ... hopefully. ~ichard26 @mypyc_attr(patchable=True) def reformat_one( - src: Path, fast: bool, write_back: WriteBack, mode: Mode, report: "Report" + src: Path, + fast: bool, + write_back: WriteBack, + mode: Mode, + report: "Report", + *, + lines: Collection[Tuple[int, int]] = (), ) -> None: """Reformat a single file under `src` without spawning child processes. @@ -766,7 +821,9 @@ def reformat_one( mode = replace(mode, is_pyi=True) elif src.suffix == ".ipynb": mode = replace(mode, is_ipynb=True) - if format_stdin_to_stdout(fast=fast, write_back=write_back, mode=mode): + if format_stdin_to_stdout( + fast=fast, write_back=write_back, mode=mode, lines=lines + ): changed = Changed.YES else: cache = Cache.read(mode) @@ -774,7 +831,7 @@ def reformat_one( if not cache.is_changed(src): changed = Changed.CACHED if changed is not Changed.CACHED and format_file_in_place( - src, fast=fast, write_back=write_back, mode=mode + src, fast=fast, write_back=write_back, mode=mode, lines=lines ): changed = Changed.YES if (write_back is WriteBack.YES and changed is not Changed.CACHED) or ( @@ -794,6 +851,8 @@ def format_file_in_place( mode: Mode, write_back: WriteBack = WriteBack.NO, lock: Any = None, # multiprocessing.Manager().Lock() is some crazy proxy + *, + lines: Collection[Tuple[int, int]] = (), ) -> bool: """Format file under `src` path. Return True if changed. @@ -813,7 +872,9 @@ def format_file_in_place( header = buf.readline() src_contents, encoding, newline = decode_bytes(buf.read()) try: - dst_contents = format_file_contents(src_contents, fast=fast, mode=mode) + dst_contents = format_file_contents( + src_contents, fast=fast, mode=mode, lines=lines + ) except NothingChanged: return False except JSONDecodeError: @@ -858,6 +919,7 @@ def format_stdin_to_stdout( content: Optional[str] = None, write_back: WriteBack = WriteBack.NO, mode: Mode, + lines: Collection[Tuple[int, int]] = (), ) -> bool: """Format file on stdin. Return True if changed. @@ -876,7 +938,7 @@ def format_stdin_to_stdout( dst = src try: - dst = format_file_contents(src, fast=fast, mode=mode) + dst = format_file_contents(src, fast=fast, mode=mode, lines=lines) return True except NothingChanged: @@ -904,7 +966,11 @@ def format_stdin_to_stdout( def check_stability_and_equivalence( - src_contents: str, dst_contents: str, *, mode: Mode + src_contents: str, + dst_contents: str, + *, + mode: Mode, + lines: Collection[Tuple[int, int]] = (), ) -> None: """Perform stability and equivalence checks. @@ -913,10 +979,16 @@ def check_stability_and_equivalence( content differently. """ assert_equivalent(src_contents, dst_contents) - assert_stable(src_contents, dst_contents, mode=mode) + assert_stable(src_contents, dst_contents, mode=mode, lines=lines) -def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileContent: +def format_file_contents( + src_contents: str, + *, + fast: bool, + mode: Mode, + lines: Collection[Tuple[int, int]] = (), +) -> FileContent: """Reformat contents of a file and return new contents. If `fast` is False, additionally confirm that the reformatted code is @@ -926,13 +998,15 @@ def format_file_contents(src_contents: str, *, fast: bool, mode: Mode) -> FileCo if mode.is_ipynb: dst_contents = format_ipynb_string(src_contents, fast=fast, mode=mode) else: - dst_contents = format_str(src_contents, mode=mode) + dst_contents = format_str(src_contents, mode=mode, lines=lines) if src_contents == dst_contents: raise NothingChanged if not fast and not mode.is_ipynb: # Jupyter notebooks will already have been checked above. - check_stability_and_equivalence(src_contents, dst_contents, mode=mode) + check_stability_and_equivalence( + src_contents, dst_contents, mode=mode, lines=lines + ) return dst_contents @@ -1043,7 +1117,9 @@ def format_ipynb_string(src_contents: str, *, fast: bool, mode: Mode) -> FileCon raise NothingChanged -def format_str(src_contents: str, *, mode: Mode) -> str: +def format_str( + src_contents: str, *, mode: Mode, lines: Collection[Tuple[int, int]] = () +) -> str: """Reformat a string and return new contents. `mode` determines formatting options, such as how many characters per line are @@ -1073,16 +1149,20 @@ def f( hey """ - dst_contents = _format_str_once(src_contents, mode=mode) + dst_contents = _format_str_once(src_contents, mode=mode, lines=lines) # Forced second pass to work around optional trailing commas (becoming # forced trailing commas on pass 2) interacting differently with optional # parentheses. Admittedly ugly. if src_contents != dst_contents: - return _format_str_once(dst_contents, mode=mode) + if lines: + lines = adjusted_lines(lines, src_contents, dst_contents) + return _format_str_once(dst_contents, mode=mode, lines=lines) return dst_contents -def _format_str_once(src_contents: str, *, mode: Mode) -> str: +def _format_str_once( + src_contents: str, *, mode: Mode, lines: Collection[Tuple[int, int]] = () +) -> str: src_node = lib2to3_parse(src_contents.lstrip(), mode.target_versions) dst_blocks: List[LinesBlock] = [] if mode.target_versions: @@ -1097,7 +1177,11 @@ def _format_str_once(src_contents: str, *, mode: Mode) -> str: if supports_feature(versions, feature) } normalize_fmt_off(src_node, mode) - lines = LineGenerator(mode=mode, features=context_manager_features) + if lines: + # This should be called after normalize_fmt_off. + convert_unchanged_lines(src_node, lines) + + line_generator = LineGenerator(mode=mode, features=context_manager_features) elt = EmptyLineTracker(mode=mode) split_line_features = { feature @@ -1105,7 +1189,7 @@ def _format_str_once(src_contents: str, *, mode: Mode) -> str: if supports_feature(versions, feature) } block: Optional[LinesBlock] = None - for current_line in lines.visit(src_node): + for current_line in line_generator.visit(src_node): block = elt.maybe_empty_lines(current_line) dst_blocks.append(block) for line in transform_line( @@ -1373,12 +1457,16 @@ def assert_equivalent(src: str, dst: str) -> None: ) from None -def assert_stable(src: str, dst: str, mode: Mode) -> None: +def assert_stable( + src: str, dst: str, mode: Mode, *, lines: Collection[Tuple[int, int]] = () +) -> None: """Raise AssertionError if `dst` reformats differently the second time.""" # We shouldn't call format_str() here, because that formats the string # twice and may hide a bug where we bounce back and forth between two # versions. - newdst = _format_str_once(dst, mode=mode) + if lines: + lines = adjusted_lines(lines, src, dst) + newdst = _format_str_once(dst, mode=mode, lines=lines) if dst != newdst: log = dump_to_file( str(mode), diff --git a/src/black/nodes.py b/src/black/nodes.py index fff8e05a118..9251b0defb0 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -935,3 +935,31 @@ def is_part_of_annotation(leaf: Leaf) -> bool: return True ancestor = ancestor.parent return False + + +def first_leaf(node: LN) -> Optional[Leaf]: + """Returns the first leaf of the ancestor node.""" + if isinstance(node, Leaf): + return node + elif not node.children: + return None + else: + return first_leaf(node.children[0]) + + +def last_leaf(node: LN) -> Optional[Leaf]: + """Returns the last leaf of the ancestor node.""" + if isinstance(node, Leaf): + return node + elif not node.children: + return None + else: + return last_leaf(node.children[-1]) + + +def furthest_ancestor_with_last_leaf(leaf: Leaf) -> LN: + """Returns the furthest ancestor that has this leaf node as the last leaf.""" + node: LN = leaf + while node.parent and node.parent.children and node is node.parent.children[-1]: + node = node.parent + return node diff --git a/src/black/ranges.py b/src/black/ranges.py new file mode 100644 index 00000000000..b0c312e6274 --- /dev/null +++ b/src/black/ranges.py @@ -0,0 +1,496 @@ +"""Functions related to Black's formatting by line ranges feature.""" + +import difflib +from dataclasses import dataclass +from typing import Collection, Iterator, List, Sequence, Set, Tuple, Union + +from black.nodes import ( + LN, + STANDALONE_COMMENT, + Leaf, + Node, + Visitor, + first_leaf, + furthest_ancestor_with_last_leaf, + last_leaf, + syms, +) +from blib2to3.pgen2.token import ASYNC, NEWLINE + + +def parse_line_ranges(line_ranges: Sequence[str]) -> List[Tuple[int, int]]: + lines: List[Tuple[int, int]] = [] + for lines_str in line_ranges: + parts = lines_str.split("-") + if len(parts) != 2: + raise ValueError( + "Incorrect --line-ranges format, expect 'START-END', found" + f" {lines_str!r}" + ) + try: + start = int(parts[0]) + end = int(parts[1]) + except ValueError: + raise ValueError( + "Incorrect --line-ranges value, expect integer ranges, found" + f" {lines_str!r}" + ) from None + else: + lines.append((start, end)) + return lines + + +def is_valid_line_range(lines: Tuple[int, int]) -> bool: + """Returns whether the line range is valid.""" + return not lines or lines[0] <= lines[1] + + +def adjusted_lines( + lines: Collection[Tuple[int, int]], + original_source: str, + modified_source: str, +) -> List[Tuple[int, int]]: + """Returns the adjusted line ranges based on edits from the original code. + + This computes the new line ranges by diffing original_source and + modified_source, and adjust each range based on how the range overlaps with + the diffs. + + Note the diff can contain lines outside of the original line ranges. This can + happen when the formatting has to be done in adjacent to maintain consistent + local results. For example: + + 1. def my_func(arg1, arg2, + 2. arg3,): + 3. pass + + If it restricts to line 2-2, it can't simply reformat line 2, it also has + to reformat line 1: + + 1. def my_func( + 2. arg1, + 3. arg2, + 4. arg3, + 5. ): + 6. pass + + In this case, we will expand the line ranges to also include the whole diff + block. + + Args: + lines: a collection of line ranges. + original_source: the original source. + modified_source: the modified source. + """ + lines_mappings = _calculate_lines_mappings(original_source, modified_source) + + new_lines = [] + # Keep an index of the current search. Since the lines and lines_mappings are + # sorted, this makes the search complexity linear. + current_mapping_index = 0 + for start, end in sorted(lines): + start_mapping_index = _find_lines_mapping_index( + start, + lines_mappings, + current_mapping_index, + ) + end_mapping_index = _find_lines_mapping_index( + end, + lines_mappings, + start_mapping_index, + ) + current_mapping_index = start_mapping_index + if start_mapping_index >= len(lines_mappings) or end_mapping_index >= len( + lines_mappings + ): + # Protect against invalid inputs. + continue + start_mapping = lines_mappings[start_mapping_index] + end_mapping = lines_mappings[end_mapping_index] + if start_mapping.is_changed_block: + # When the line falls into a changed block, expands to the whole block. + new_start = start_mapping.modified_start + else: + new_start = ( + start - start_mapping.original_start + start_mapping.modified_start + ) + if end_mapping.is_changed_block: + # When the line falls into a changed block, expands to the whole block. + new_end = end_mapping.modified_end + else: + new_end = end - end_mapping.original_start + end_mapping.modified_start + new_range = (new_start, new_end) + if is_valid_line_range(new_range): + new_lines.append(new_range) + return new_lines + + +def convert_unchanged_lines(src_node: Node, lines: Collection[Tuple[int, int]]) -> None: + """Converts unchanged lines to STANDALONE_COMMENT. + + The idea is similar to how `# fmt: on/off` is implemented. It also converts the + nodes between those markers as a single `STANDALONE_COMMENT` leaf node with + the unformatted code as its value. `STANDALONE_COMMENT` is a "fake" token + that will be formatted as-is with its prefix normalized. + + Here we perform two passes: + + 1. Visit the top-level statements, and convert them to a single + `STANDALONE_COMMENT` when unchanged. This speeds up formatting when some + of the top-level statements aren't changed. + 2. Convert unchanged "unwrapped lines" to `STANDALONE_COMMENT` nodes line by + line. "unwrapped lines" are divided by the `NEWLINE` token. e.g. a + multi-line statement is *one* "unwrapped line" that ends with `NEWLINE`, + even though this statement itself can span multiple lines, and the + tokenizer only sees the last '\n' as the `NEWLINE` token. + + NOTE: During pass (2), comment prefixes and indentations are ALWAYS + normalized even when the lines aren't changed. This is fixable by moving + more formatting to pass (1). However, it's hard to get it correct when + incorrect indentations are used. So we defer this to future optimizations. + """ + lines_set: Set[int] = set() + for start, end in lines: + lines_set.update(range(start, end + 1)) + visitor = _TopLevelStatementsVisitor(lines_set) + _ = list(visitor.visit(src_node)) # Consume all results. + _convert_unchanged_line_by_line(src_node, lines_set) + + +def _contains_standalone_comment(node: LN) -> bool: + if isinstance(node, Leaf): + return node.type == STANDALONE_COMMENT + else: + for child in node.children: + if _contains_standalone_comment(child): + return True + return False + + +class _TopLevelStatementsVisitor(Visitor[None]): + """ + A node visitor that converts unchanged top-level statements to + STANDALONE_COMMENT. + + This is used in addition to _convert_unchanged_lines_by_flatterning, to + speed up formatting when there are unchanged top-level + classes/functions/statements. + """ + + def __init__(self, lines_set: Set[int]): + self._lines_set = lines_set + + def visit_simple_stmt(self, node: Node) -> Iterator[None]: + # This is only called for top-level statements, since `visit_suite` + # won't visit its children nodes. + yield from [] + newline_leaf = last_leaf(node) + if not newline_leaf: + return + assert ( + newline_leaf.type == NEWLINE + ), f"Unexpectedly found leaf.type={newline_leaf.type}" + # We need to find the furthest ancestor with the NEWLINE as the last + # leaf, since a `suite` can simply be a `simple_stmt` when it puts + # its body on the same line. Example: `if cond: pass`. + ancestor = furthest_ancestor_with_last_leaf(newline_leaf) + if not _get_line_range(ancestor).intersection(self._lines_set): + _convert_node_to_standalone_comment(ancestor) + + def visit_suite(self, node: Node) -> Iterator[None]: + yield from [] + # If there is a STANDALONE_COMMENT node, it means parts of the node tree + # have fmt on/off/skip markers. Those STANDALONE_COMMENT nodes can't + # be simply converted by calling str(node). So we just don't convert + # here. + if _contains_standalone_comment(node): + return + # Find the semantic parent of this suite. For `async_stmt` and + # `async_funcdef`, the ASYNC token is defined on a separate level by the + # grammar. + semantic_parent = node.parent + if semantic_parent is not None: + if ( + semantic_parent.prev_sibling is not None + and semantic_parent.prev_sibling.type == ASYNC + ): + semantic_parent = semantic_parent.parent + if semantic_parent is not None and not _get_line_range( + semantic_parent + ).intersection(self._lines_set): + _convert_node_to_standalone_comment(semantic_parent) + + +def _convert_unchanged_line_by_line(node: Node, lines_set: Set[int]) -> None: + """Converts unchanged to STANDALONE_COMMENT line by line.""" + for leaf in node.leaves(): + if leaf.type != NEWLINE: + # We only consider "unwrapped lines", which are divided by the NEWLINE + # token. + continue + if leaf.parent and leaf.parent.type == syms.match_stmt: + # The `suite` node is defined as: + # match_stmt: "match" subject_expr ':' NEWLINE INDENT case_block+ DEDENT + # Here we need to check `subject_expr`. The `case_block+` will be + # checked by their own NEWLINEs. + nodes_to_ignore: List[LN] = [] + prev_sibling = leaf.prev_sibling + while prev_sibling: + nodes_to_ignore.insert(0, prev_sibling) + prev_sibling = prev_sibling.prev_sibling + if not _get_line_range(nodes_to_ignore).intersection(lines_set): + _convert_nodes_to_standalone_comment(nodes_to_ignore, newline=leaf) + elif leaf.parent and leaf.parent.type == syms.suite: + # The `suite` node is defined as: + # suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT + # We will check `simple_stmt` and `stmt+` separately against the lines set + parent_sibling = leaf.parent.prev_sibling + nodes_to_ignore = [] + while parent_sibling and not parent_sibling.type == syms.suite: + # NOTE: Multiple suite nodes can exist as siblings in e.g. `if_stmt`. + nodes_to_ignore.insert(0, parent_sibling) + parent_sibling = parent_sibling.prev_sibling + # Special case for `async_stmt` and `async_funcdef` where the ASYNC + # token is on the grandparent node. + grandparent = leaf.parent.parent + if ( + grandparent is not None + and grandparent.prev_sibling is not None + and grandparent.prev_sibling.type == ASYNC + ): + nodes_to_ignore.insert(0, grandparent.prev_sibling) + if not _get_line_range(nodes_to_ignore).intersection(lines_set): + _convert_nodes_to_standalone_comment(nodes_to_ignore, newline=leaf) + else: + ancestor = furthest_ancestor_with_last_leaf(leaf) + # Consider multiple decorators as a whole block, as their + # newlines have different behaviors than the rest of the grammar. + if ( + ancestor.type == syms.decorator + and ancestor.parent + and ancestor.parent.type == syms.decorators + ): + ancestor = ancestor.parent + if not _get_line_range(ancestor).intersection(lines_set): + _convert_node_to_standalone_comment(ancestor) + + +def _convert_node_to_standalone_comment(node: LN) -> None: + """Convert node to STANDALONE_COMMENT by modifying the tree inline.""" + parent = node.parent + if not parent: + return + first = first_leaf(node) + last = last_leaf(node) + if not first or not last: + return + if first is last: + # This can happen on the following edge cases: + # 1. A block of `# fmt: off/on` code except the `# fmt: on` is placed + # on the end of the last line instead of on a new line. + # 2. A single backslash on its own line followed by a comment line. + # Ideally we don't want to format them when not requested, but fixing + # isn't easy. These cases are also badly formatted code, so it isn't + # too bad we reformat them. + return + # The prefix contains comments and indentation whitespaces. They are + # reformatted accordingly to the correct indentation level. + # This also means the indentation will be changed on the unchanged lines, and + # this is actually required to not break incremental reformatting. + prefix = first.prefix + first.prefix = "" + index = node.remove() + if index is not None: + # Remove the '\n', as STANDALONE_COMMENT will have '\n' appended when + # genearting the formatted code. + value = str(node)[:-1] + parent.insert_child( + index, + Leaf( + STANDALONE_COMMENT, + value, + prefix=prefix, + fmt_pass_converted_first_leaf=first, + ), + ) + + +def _convert_nodes_to_standalone_comment(nodes: Sequence[LN], *, newline: Leaf) -> None: + """Convert nodes to STANDALONE_COMMENT by modifying the tree inline.""" + if not nodes: + return + parent = nodes[0].parent + first = first_leaf(nodes[0]) + if not parent or not first: + return + prefix = first.prefix + first.prefix = "" + value = "".join(str(node) for node in nodes) + # The prefix comment on the NEWLINE leaf is the trailing comment of the statement. + if newline.prefix: + value += newline.prefix + newline.prefix = "" + index = nodes[0].remove() + for node in nodes[1:]: + node.remove() + if index is not None: + parent.insert_child( + index, + Leaf( + STANDALONE_COMMENT, + value, + prefix=prefix, + fmt_pass_converted_first_leaf=first, + ), + ) + + +def _leaf_line_end(leaf: Leaf) -> int: + """Returns the line number of the leaf node's last line.""" + if leaf.type == NEWLINE: + return leaf.lineno + else: + # Leaf nodes like multiline strings can occupy multiple lines. + return leaf.lineno + str(leaf).count("\n") + + +def _get_line_range(node_or_nodes: Union[LN, List[LN]]) -> Set[int]: + """Returns the line range of this node or list of nodes.""" + if isinstance(node_or_nodes, list): + nodes = node_or_nodes + if not nodes: + return set() + first = first_leaf(nodes[0]) + last = last_leaf(nodes[-1]) + if first and last: + line_start = first.lineno + line_end = _leaf_line_end(last) + return set(range(line_start, line_end + 1)) + else: + return set() + else: + node = node_or_nodes + if isinstance(node, Leaf): + return set(range(node.lineno, _leaf_line_end(node) + 1)) + else: + first = first_leaf(node) + last = last_leaf(node) + if first and last: + return set(range(first.lineno, _leaf_line_end(last) + 1)) + else: + return set() + + +@dataclass +class _LinesMapping: + """1-based lines mapping from original source to modified source. + + Lines [original_start, original_end] from original source + are mapped to [modified_start, modified_end]. + + The ranges are inclusive on both ends. + """ + + original_start: int + original_end: int + modified_start: int + modified_end: int + # Whether this range corresponds to a changed block, or an unchanged block. + is_changed_block: bool + + +def _calculate_lines_mappings( + original_source: str, + modified_source: str, +) -> Sequence[_LinesMapping]: + """Returns a sequence of _LinesMapping by diffing the sources. + + For example, given the following diff: + import re + - def func(arg1, + - arg2, arg3): + + def func(arg1, arg2, arg3): + pass + It returns the following mappings: + original -> modified + (1, 1) -> (1, 1), is_changed_block=False (the "import re" line) + (2, 3) -> (2, 2), is_changed_block=True (the diff) + (4, 4) -> (3, 3), is_changed_block=False (the "pass" line) + + You can think of this visually as if it brings up a side-by-side diff, and tries + to map the line ranges from the left side to the right side: + + (1, 1)->(1, 1) 1. import re 1. import re + (2, 3)->(2, 2) 2. def func(arg1, 2. def func(arg1, arg2, arg3): + 3. arg2, arg3): + (4, 4)->(3, 3) 4. pass 3. pass + + Args: + original_source: the original source. + modified_source: the modified source. + """ + matcher = difflib.SequenceMatcher( + None, + original_source.splitlines(keepends=True), + modified_source.splitlines(keepends=True), + ) + matching_blocks = matcher.get_matching_blocks() + lines_mappings: List[_LinesMapping] = [] + # matching_blocks is a sequence of "same block of code ranges", see + # https://docs.python.org/3/library/difflib.html#difflib.SequenceMatcher.get_matching_blocks + # Each block corresponds to a _LinesMapping with is_changed_block=False, + # and the ranges between two blocks corresponds to a _LinesMapping with + # is_changed_block=True, + # NOTE: matching_blocks is 0-based, but _LinesMapping is 1-based. + for i, block in enumerate(matching_blocks): + if i == 0: + if block.a != 0 or block.b != 0: + lines_mappings.append( + _LinesMapping( + original_start=1, + original_end=block.a, + modified_start=1, + modified_end=block.b, + is_changed_block=False, + ) + ) + else: + previous_block = matching_blocks[i - 1] + lines_mappings.append( + _LinesMapping( + original_start=previous_block.a + previous_block.size + 1, + original_end=block.a, + modified_start=previous_block.b + previous_block.size + 1, + modified_end=block.b, + is_changed_block=True, + ) + ) + if i < len(matching_blocks) - 1: + lines_mappings.append( + _LinesMapping( + original_start=block.a + 1, + original_end=block.a + block.size, + modified_start=block.b + 1, + modified_end=block.b + block.size, + is_changed_block=False, + ) + ) + return lines_mappings + + +def _find_lines_mapping_index( + original_line: int, + lines_mappings: Sequence[_LinesMapping], + start_index: int, +) -> int: + """Returns the original index of the lines mappings for the original line.""" + index = start_index + while index < len(lines_mappings): + mapping = lines_mappings[index] + if ( + mapping.original_start <= original_line + and original_line <= mapping.original_end + ): + return index + index += 1 + return index diff --git a/tests/data/cases/line_ranges_basic.py b/tests/data/cases/line_ranges_basic.py new file mode 100644 index 00000000000..9f0fb2da70e --- /dev/null +++ b/tests/data/cases/line_ranges_basic.py @@ -0,0 +1,107 @@ +# flags: --line-ranges=5-6 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. +def foo1(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass +def foo2(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass +def foo3(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass +def foo4(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass + +# Adding some unformated code covering a wide range of syntaxes. + +if True: + # Incorrectly indented prefix comments. + pass + +import typing +from typing import ( + Any , + ) +class MyClass( object): # Trailing comment with extra leading space. + #NOTE: The following indentation is incorrect: + @decor( 1 * 3 ) + def my_func( arg): + pass + +try: # Trailing comment with extra leading space. + for i in range(10): # Trailing comment with extra leading space. + while condition: + if something: + then_something( ) + elif something_else: + then_something_else( ) +except ValueError as e: + unformatted( ) +finally: + unformatted( ) + +async def test_async_unformatted( ): # Trailing comment with extra leading space. + async for i in some_iter( unformatted ): # Trailing comment with extra leading space. + await asyncio.sleep( 1 ) + async with some_context( unformatted ): + print( "unformatted" ) + + +# output +# flags: --line-ranges=5-6 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. +def foo1(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass +def foo2( + parameter_1, + parameter_2, + parameter_3, + parameter_4, + parameter_5, + parameter_6, + parameter_7, +): + pass + + +def foo3( + parameter_1, + parameter_2, + parameter_3, + parameter_4, + parameter_5, + parameter_6, + parameter_7, +): + pass + + +def foo4(parameter_1, parameter_2, parameter_3, parameter_4, parameter_5, parameter_6, parameter_7): pass + +# Adding some unformated code covering a wide range of syntaxes. + +if True: + # Incorrectly indented prefix comments. + pass + +import typing +from typing import ( + Any , + ) +class MyClass( object): # Trailing comment with extra leading space. + #NOTE: The following indentation is incorrect: + @decor( 1 * 3 ) + def my_func( arg): + pass + +try: # Trailing comment with extra leading space. + for i in range(10): # Trailing comment with extra leading space. + while condition: + if something: + then_something( ) + elif something_else: + then_something_else( ) +except ValueError as e: + unformatted( ) +finally: + unformatted( ) + +async def test_async_unformatted( ): # Trailing comment with extra leading space. + async for i in some_iter( unformatted ): # Trailing comment with extra leading space. + await asyncio.sleep( 1 ) + async with some_context( unformatted ): + print( "unformatted" ) diff --git a/tests/data/cases/line_ranges_fmt_off.py b/tests/data/cases/line_ranges_fmt_off.py new file mode 100644 index 00000000000..b51cef58fe5 --- /dev/null +++ b/tests/data/cases/line_ranges_fmt_off.py @@ -0,0 +1,49 @@ +# flags: --line-ranges=7-7 --line-ranges=17-23 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# fmt: off +import os +def myfunc( ): # Intentionally unformatted. + pass +# fmt: on + + +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: off +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: on + + +def myfunc( ): # This will be reformatted. + print( {"this will be reformatted"} ) + +# output + +# flags: --line-ranges=7-7 --line-ranges=17-23 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# fmt: off +import os +def myfunc( ): # Intentionally unformatted. + pass +# fmt: on + + +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: off +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: on + + +def myfunc(): # This will be reformatted. + print({"this will be reformatted"}) diff --git a/tests/data/cases/line_ranges_fmt_off_decorator.py b/tests/data/cases/line_ranges_fmt_off_decorator.py new file mode 100644 index 00000000000..14aa1dda02d --- /dev/null +++ b/tests/data/cases/line_ranges_fmt_off_decorator.py @@ -0,0 +1,27 @@ +# flags: --line-ranges=12-12 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# Regression test for an edge case involving decorators and fmt: off/on. +class MyClass: + + # fmt: off + @decorator ( ) + # fmt: on + def method(): + print ( "str" ) + +# output + +# flags: --line-ranges=12-12 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# Regression test for an edge case involving decorators and fmt: off/on. +class MyClass: + + # fmt: off + @decorator ( ) + # fmt: on + def method(): + print("str") diff --git a/tests/data/cases/line_ranges_fmt_off_overlap.py b/tests/data/cases/line_ranges_fmt_off_overlap.py new file mode 100644 index 00000000000..0391d17a843 --- /dev/null +++ b/tests/data/cases/line_ranges_fmt_off_overlap.py @@ -0,0 +1,37 @@ +# flags: --line-ranges=11-17 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + + +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: off +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: on + + +def myfunc( ): # This will be reformatted. + print( {"this will be reformatted"} ) + +# output + +# flags: --line-ranges=11-17 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + + +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: off +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +def myfunc( ): # This will not be reformatted. + print( {"also won't be reformatted"} ) +# fmt: on + + +def myfunc(): # This will be reformatted. + print({"this will be reformatted"}) diff --git a/tests/data/cases/line_ranges_imports.py b/tests/data/cases/line_ranges_imports.py new file mode 100644 index 00000000000..76b18ffecb3 --- /dev/null +++ b/tests/data/cases/line_ranges_imports.py @@ -0,0 +1,9 @@ +# flags: --line-ranges=8-8 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# This test ensures no empty lines are added around import lines. +# It caused an issue before https://github.com/psf/black/pull/3610 is merged. +import os +import re +import sys diff --git a/tests/data/cases/line_ranges_indentation.py b/tests/data/cases/line_ranges_indentation.py new file mode 100644 index 00000000000..82d3ad69a5e --- /dev/null +++ b/tests/data/cases/line_ranges_indentation.py @@ -0,0 +1,27 @@ +# flags: --line-ranges=5-5 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. +if cond1: + print("first") + if cond2: + print("second") + else: + print("else") + +if another_cond: + print("will not be changed") + +# output + +# flags: --line-ranges=5-5 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. +if cond1: + print("first") + if cond2: + print("second") + else: + print("else") + +if another_cond: + print("will not be changed") diff --git a/tests/data/cases/line_ranges_two_passes.py b/tests/data/cases/line_ranges_two_passes.py new file mode 100644 index 00000000000..aeed3260b8e --- /dev/null +++ b/tests/data/cases/line_ranges_two_passes.py @@ -0,0 +1,27 @@ +# flags: --line-ranges=9-11 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# This is a specific case for Black's two-pass formatting behavior in `format_str`. +# The second pass must respect the line ranges before the first pass. + + +def restrict_to_this_line(arg1, + arg2, + arg3): + print ( "This should not be formatted." ) + print ( "Note that in the second pass, the original line range 9-11 will cover these print lines.") + +# output + +# flags: --line-ranges=9-11 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# This is a specific case for Black's two-pass formatting behavior in `format_str`. +# The second pass must respect the line ranges before the first pass. + + +def restrict_to_this_line(arg1, arg2, arg3): + print ( "This should not be formatted." ) + print ( "Note that in the second pass, the original line range 9-11 will cover these print lines.") diff --git a/tests/data/cases/line_ranges_unwrapping.py b/tests/data/cases/line_ranges_unwrapping.py new file mode 100644 index 00000000000..cd7751b9417 --- /dev/null +++ b/tests/data/cases/line_ranges_unwrapping.py @@ -0,0 +1,25 @@ +# flags: --line-ranges=5-5 --line-ranges=9-9 --line-ranges=13-13 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. +alist = [ + 1, 2 +] + +adict = { + "key" : "value" +} + +func_call ( + arg = value +) + +# output + +# flags: --line-ranges=5-5 --line-ranges=9-9 --line-ranges=13-13 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. +alist = [1, 2] + +adict = {"key": "value"} + +func_call(arg=value) diff --git a/tests/data/invalid_line_ranges.toml b/tests/data/invalid_line_ranges.toml new file mode 100644 index 00000000000..791573f2625 --- /dev/null +++ b/tests/data/invalid_line_ranges.toml @@ -0,0 +1,2 @@ +[tool.black] +line-ranges = "1-1" diff --git a/tests/data/line_ranges_formatted/basic.py b/tests/data/line_ranges_formatted/basic.py new file mode 100644 index 00000000000..b419b1f16ae --- /dev/null +++ b/tests/data/line_ranges_formatted/basic.py @@ -0,0 +1,50 @@ +"""Module doc.""" + +from typing import ( + Callable, + Literal, +) + + +# fmt: off +class Unformatted: + def should_also_work(self): + pass +# fmt: on + + +a = [1, 2] # fmt: skip + + +# This should cover as many syntaxes as possible. +class Foo: + """Class doc.""" + + def __init__(self) -> None: + pass + + @add_logging + @memoize.memoize(max_items=2) + def plus_one( + self, + number: int, + ) -> int: + return number + 1 + + async def async_plus_one(self, number: int) -> int: + await asyncio.sleep(1) + async with some_context(): + return number + 1 + + +try: + for i in range(10): + while condition: + if something: + then_something() + elif something_else: + then_something_else() +except ValueError as e: + handle(e) +finally: + done() diff --git a/tests/data/line_ranges_formatted/pattern_matching.py b/tests/data/line_ranges_formatted/pattern_matching.py new file mode 100644 index 00000000000..cd98efdd504 --- /dev/null +++ b/tests/data/line_ranges_formatted/pattern_matching.py @@ -0,0 +1,25 @@ +# flags: --minimum-version=3.10 + + +def pattern_matching(): + match status: + case 1: + return "1" + case [single]: + return "single" + case [ + action, + obj, + ]: + return "act on obj" + case Point(x=0): + return "class pattern" + case {"text": message}: + return "mapping" + case { + "text": message, + "format": _, + }: + return "mapping" + case _: + return "fallback" diff --git a/tests/test_black.py b/tests/test_black.py index c7196098e14..c9819742425 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -8,6 +8,7 @@ import os import re import sys +import textwrap import types from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager, redirect_stderr @@ -1269,7 +1270,7 @@ def test_reformat_one_with_stdin_filename(self) -> None: report=report, ) fsts.assert_called_once_with( - fast=True, write_back=black.WriteBack.YES, mode=DEFAULT_MODE + fast=True, write_back=black.WriteBack.YES, mode=DEFAULT_MODE, lines=() ) # __BLACK_STDIN_FILENAME__ should have been stripped report.done.assert_called_with(expected, black.Changed.YES) @@ -1295,6 +1296,7 @@ def test_reformat_one_with_stdin_filename_pyi(self) -> None: fast=True, write_back=black.WriteBack.YES, mode=replace(DEFAULT_MODE, is_pyi=True), + lines=(), ) # __BLACK_STDIN_FILENAME__ should have been stripped report.done.assert_called_with(expected, black.Changed.YES) @@ -1320,6 +1322,7 @@ def test_reformat_one_with_stdin_filename_ipynb(self) -> None: fast=True, write_back=black.WriteBack.YES, mode=replace(DEFAULT_MODE, is_ipynb=True), + lines=(), ) # __BLACK_STDIN_FILENAME__ should have been stripped report.done.assert_called_with(expected, black.Changed.YES) @@ -1941,6 +1944,88 @@ def test_equivalency_ast_parse_failure_includes_error(self) -> None: err.match("invalid character") err.match(r"\(, line 1\)") + def test_line_ranges_with_code_option(self) -> None: + code = textwrap.dedent("""\ + if a == b: + print ( "OK" ) + """) + args = ["--line-ranges=1-1", "--code", code] + result = CliRunner().invoke(black.main, args) + + expected = textwrap.dedent("""\ + if a == b: + print ( "OK" ) + """) + self.compare_results(result, expected, expected_exit_code=0) + + def test_line_ranges_with_stdin(self) -> None: + code = textwrap.dedent("""\ + if a == b: + print ( "OK" ) + """) + runner = BlackRunner() + result = runner.invoke( + black.main, ["--line-ranges=1-1", "-"], input=BytesIO(code.encode("utf-8")) + ) + + expected = textwrap.dedent("""\ + if a == b: + print ( "OK" ) + """) + self.compare_results(result, expected, expected_exit_code=0) + + def test_line_ranges_with_source(self) -> None: + with TemporaryDirectory() as workspace: + test_file = Path(workspace) / "test.py" + test_file.write_text( + textwrap.dedent("""\ + if a == b: + print ( "OK" ) + """), + encoding="utf-8", + ) + args = ["--line-ranges=1-1", str(test_file)] + result = CliRunner().invoke(black.main, args) + assert not result.exit_code + + formatted = test_file.read_text(encoding="utf-8") + expected = textwrap.dedent("""\ + if a == b: + print ( "OK" ) + """) + assert expected == formatted + + def test_line_ranges_with_multiple_sources(self) -> None: + with TemporaryDirectory() as workspace: + test1_file = Path(workspace) / "test1.py" + test1_file.write_text("", encoding="utf-8") + test2_file = Path(workspace) / "test2.py" + test2_file.write_text("", encoding="utf-8") + args = ["--line-ranges=1-1", str(test1_file), str(test2_file)] + result = CliRunner().invoke(black.main, args) + assert result.exit_code == 1 + assert "Cannot use --line-ranges to format multiple files" in result.output + + def test_line_ranges_with_ipynb(self) -> None: + with TemporaryDirectory() as workspace: + test_file = Path(workspace) / "test.ipynb" + test_file.write_text("{}", encoding="utf-8") + args = ["--line-ranges=1-1", "--ipynb", str(test_file)] + result = CliRunner().invoke(black.main, args) + assert "Cannot use --line-ranges with ipynb files" in result.output + assert result.exit_code == 1 + + def test_line_ranges_in_pyproject_toml(self) -> None: + config = THIS_DIR / "data" / "invalid_line_ranges.toml" + result = BlackRunner().invoke( + black.main, ["--code", "print()", "--config", str(config)] + ) + assert result.exit_code == 2 + assert result.stderr_bytes is not None + assert ( + b"Cannot use line-ranges in the pyproject.toml file." in result.stderr_bytes + ) + class TestCaching: def test_get_cache_dir( diff --git a/tests/test_format.py b/tests/test_format.py index 4e863c6c54b..6c2eca8c618 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -29,13 +29,19 @@ def check_file(subdir: str, filename: str, *, data: bool = True) -> None: args.mode, fast=args.fast, minimum_version=args.minimum_version, + lines=args.lines, ) if args.minimum_version is not None: major, minor = args.minimum_version target_version = TargetVersion[f"PY{major}{minor}"] mode = replace(args.mode, target_versions={target_version}) assert_format( - source, expected, mode, fast=args.fast, minimum_version=args.minimum_version + source, + expected, + mode, + fast=args.fast, + minimum_version=args.minimum_version, + lines=args.lines, ) @@ -45,6 +51,24 @@ def test_simple_format(filename: str) -> None: check_file("cases", filename) +@pytest.mark.parametrize("filename", all_data_cases("line_ranges_formatted")) +def test_line_ranges_line_by_line(filename: str) -> None: + args, source, expected = read_data_with_mode("line_ranges_formatted", filename) + assert ( + source == expected + ), "Test cases in line_ranges_formatted must already be formatted." + line_count = len(source.splitlines()) + for line in range(1, line_count + 1): + assert_format( + source, + expected, + args.mode, + fast=args.fast, + minimum_version=args.minimum_version, + lines=[(line, line)], + ) + + # =============== # # Unusual cases # =============== # diff --git a/tests/test_ranges.py b/tests/test_ranges.py new file mode 100644 index 00000000000..d9fa9171a7f --- /dev/null +++ b/tests/test_ranges.py @@ -0,0 +1,185 @@ +"""Test the black.ranges module.""" + +from typing import List, Tuple + +import pytest + +from black.ranges import adjusted_lines + + +@pytest.mark.parametrize( + "lines", + [[(1, 1)], [(1, 3)], [(1, 1), (3, 4)]], +) +def test_no_diff(lines: List[Tuple[int, int]]) -> None: + source = """\ +import re + +def func(): +pass +""" + assert lines == adjusted_lines(lines, source, source) + + +@pytest.mark.parametrize( + "lines", + [ + [(1, 0)], + [(-8, 0)], + [(-8, 8)], + [(1, 100)], + [(2, 1)], + [(0, 8), (3, 1)], + ], +) +def test_invalid_lines(lines: List[Tuple[int, int]]) -> None: + original_source = """\ +import re +def foo(arg): +'''This is the foo function. + +This is foo function's +docstring with more descriptive texts. +''' + +def func(arg1, +arg2, arg3): +pass +""" + modified_source = """\ +import re +def foo(arg): +'''This is the foo function. + +This is foo function's +docstring with more descriptive texts. +''' + +def func(arg1, arg2, arg3): +pass +""" + assert not adjusted_lines(lines, original_source, modified_source) + + +@pytest.mark.parametrize( + "lines,adjusted", + [ + ( + [(1, 1)], + [(1, 1)], + ), + ( + [(1, 2)], + [(1, 1)], + ), + ( + [(1, 6)], + [(1, 2)], + ), + ( + [(6, 6)], + [], + ), + ], +) +def test_removals( + lines: List[Tuple[int, int]], adjusted: List[Tuple[int, int]] +) -> None: + original_source = """\ +1. first line +2. second line +3. third line +4. fourth line +5. fifth line +6. sixth line +""" + modified_source = """\ +2. second line +5. fifth line +""" + assert adjusted == adjusted_lines(lines, original_source, modified_source) + + +@pytest.mark.parametrize( + "lines,adjusted", + [ + ( + [(1, 1)], + [(2, 2)], + ), + ( + [(1, 2)], + [(2, 5)], + ), + ( + [(2, 2)], + [(5, 5)], + ), + ], +) +def test_additions( + lines: List[Tuple[int, int]], adjusted: List[Tuple[int, int]] +) -> None: + original_source = """\ +1. first line +2. second line +""" + modified_source = """\ +this is added +1. first line +this is added +this is added +2. second line +this is added +""" + assert adjusted == adjusted_lines(lines, original_source, modified_source) + + +@pytest.mark.parametrize( + "lines,adjusted", + [ + ( + [(1, 11)], + [(1, 10)], + ), + ( + [(1, 12)], + [(1, 11)], + ), + ( + [(10, 10)], + [(9, 9)], + ), + ([(1, 1), (9, 10)], [(1, 1), (9, 9)]), + ([(9, 10), (1, 1)], [(1, 1), (9, 9)]), + ], +) +def test_diffs(lines: List[Tuple[int, int]], adjusted: List[Tuple[int, int]]) -> None: + original_source = """\ + 1. import re + 2. def foo(arg): + 3. '''This is the foo function. + 4. + 5. This is foo function's + 6. docstring with more descriptive texts. + 7. ''' + 8. + 9. def func(arg1, +10. arg2, arg3): +11. pass +12. # last line +""" + modified_source = """\ + 1. import re # changed + 2. def foo(arg): + 3. '''This is the foo function. + 4. + 5. This is foo function's + 6. docstring with more descriptive texts. + 7. ''' + 8. + 9. def func(arg1, arg2, arg3): +11. pass +12. # last line changed +""" + assert adjusted == adjusted_lines(lines, original_source, modified_source) diff --git a/tests/util.py b/tests/util.py index a31ae0992c2..c8699d335ab 100644 --- a/tests/util.py +++ b/tests/util.py @@ -8,13 +8,14 @@ from dataclasses import dataclass, field, replace from functools import partial from pathlib import Path -from typing import Any, Iterator, List, Optional, Tuple +from typing import Any, Collection, Iterator, List, Optional, Tuple import black from black.const import DEFAULT_LINE_LENGTH from black.debug import DebugVisitor from black.mode import TargetVersion from black.output import diff, err, out +from black.ranges import parse_line_ranges from . import conftest @@ -44,6 +45,7 @@ class TestCaseArgs: mode: black.Mode = field(default_factory=black.Mode) fast: bool = False minimum_version: Optional[Tuple[int, int]] = None + lines: Collection[Tuple[int, int]] = () def _assert_format_equal(expected: str, actual: str) -> None: @@ -93,6 +95,7 @@ def assert_format( *, fast: bool = False, minimum_version: Optional[Tuple[int, int]] = None, + lines: Collection[Tuple[int, int]] = (), ) -> None: """Convenience function to check that Black formats as expected. @@ -101,7 +104,7 @@ def assert_format( separate from TargetVerson Mode configuration. """ _assert_format_inner( - source, expected, mode, fast=fast, minimum_version=minimum_version + source, expected, mode, fast=fast, minimum_version=minimum_version, lines=lines ) # For both preview and non-preview tests, ensure that Black doesn't crash on @@ -113,6 +116,7 @@ def assert_format( replace(mode, preview=not mode.preview), fast=fast, minimum_version=minimum_version, + lines=lines, ) except Exception as e: text = "non-preview" if mode.preview else "preview" @@ -129,6 +133,7 @@ def assert_format( replace(mode, preview=False, line_length=1), fast=fast, minimum_version=minimum_version, + lines=lines, ) except Exception as e: raise FormatFailure( @@ -143,8 +148,9 @@ def _assert_format_inner( *, fast: bool = False, minimum_version: Optional[Tuple[int, int]] = None, + lines: Collection[Tuple[int, int]] = (), ) -> None: - actual = black.format_str(source, mode=mode) + actual = black.format_str(source, mode=mode, lines=lines) if expected is not None: _assert_format_equal(expected, actual) # It's not useful to run safety checks if we're expecting no changes anyway. The @@ -156,7 +162,7 @@ def _assert_format_inner( # when checking modern code on older versions. if minimum_version is None or sys.version_info >= minimum_version: black.assert_equivalent(source, actual) - black.assert_stable(source, actual, mode=mode) + black.assert_stable(source, actual, mode=mode, lines=lines) def dump_to_stderr(*output: str) -> str: @@ -239,6 +245,7 @@ def get_flags_parser() -> argparse.ArgumentParser: " version works correctly." ), ) + parser.add_argument("--line-ranges", action="append") return parser @@ -254,7 +261,13 @@ def parse_mode(flags_line: str) -> TestCaseArgs: magic_trailing_comma=not args.skip_magic_trailing_comma, preview=args.preview, ) - return TestCaseArgs(mode=mode, fast=args.fast, minimum_version=args.minimum_version) + if args.line_ranges: + lines = parse_line_ranges(args.line_ranges) + else: + lines = [] + return TestCaseArgs( + mode=mode, fast=args.fast, minimum_version=args.minimum_version, lines=lines + ) def read_data_from_file(file_name: Path) -> Tuple[TestCaseArgs, str, str]: @@ -267,6 +280,12 @@ def read_data_from_file(file_name: Path) -> Tuple[TestCaseArgs, str, str]: for line in lines: if not _input and line.startswith("# flags: "): mode = parse_mode(line[len("# flags: ") :]) + if mode.lines: + # Retain the `# flags: ` line when using --line-ranges=. This requires + # the `# output` section to also include this line, but retaining the + # line is important to make the line ranges match what you see in the + # test file. + result.append(line) continue line = line.replace(EMPTY_LINE, "") if line.rstrip() == "# output": From 50ed6221d97b265025abaa66116a7b185f2df5e2 Mon Sep 17 00:00:00 2001 From: rdrll <13176405+rdrll@users.noreply.github.com> Date: Tue, 7 Nov 2023 06:31:58 -0800 Subject: [PATCH 183/279] Fix long case blocks not split into multiple lines (#4024) Co-authored-by: Jelle Zijlstra --- CHANGES.md | 2 + src/black/linegen.py | 24 +++++++++++- src/black/mode.py | 1 + tests/data/cases/pattern_matching_extras.py | 22 +---------- .../cases/preview_pattern_matching_long.py | 34 ++++++++++++++++ ...preview_pattern_matching_trailing_comma.py | 39 +++++++++++++++++++ 6 files changed, 101 insertions(+), 21 deletions(-) create mode 100644 tests/data/cases/preview_pattern_matching_long.py create mode 100644 tests/data/cases/preview_pattern_matching_trailing_comma.py diff --git a/CHANGES.md b/CHANGES.md index 780a00247ce..c8ba83b5ae9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -25,6 +25,8 @@ indented less (#3964) - Multiline list and dict unpacking as the sole argument to a function is now also indented less (#3992) +- Fix a bug where long `case` blocks were not split into multiple lines. Also enable + general trailing comma rules on `case` blocks (#4024) - Keep requiring two empty lines between module-level docstring and first function or class definition. (#4028) diff --git a/src/black/linegen.py b/src/black/linegen.py index b13b95d9b31..30cfff3e846 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -1229,7 +1229,7 @@ def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None: leaf.prefix = "" -def normalize_invisible_parens( +def normalize_invisible_parens( # noqa: C901 node: Node, parens_after: Set[str], *, mode: Mode, features: Collection[Feature] ) -> None: """Make existing optional parentheses invisible or create new ones. @@ -1260,6 +1260,17 @@ def normalize_invisible_parens( child, parens_after=parens_after, mode=mode, features=features ) + # Fixes a bug where invisible parens are not properly wrapped around + # case blocks. + if ( + isinstance(child, Node) + and child.type == syms.case_block + and Preview.long_case_block_line_splitting in mode + ): + normalize_invisible_parens( + child, parens_after={"case"}, mode=mode, features=features + ) + # Add parentheses around long tuple unpacking in assignments. if ( index == 0 @@ -1305,6 +1316,17 @@ def normalize_invisible_parens( # invisible parentheses to work more precisely. continue + elif ( + isinstance(child, Leaf) + and child.next_sibling is not None + and child.next_sibling.type == token.COLON + and child.value == "case" + and Preview.long_case_block_line_splitting in mode + ): + # A special patch for "case case:" scenario, the second occurrence + # of case will be not parsed as a Python keyword. + break + elif not (isinstance(child, Leaf) and is_multiline_string(child)): wrap_in_parentheses(node, child, visible=False) diff --git a/src/black/mode.py b/src/black/mode.py index 4e4effffb86..1aa5cbecc86 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -193,6 +193,7 @@ class Preview(Enum): hug_parens_with_braces_and_square_brackets = auto() allow_empty_first_line_before_new_block_or_comment = auto() single_line_format_skip_with_multiple_comments = auto() + long_case_block_line_splitting = auto() class Deprecated(UserWarning): diff --git a/tests/data/cases/pattern_matching_extras.py b/tests/data/cases/pattern_matching_extras.py index 1e1481d7bbe..1aef8f16b5a 100644 --- a/tests/data/cases/pattern_matching_extras.py +++ b/tests/data/cases/pattern_matching_extras.py @@ -30,22 +30,6 @@ def func(match: case, case: match) -> case: ... -match maybe, multiple: - case perhaps, 5: - pass - case perhaps, 6,: - pass - - -match more := (than, one), indeed,: - case _, (5, 6): - pass - case [[5], (6)], [7],: - pass - case _: - pass - - match a, *b, c: case [*_]: assert "seq" == _ @@ -67,12 +51,12 @@ def func(match: case, case: match) -> case: ), ): pass - case [a as match]: pass - case case: pass + case something: + pass match match: @@ -98,10 +82,8 @@ def func(match: case, case: match) -> case: match something: case 1 as a: pass - case 2 as b, 3 as c: pass - case 4 as d, (5 as e), (6 | 7 as g), *h: pass diff --git a/tests/data/cases/preview_pattern_matching_long.py b/tests/data/cases/preview_pattern_matching_long.py new file mode 100644 index 00000000000..df849fdc4f2 --- /dev/null +++ b/tests/data/cases/preview_pattern_matching_long.py @@ -0,0 +1,34 @@ +# flags: --preview --minimum-version=3.10 +match x: + case "abcd" | "abcd" | "abcd" : + pass + case "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd" | "abcd": + pass + case xxxxxxxxxxxxxxxxxxxxxxx: + pass + +# output + +match x: + case "abcd" | "abcd" | "abcd": + pass + case ( + "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + | "abcd" + ): + pass + case xxxxxxxxxxxxxxxxxxxxxxx: + pass diff --git a/tests/data/cases/preview_pattern_matching_trailing_comma.py b/tests/data/cases/preview_pattern_matching_trailing_comma.py new file mode 100644 index 00000000000..e6c0d88bb80 --- /dev/null +++ b/tests/data/cases/preview_pattern_matching_trailing_comma.py @@ -0,0 +1,39 @@ +# flags: --preview --minimum-version=3.10 +match maybe, multiple: + case perhaps, 5: + pass + case perhaps, 6,: + pass + + +match more := (than, one), indeed,: + case _, (5, 6): + pass + case [[5], (6)], [7],: + pass + case _: + pass + + +# output + +match maybe, multiple: + case perhaps, 5: + pass + case ( + perhaps, + 6, + ): + pass + + +match more := (than, one), indeed,: + case _, (5, 6): + pass + case ( + [[5], (6)], + [7], + ): + pass + case _: + pass \ No newline at end of file From 66008fda5dc07f5626e5f5d0dcefc476a9c12ab8 Mon Sep 17 00:00:00 2001 From: Henri Holopainen Date: Tue, 7 Nov 2023 21:29:24 +0200 Subject: [PATCH 184/279] [563] Fix standalone comments inside complex blocks crashing Black (#4016) Bracket depth is not an accurate indicator of standalone comment position inside more complex blocks because bracket depth can be virtual (in loops' and lambdas' parameter blocks) or from optional parens. Here we try to stop cumulating lines upon standalone comments in complex blocks, and try to make standalone comment processing more simple. The fundamental idea is, that if we have a standalone comment, it needs to go on its own line, so we always have to split. This is not perfect, but at least a first step. --- CHANGES.md | 1 + src/black/brackets.py | 7 ++ src/black/linegen.py | 4 +- src/black/lines.py | 27 ++++- tests/data/cases/comments_in_blocks.py | 111 ++++++++++++++++++ ..._parens_with_braces_and_square_brackets.py | 20 ++++ 6 files changed, 163 insertions(+), 7 deletions(-) create mode 100644 tests/data/cases/comments_in_blocks.py diff --git a/CHANGES.md b/CHANGES.md index c8ba83b5ae9..d30622b7786 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,6 +13,7 @@ - Fix crash on formatting bytes strings that look like docstrings (#4003) - Fix crash when whitespace followed a backslash before newline in a docstring (#4008) +- Fix standalone comments inside complex blocks crashing Black (#4016) - Fix crash on formatting code like `await (a ** b)` (#3994) diff --git a/src/black/brackets.py b/src/black/brackets.py index 85dac6edd1e..3020cc0d390 100644 --- a/src/black/brackets.py +++ b/src/black/brackets.py @@ -127,6 +127,13 @@ def mark(self, leaf: Leaf) -> None: self.maybe_increment_lambda_arguments(leaf) self.maybe_increment_for_loop_variable(leaf) + def any_open_for_or_lambda(self) -> bool: + """Return True if there is an open for or lambda expression on the line. + + See maybe_increment_for_loop_variable and maybe_increment_lambda_arguments + for details.""" + return bool(self._for_loop_depths or self._lambda_argument_depths) + def any_open_brackets(self) -> bool: """Return True if there is an yet unmatched open bracket on the line.""" return bool(self.bracket_match) diff --git a/src/black/linegen.py b/src/black/linegen.py index 30cfff3e846..e2c961d7a01 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -861,8 +861,6 @@ def _maybe_split_omitting_optional_parens( # it's not an import (optional parens are the only thing we can split on # in this case; attempting a split without them is a waste of time) and not line.is_import - # there are no standalone comments in the body - and not rhs.body.contains_standalone_comments(0) # and we can actually remove the parens and can_omit_invisible_parens(rhs, mode.line_length) ): @@ -1181,7 +1179,7 @@ def standalone_comment_split( line: Line, features: Collection[Feature], mode: Mode ) -> Iterator[Line]: """Split standalone comments from the rest of the line.""" - if not line.contains_standalone_comments(0): + if not line.contains_standalone_comments(): raise CannotSplit("Line does not have any standalone comments") current_line = Line( diff --git a/src/black/lines.py b/src/black/lines.py index 23c1a93d3d4..f0cf25ba3e7 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -1,6 +1,5 @@ import itertools import math -import sys from dataclasses import dataclass, field from typing import ( Callable, @@ -103,7 +102,10 @@ def append_safe(self, leaf: Leaf, preformatted: bool = False) -> None: Raises ValueError when any `leaf` is appended after a standalone comment or when a standalone comment is not the first leaf on the line. """ - if self.bracket_tracker.depth == 0: + if ( + self.bracket_tracker.depth == 0 + or self.bracket_tracker.any_open_for_or_lambda() + ): if self.is_comment: raise ValueError("cannot append to standalone comments") @@ -233,10 +235,10 @@ def is_fmt_pass_converted( leaf.fmt_pass_converted_first_leaf ) - def contains_standalone_comments(self, depth_limit: int = sys.maxsize) -> bool: + def contains_standalone_comments(self) -> bool: """If so, needs to be split before emitting.""" for leaf in self.leaves: - if leaf.type == STANDALONE_COMMENT and leaf.bracket_depth <= depth_limit: + if leaf.type == STANDALONE_COMMENT: return True return False @@ -982,6 +984,23 @@ def can_omit_invisible_parens( are too long. """ line = rhs.body + + # We need optional parens in order to split standalone comments to their own lines + # if there are no nested parens around the standalone comments + closing_bracket: Optional[Leaf] = None + for leaf in reversed(line.leaves): + if closing_bracket and leaf is closing_bracket.opening_bracket: + closing_bracket = None + if leaf.type == STANDALONE_COMMENT and not closing_bracket: + return False + if ( + not closing_bracket + and leaf.type in CLOSING_BRACKETS + and leaf.opening_bracket in line.leaves + and leaf.value + ): + closing_bracket = leaf + bt = line.bracket_tracker if not bt.delimiters: # Without delimiters the optional parentheses are useless. diff --git a/tests/data/cases/comments_in_blocks.py b/tests/data/cases/comments_in_blocks.py new file mode 100644 index 00000000000..1221139b6d8 --- /dev/null +++ b/tests/data/cases/comments_in_blocks.py @@ -0,0 +1,111 @@ +# Test cases from: +# - https://github.com/psf/black/issues/1798 +# - https://github.com/psf/black/issues/1499 +# - https://github.com/psf/black/issues/1211 +# - https://github.com/psf/black/issues/563 + +( + lambda + # a comment + : None +) + +( + lambda: + # b comment + None +) + +( + lambda + # a comment + : + # b comment + None +) + +[ + x + # Let's do this + for + # OK? + x + # Some comment + # And another + in + # One more + y +] + +return [ + (offers[offer_index], 1.0) + for offer_index, _ + # avoid returning any offers that don't match the grammar so + # that the return values here are consistent with what would be + # returned in AcceptValidHeader + in self._parse_and_normalize_offers(offers) +] + +from foo import ( + bar, + # qux +) + + +def convert(collection): + # replace all variables by integers + replacement_dict = { + variable: f"{index}" + for index, variable + # 0 is reserved as line terminator + in enumerate(collection.variables(), start=1) + } + + +{ + i: i + for i + # a comment + in range(5) +} + + +def get_subtree_proof_nodes( + chunk_index_groups: Sequence[Tuple[int, ...], ...], +) -> Tuple[int, ...]: + subtree_node_paths = ( + # We take a candidate element from each group and shift it to + # remove the bits that are not common to other group members, then + # we convert it to a tree path that all elements from this group + # have in common. + chunk_index + for chunk_index, bits_to_truncate + # Each group will contain an even "power-of-two" number of# elements. + # This tells us how many tailing bits each element has# which need to + # be truncated to get the group's common prefix. + in ((group[0], (len(group) - 1).bit_length()) for group in chunk_index_groups) + ) + return subtree_node_paths + + +if ( + # comment1 + a + # comment2 + or ( + # comment3 + ( + # comment4 + b + ) + # comment5 + and + # comment6 + c + or ( + # comment7 + d + ) + ) +): + print("Foo") diff --git a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py index 51fe516add5..97b5b2e8dd1 100644 --- a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py +++ b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py @@ -152,6 +152,16 @@ def foo_square_brackets(request): foo(**{x: y for x, y in enumerate(["long long long long line","long long long long line"])}) +for foo in ["a", "b"]: + output.extend([ + individual + for + # Foobar + container in xs_by_y[foo] + # Foobar + for individual in container["nested"] + ]) + # output def foo_brackets(request): return JsonResponse({ @@ -323,3 +333,13 @@ def foo_square_brackets(request): foo(**{ x: y for x, y in enumerate(["long long long long line", "long long long long line"]) }) + +for foo in ["a", "b"]: + output.extend([ + individual + for + # Foobar + container in xs_by_y[foo] + # Foobar + for individual in container["nested"] + ]) From 2e4fac9d87615e904a49e46a9cab2293e0b13126 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Tue, 7 Nov 2023 11:31:44 -0800 Subject: [PATCH 185/279] Apply force exclude logic before symlink resolution (#4015) --- CHANGES.md | 2 +- src/black/__init__.py | 24 ++++++++++++++---------- tests/test_black.py | 17 +++++++++++++++++ 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index d30622b7786..17882194aad 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -34,7 +34,7 @@ ### Configuration - Add support for single-line format skip with other comments on the same line (#3959) - +- Consistently apply force exclusion logic before resolving symlinks (#4015) - Fix a bug in the matching of absolute path names in `--include` (#3976) ### Packaging diff --git a/src/black/__init__.py b/src/black/__init__.py index 5aca3316df0..2455e8648fc 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -682,7 +682,19 @@ def get_sources( path = Path(s) is_stdin = False + # Compare the logic here to the logic in `gen_python_files`. if is_stdin or path.is_file(): + root_relative_path = path.absolute().relative_to(root).as_posix() + + root_relative_path = "/" + root_relative_path + + # Hard-exclude any files that matches the `--force-exclude` regex. + if path_is_excluded(root_relative_path, force_exclude): + report.path_ignored( + path, "matches the --force-exclude regular expression" + ) + continue + normalized_path: Optional[str] = normalize_path_maybe_ignore( path, root, report ) @@ -690,16 +702,6 @@ def get_sources( if verbose: out(f'Skipping invalid source: "{normalized_path}"', fg="red") continue - if verbose: - out(f'Found input source: "{normalized_path}"', fg="blue") - - normalized_path = "/" + normalized_path - # Hard-exclude any files that matches the `--force-exclude` regex. - if path_is_excluded(normalized_path, force_exclude): - report.path_ignored( - path, "matches the --force-exclude regular expression" - ) - continue if is_stdin: path = Path(f"{STDIN_PLACEHOLDER}{str(path)}") @@ -709,6 +711,8 @@ def get_sources( ): continue + if verbose: + out(f'Found input source: "{normalized_path}"', fg="blue") sources.add(path) elif path.is_dir(): path = root / (path.resolve().relative_to(root)) diff --git a/tests/test_black.py b/tests/test_black.py index c9819742425..899cbeb111d 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -2637,6 +2637,23 @@ def test_get_sources_with_stdin_filename_and_force_exclude(self) -> None: stdin_filename=stdin_filename, ) + @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None)) + def test_get_sources_with_stdin_filename_and_force_exclude_and_symlink( + self, + ) -> None: + # Force exclude should exclude a symlink based on the symlink, not its target + path = THIS_DIR / "data" / "include_exclude_tests" + stdin_filename = str(path / "symlink.py") + expected = [f"__BLACK_STDIN_FILENAME__{stdin_filename}"] + target = path / "b/exclude/a.py" + with patch("pathlib.Path.resolve", return_value=target): + assert_collected_sources( + src=["-"], + expected=expected, + force_exclude=r"exclude/a\.py", + stdin_filename=stdin_filename, + ) + class TestDeFactoAPI: """Test that certain symbols that are commonly used externally keep working. From f4c7be5445c87d9af5eba3d12faea62d2635e3d8 Mon Sep 17 00:00:00 2001 From: Abdenour Madani <61651582+Ab2nour@users.noreply.github.com> Date: Wed, 8 Nov 2023 00:40:10 +0100 Subject: [PATCH 186/279] docs: fix minor typo (#4030) Replace "E950" with "B950" From 1a7d9c2f58de1ffcbbe6d133f60f283601ba3f54 Mon Sep 17 00:00:00 2001 From: Henri Holopainen Date: Wed, 8 Nov 2023 06:19:32 +0200 Subject: [PATCH 187/279] Preserve visible quote types for f-string debug expressions (#4005) Co-authored-by: Jelle Zijlstra --- CHANGES.md | 2 + src/black/trans.py | 21 +++++-- tests/data/cases/preview_long_strings.py | 80 ++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 17882194aad..26e4db5848b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -26,6 +26,8 @@ indented less (#3964) - Multiline list and dict unpacking as the sole argument to a function is now also indented less (#3992) +- In f-string debug expressions preserve quote types that are visible in the final + string (#4005) - Fix a bug where long `case` blocks were not split into multiple lines. Also enable general trailing comma rules on `case` blocks (#4024) - Keep requiring two empty lines between module-level docstring and first function or diff --git a/src/black/trans.py b/src/black/trans.py index a3f6467cc9e..ab3197fa6df 100644 --- a/src/black/trans.py +++ b/src/black/trans.py @@ -590,11 +590,22 @@ def make_naked(string: str, string_prefix: str) -> str: """ assert_is_leaf_string(string) if "f" in string_prefix: - string = _toggle_fexpr_quotes(string, QUOTE) - # After quotes toggling, quotes in expressions won't be escaped - # because quotes can't be reused in f-strings. So we can simply - # let the escaping logic below run without knowing f-string - # expressions. + f_expressions = ( + string[span[0] + 1 : span[1] - 1] # +-1 to get rid of curly braces + for span in iter_fexpr_spans(string) + ) + debug_expressions_contain_visible_quotes = any( + re.search(r".*[\'\"].*(? Date: Wed, 8 Nov 2023 06:21:33 +0200 Subject: [PATCH 188/279] Remove redundant condition from `has_magic_trailing_comma` (#4023) The second `if` cannot be true at its execution point, because it is already covered by the first `if`. The condition `comma.parent.type == syms.subscriptlist` always holds if `closing.parent.type == syms.trailer` holds, because `subscriptlist` only appears inside `trailer` in the grammar: ``` trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME subscriptlist: (subscript|star_expr) (',' (subscript|star_expr))* [','] ``` --- src/black/lines.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/src/black/lines.py b/src/black/lines.py index f0cf25ba3e7..3ade0a5f4a5 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -353,9 +353,9 @@ def has_magic_trailing_comma( if closing.type == token.RSQB: if ( - closing.parent + closing.parent is not None and closing.parent.type == syms.trailer - and closing.opening_bracket + and closing.opening_bracket is not None and is_one_sequence_between( closing.opening_bracket, closing, @@ -365,22 +365,7 @@ def has_magic_trailing_comma( ): return False - if not ensure_removable: - return True - - comma = self.leaves[-1] - if comma.parent is None: - return False - return ( - comma.parent.type != syms.subscriptlist - or closing.opening_bracket is None - or not is_one_sequence_between( - closing.opening_bracket, - closing, - self.leaves, - brackets=(token.LSQB, token.RSQB), - ) - ) + return True if self.is_import: return True From 2a1c67e0b2f81df602ec1f6e7aeb030b9709dc7c Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 7 Nov 2023 20:44:46 -0800 Subject: [PATCH 189/279] Prepare release 23.11.0 (#4032) --- CHANGES.md | 70 +++------------------ docs/integrations/source_version_control.md | 4 +- docs/usage_and_configuration/the_basics.md | 6 +- 3 files changed, 14 insertions(+), 66 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 26e4db5848b..b565d510a71 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,78 +1,49 @@ # Change Log -## Unreleased +## 23.11.0 ### Highlights - - - Support formatting ranges of lines with the new `--line-ranges` command-line option - (#4020). + (#4020) ### Stable style - Fix crash on formatting bytes strings that look like docstrings (#4003) - Fix crash when whitespace followed a backslash before newline in a docstring (#4008) - Fix standalone comments inside complex blocks crashing Black (#4016) - - Fix crash on formatting code like `await (a ** b)` (#3994) - - No longer treat leading f-strings as docstrings. This matches Python's behaviour and fixes a crash (#4019) ### Preview style -- Multiline dictionaries and lists that are the sole argument to a function are now - indented less (#3964) -- Multiline list and dict unpacking as the sole argument to a function is now also +- Multiline dicts and lists that are the sole argument to a function are now indented + less (#3964) +- Multiline unpacked dicts and lists as the sole argument to a function are now also indented less (#3992) -- In f-string debug expressions preserve quote types that are visible in the final - string (#4005) +- In f-string debug expressions, quote types that are visible in the final string are + now preserved (#4005) - Fix a bug where long `case` blocks were not split into multiple lines. Also enable general trailing comma rules on `case` blocks (#4024) - Keep requiring two empty lines between module-level docstring and first function or - class definition. (#4028) + class definition (#4028) +- Add support for single-line format skip with other comments on the same line (#3959) ### Configuration -- Add support for single-line format skip with other comments on the same line (#3959) - Consistently apply force exclusion logic before resolving symlinks (#4015) - Fix a bug in the matching of absolute path names in `--include` (#3976) -### Packaging - - - -### Parser - - - ### Performance - - - Fix mypyc builds on arm64 on macOS (#4017) -### Output - - - -### _Blackd_ - - - ### Integrations - - - Black's pre-commit integration will now run only on git hooks appropriate for a code formatter (#3940) -### Documentation - - - ## 23.10.1 ### Highlights @@ -327,8 +298,6 @@ versions separately. ### Stable style - - - Introduce the 2023 stable style, which incorporates most aspects of last year's preview style (#3418). Specific changes: - Enforce empty lines before classes and functions with sticky leading comments @@ -362,8 +331,6 @@ versions separately. ### Preview style - - - Format hex codes in unicode escape sequences in string literals (#2916) - Add parentheses around `if`-`else` expressions (#2278) - Improve performance on large expressions that contain many strings (#3467) @@ -394,15 +361,11 @@ versions separately. ### Configuration - - - Black now tries to infer its `--target-version` from the project metadata specified in `pyproject.toml` (#3219) ### Packaging - - - Upgrade mypyc from `0.971` to `0.991` so mypycified _Black_ can be built on armv7 (#3380) - This also fixes some crashes while using compiled Black with a debug build of @@ -415,8 +378,6 @@ versions separately. ### Output - - - Calling `black --help` multiple times will return the same help contents each time (#3516) - Verbose logging now shows the values of `pyproject.toml` configuration variables @@ -426,25 +387,18 @@ versions separately. ### Integrations - - - Move 3.11 CI to normal flow now that all dependencies support 3.11 (#3446) - Docker: Add new `latest_prerelease` tag automation to follow latest black alpha release on docker images (#3465) ### Documentation - - - Expand `vim-plug` installation instructions to offer more explicit options (#3468) ## 22.12.0 ### Preview style - - - Enforce empty lines before classes and functions with sticky leading comments (#3302) - Reformat empty and whitespace-only files as either an empty file (if no newline is present) or as a single newline character (if a newline is present) (#3348) @@ -457,8 +411,6 @@ versions separately. ### Configuration - - - Fix incorrectly applied `.gitignore` rules by considering the `.gitignore` location and the relative path to the target file (#3338) - Fix incorrectly ignoring `.gitignore` presence when more than one source directory is @@ -466,16 +418,12 @@ versions separately. ### Parser - - - Parsing support has been added for walruses inside generator expression that are passed as function args (for example, `any(match := my_re.match(text) for text in texts)`) (#3327). ### Integrations - - - Vim plugin: Optionally allow using the system installation of Black via `let g:black_use_virtualenv = 0`(#3309) diff --git a/docs/integrations/source_version_control.md b/docs/integrations/source_version_control.md index 597a6b993c7..3c7ef89918f 100644 --- a/docs/integrations/source_version_control.md +++ b/docs/integrations/source_version_control.md @@ -8,7 +8,7 @@ Use [pre-commit](https://pre-commit.com/). Once you repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.10.1 + rev: 23.11.0 hooks: - id: black # It is recommended to specify the latest version of Python @@ -35,7 +35,7 @@ include Jupyter Notebooks. To use this hook, simply replace the hook's `id: blac repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.10.1 + rev: 23.11.0 hooks: - id: black-jupyter # It is recommended to specify the latest version of Python diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index dbd8c7ba434..6e7ee584cf9 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -211,8 +211,8 @@ configuration file for consistent results across environments. ```console $ black --version -black, 23.10.1 (compiled: yes) -$ black --required-version 23.10.1 -c "format = 'this'" +black, 23.11.0 (compiled: yes) +$ black --required-version 23.11.0 -c "format = 'this'" format = "this" $ black --required-version 31.5b2 -c "still = 'beta?!'" Oh no! 💥 💔 💥 The required version does not match the running version! @@ -303,7 +303,7 @@ You can check the version of _Black_ you have installed using the `--version` fl ```console $ black --version -black, 23.10.1 +black, 23.11.0 ``` #### `--config` From 58f31a70efe6509ce8213afac998bc5d5bb7e34d Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 7 Nov 2023 22:10:35 -0800 Subject: [PATCH 190/279] Add new release template --- CHANGES.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index b565d510a71..9446927b8d1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,52 @@ # Change Log +## Unreleased + +### Highlights + + + +### Stable style + + + +### Preview style + + + +### Configuration + + + +### Packaging + + + +### Parser + + + +### Performance + + + +### Output + + + +### _Blackd_ + + + +### Integrations + + + +### Documentation + + + ## 23.11.0 ### Highlights From 1b6b0bfcac37428f7f2eb6c97fd0a25628324db7 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 17 Nov 2023 15:57:00 +0000 Subject: [PATCH 191/279] Improve annotations for `black.concurrency.cancel` (#4047) --- src/black/concurrency.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/black/concurrency.py b/src/black/concurrency.py index 55c96b66c86..ff0a8f5fd32 100644 --- a/src/black/concurrency.py +++ b/src/black/concurrency.py @@ -38,7 +38,7 @@ def maybe_install_uvloop() -> None: pass -def cancel(tasks: Iterable["asyncio.Task[Any]"]) -> None: +def cancel(tasks: Iterable["asyncio.Future[Any]"]) -> None: """asyncio signal handler that cancels all `tasks` and reports to stderr.""" err("Aborted!") for task in tasks: From 5773d5cd2b532da185808f974a5875ca09064e28 Mon Sep 17 00:00:00 2001 From: cobalt <61329810+RedGuy12@users.noreply.github.com> Date: Fri, 17 Nov 2023 15:39:44 -0600 Subject: [PATCH 192/279] Document target version inference (#4048) --- docs/usage_and_configuration/the_basics.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 6e7ee584cf9..546fdc474e8 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -67,6 +67,10 @@ In a [configuration file](#configuration-via-a-file), you can write: target-version = ["py38", "py39", "py310", "py311"] ``` +By default, Black will infer target versions from the project metadata in +`pyproject.toml`, specifically the `[project.requires-python]` field. If this does not +yield conclusive results, Black will use per-file auto-detection. + _Black_ uses this option to decide what grammar to use to parse your code. In addition, it may use it to decide what style to use. For example, support for a trailing comma after `*args` in a function call was added in Python 3.5, so _Black_ will add this comma From 85b1c71a3445f32860d7b139ae4de4824f6ae102 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sat, 18 Nov 2023 19:15:07 +0000 Subject: [PATCH 193/279] Block aiohttp==3.9.0 from being installed in CI on Windows/pypy (#4051) --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c0302d2302a..bea8e77ba04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,7 +76,8 @@ dynamic = ["readme", "version"] colorama = ["colorama>=0.4.3"] uvloop = ["uvloop>=0.15.2"] d = [ - "aiohttp>=3.7.4", + "aiohttp>=3.7.4; sys_platform != 'win32' or implementation_name != 'pypy'", + "aiohttp>=3.7.4, !=3.9.0; sys_platform == 'win32' and implementation_name == 'pypy'", ] jupyter = [ "ipython>=7.8.0", From c4cd200a063a4d5546b547809aa1e607f03c3f59 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sat, 18 Nov 2023 19:41:46 +0000 Subject: [PATCH 194/279] Make flake8 pass when run with Python 3.12 (#4050) --- src/black/parsing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/black/parsing.py b/src/black/parsing.py index ea282d1805c..178a7ef10e2 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -175,7 +175,7 @@ def stringify_ast(node: ast.AST, depth: int = 0) -> Iterator[str]: except AttributeError: continue - yield f"{' ' * (depth+1)}{field}=" + yield f"{' ' * (depth + 1)}{field}=" if isinstance(value, list): for item in value: @@ -211,6 +211,6 @@ def stringify_ast(node: ast.AST, depth: int = 0) -> Iterator[str]: normalized = value.rstrip() else: normalized = value - yield f"{' ' * (depth+2)}{normalized!r}, # {value.__class__.__name__}" + yield f"{' ' * (depth + 2)}{normalized!r}, # {value.__class__.__name__}" yield f"{' ' * depth}) # /{node.__class__.__name__}" From d93a942a79762484a0f72c6fa271b45ec377009b Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sat, 18 Nov 2023 19:42:36 +0000 Subject: [PATCH 195/279] Upgrade mypy to 1.6.1 (#4049) --- .pre-commit-config.yaml | 2 +- CHANGES.md | 2 +- pyproject.toml | 4 ++-- scripts/check_pre_commit_rev_in_example.py | 2 +- scripts/check_version_in_basics_example.py | 2 +- scripts/diff_shades_gha_helper.py | 2 +- scripts/fuzz.py | 2 +- scripts/make_width_table.py | 2 +- src/blackd/__init__.py | 4 +--- tests/optional.py | 10 +++++++++- 10 files changed, 19 insertions(+), 13 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 623e661ac07..c153746b621 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: exclude: ^src/blib2to3/ - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.5.1 + rev: v1.6.1 hooks: - id: mypy exclude: ^docs/conf.py diff --git a/CHANGES.md b/CHANGES.md index 9446927b8d1..13b6c7bdb21 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,7 +20,7 @@ ### Packaging - +- Upgrade to mypy 1.6.1 (#4049) ### Parser diff --git a/pyproject.toml b/pyproject.toml index bea8e77ba04..f8f5155e898 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -121,7 +121,7 @@ macos-max-compat = true enable-by-default = false dependencies = [ "hatch-mypyc>=0.16.0", - "mypy==1.5.1", + "mypy==1.6.1", "click==8.1.3", # avoid https://github.com/pallets/click/issues/2558 ] require-runtime-dependencies = true @@ -187,7 +187,7 @@ CC = "clang" build-frontend = { name = "build", args = ["--no-isolation"] } # Unfortunately, hatch doesn't respect MACOSX_DEPLOYMENT_TARGET before-build = [ - "python -m pip install 'hatchling==1.18.0' hatch-vcs hatch-fancy-pypi-readme 'hatch-mypyc>=0.16.0' 'mypy==1.5.1' 'click==8.1.3'", + "python -m pip install 'hatchling==1.18.0' hatch-vcs hatch-fancy-pypi-readme 'hatch-mypyc>=0.16.0' 'mypy==1.6.1' 'click==8.1.3'", """sed -i '' -e "600,700s/'10_16'/os.environ['MACOSX_DEPLOYMENT_TARGET'].replace('.', '_')/" $(python -c 'import hatchling.builders.wheel as h; print(h.__file__)') """, ] diff --git a/scripts/check_pre_commit_rev_in_example.py b/scripts/check_pre_commit_rev_in_example.py index 107c6444dca..cc45a31e1ed 100644 --- a/scripts/check_pre_commit_rev_in_example.py +++ b/scripts/check_pre_commit_rev_in_example.py @@ -14,7 +14,7 @@ import commonmark import yaml -from bs4 import BeautifulSoup # type: ignore[import] +from bs4 import BeautifulSoup # type: ignore[import-untyped] def main(changes: str, source_version_control: str) -> None: diff --git a/scripts/check_version_in_basics_example.py b/scripts/check_version_in_basics_example.py index 0f42bafe334..c90fdb43da5 100644 --- a/scripts/check_version_in_basics_example.py +++ b/scripts/check_version_in_basics_example.py @@ -8,7 +8,7 @@ import sys import commonmark -from bs4 import BeautifulSoup # type: ignore[import] +from bs4 import BeautifulSoup # type: ignore[import-untyped] def main(changes: str, the_basics: str) -> None: diff --git a/scripts/diff_shades_gha_helper.py b/scripts/diff_shades_gha_helper.py index 895516deb51..8cd8ba155ce 100644 --- a/scripts/diff_shades_gha_helper.py +++ b/scripts/diff_shades_gha_helper.py @@ -119,7 +119,7 @@ def main() -> None: @main.command("config", help="Acquire run configuration and metadata.") @click.argument("event", type=click.Choice(["push", "pull_request"])) def config(event: Literal["push", "pull_request"]) -> None: - import diff_shades # type: ignore[import] + import diff_shades # type: ignore[import-not-found] if event == "push": jobs = [{"mode": "preview-changes", "force-flag": "--force-preview-style"}] diff --git a/scripts/fuzz.py b/scripts/fuzz.py index 929d3eac4f5..018b66e0ea2 100644 --- a/scripts/fuzz.py +++ b/scripts/fuzz.py @@ -80,7 +80,7 @@ def test_idempotent_any_syntatically_valid_python( try: import sys - import atheris # type: ignore[import] + import atheris # type: ignore[import-not-found] except ImportError: pass else: diff --git a/scripts/make_width_table.py b/scripts/make_width_table.py index 3c7cae60f7f..1b53c1b2cc9 100644 --- a/scripts/make_width_table.py +++ b/scripts/make_width_table.py @@ -20,7 +20,7 @@ from os.path import basename, dirname, join from typing import Iterable, Tuple -import wcwidth # type: ignore[import] +import wcwidth # type: ignore[import-not-found] def make_width_table() -> Iterable[Tuple[int, int, int]]: diff --git a/src/blackd/__init__.py b/src/blackd/__init__.py index 972f24181cb..6b0f3d33295 100644 --- a/src/blackd/__init__.py +++ b/src/blackd/__init__.py @@ -74,9 +74,7 @@ def main(bind_host: str, bind_port: int) -> None: app = make_app() ver = black.__version__ black.out(f"blackd version {ver} listening on {bind_host} port {bind_port}") - # TODO: aiohttp had an incorrect annotation for `print` argument, - # It'll be fixed once aiohttp releases that code - web.run_app(app, host=bind_host, port=bind_port, handle_signals=True, print=None) # type: ignore[arg-type] + web.run_app(app, host=bind_host, port=bind_port, handle_signals=True, print=None) def make_app() -> web.Application: diff --git a/tests/optional.py b/tests/optional.py index 3f5277b6b03..70ee823e316 100644 --- a/tests/optional.py +++ b/tests/optional.py @@ -26,7 +26,15 @@ from pytest import StashKey except ImportError: # pytest < 7 - from _pytest.store import StoreKey as StashKey # type: ignore[import, no-redef] + # + # "isort: skip" is needed or it moves the "type: ignore" to the following line + # because of the line length, and then mypy complains. + # Of course, adding the "isort: skip" means that + # flake8-bugbear then also complains about the line length, + # so we *also* need a "noqa" comment for good measure :) + from _pytest.store import ( # type: ignore[import-not-found, no-redef] # isort: skip # noqa: B950 + StoreKey as StashKey, + ) log = logging.getLogger(__name__) From 11da02da72ed437facde3658bb61ddebce21e7a4 Mon Sep 17 00:00:00 2001 From: Yilei Yang Date: Sat, 18 Nov 2023 11:47:05 -0800 Subject: [PATCH 196/279] Handle more huggable immediately nested parens/brackets. (#4012) Fixes #4011 --- CHANGES.md | 3 + docs/conf.py | 36 +++--- docs/the_black_code_style/future_style.md | 17 ++- src/black/cache.py | 6 +- src/black/handle_ipynb_magics.py | 54 ++++---- src/black/linegen.py | 69 ++++++++-- ..._parens_with_braces_and_square_brackets.py | 122 ++++++++++++++++-- .../cases/preview_long_strings__regression.py | 15 +-- 8 files changed, 233 insertions(+), 89 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 13b6c7bdb21..29f037b4767 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,9 @@ +- Additional cases of immediately nested tuples, lists, and dictionaries are now + indented less (#4012) + ### Configuration diff --git a/docs/conf.py b/docs/conf.py index 6b645435325..52a849d06a4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -149,15 +149,13 @@ def make_pypi_svg(version: str) -> None: # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). -latex_documents = [ - ( - master_doc, - "black.tex", - "Documentation for Black", - "Łukasz Langa and contributors to Black", - "manual", - ) -] +latex_documents = [( + master_doc, + "black.tex", + "Documentation for Black", + "Łukasz Langa and contributors to Black", + "manual", +)] # -- Options for manual page output ------------------------------------------ @@ -172,17 +170,15 @@ def make_pypi_svg(version: str) -> None: # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) -texinfo_documents = [ - ( - master_doc, - "Black", - "Documentation for Black", - author, - "Black", - "The uncompromising Python code formatter", - "Miscellaneous", - ) -] +texinfo_documents = [( + master_doc, + "Black", + "Documentation for Black", + author, + "Black", + "The uncompromising Python code formatter", + "Miscellaneous", +)] # -- Options for Epub output ------------------------------------------------- diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index 944ffad033e..428bd87ab50 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -116,8 +116,7 @@ my_dict = { ### Improved multiline dictionary and list indentation for sole function parameter For better readability and less verticality, _Black_ now pairs parentheses ("(", ")") -with braces ("{", "}") and square brackets ("[", "]") on the same line for single -parameter function calls. For example: +with braces ("{", "}") and square brackets ("[", "]") on the same line. For example: ```python foo( @@ -127,6 +126,14 @@ foo( 3, ] ) + +nested_array = [ + [ + 1, + 2, + 3, + ] +] ``` will be changed to: @@ -137,6 +144,12 @@ foo([ 2, 3, ]) + +nested_array = [[ + 1, + 2, + 3, +]] ``` This also applies to list and dictionary unpacking: diff --git a/src/black/cache.py b/src/black/cache.py index 6a332304981..6baa096baca 100644 --- a/src/black/cache.py +++ b/src/black/cache.py @@ -124,9 +124,9 @@ def filtered_cached(self, sources: Iterable[Path]) -> Tuple[Set[Path], Set[Path] def write(self, sources: Iterable[Path]) -> None: """Update the cache file data and write a new cache file.""" - self.file_data.update(**{ - str(src.resolve()): Cache.get_file_data(src) for src in sources - }) + self.file_data.update( + **{str(src.resolve()): Cache.get_file_data(src) for src in sources} + ) try: CACHE_DIR.mkdir(parents=True, exist_ok=True) with tempfile.NamedTemporaryFile( diff --git a/src/black/handle_ipynb_magics.py b/src/black/handle_ipynb_magics.py index 55ef2267df8..5b2847cb0c4 100644 --- a/src/black/handle_ipynb_magics.py +++ b/src/black/handle_ipynb_magics.py @@ -17,36 +17,30 @@ from black.output import out from black.report import NothingChanged -TRANSFORMED_MAGICS = frozenset( - ( - "get_ipython().run_cell_magic", - "get_ipython().system", - "get_ipython().getoutput", - "get_ipython().run_line_magic", - ) -) -TOKENS_TO_IGNORE = frozenset( - ( - "ENDMARKER", - "NL", - "NEWLINE", - "COMMENT", - "DEDENT", - "UNIMPORTANT_WS", - "ESCAPED_NL", - ) -) -PYTHON_CELL_MAGICS = frozenset( - ( - "capture", - "prun", - "pypy", - "python", - "python3", - "time", - "timeit", - ) -) +TRANSFORMED_MAGICS = frozenset(( + "get_ipython().run_cell_magic", + "get_ipython().system", + "get_ipython().getoutput", + "get_ipython().run_line_magic", +)) +TOKENS_TO_IGNORE = frozenset(( + "ENDMARKER", + "NL", + "NEWLINE", + "COMMENT", + "DEDENT", + "UNIMPORTANT_WS", + "ESCAPED_NL", +)) +PYTHON_CELL_MAGICS = frozenset(( + "capture", + "prun", + "pypy", + "python", + "python3", + "time", + "timeit", +)) TOKEN_HEX = secrets.token_hex diff --git a/src/black/linegen.py b/src/black/linegen.py index e2c961d7a01..8a2cd4710b9 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -817,25 +817,66 @@ def _first_right_hand_split( body_leaves.reverse() head_leaves.reverse() - if Preview.hug_parens_with_braces_and_square_brackets in line.mode: - is_unpacking = 1 if body_leaves[0].type in [token.STAR, token.DOUBLESTAR] else 0 - if ( - tail_leaves[0].type == token.RPAR - and tail_leaves[0].value - and tail_leaves[0].opening_bracket is head_leaves[-1] - and body_leaves[-1].type in [token.RBRACE, token.RSQB] - and body_leaves[-1].opening_bracket is body_leaves[is_unpacking] + body: Optional[Line] = None + if ( + Preview.hug_parens_with_braces_and_square_brackets in line.mode + and tail_leaves[0].value + and tail_leaves[0].opening_bracket is head_leaves[-1] + ): + inner_body_leaves = list(body_leaves) + hugged_opening_leaves: List[Leaf] = [] + hugged_closing_leaves: List[Leaf] = [] + is_unpacking = body_leaves[0].type in [token.STAR, token.DOUBLESTAR] + unpacking_offset: int = 1 if is_unpacking else 0 + while ( + len(inner_body_leaves) >= 2 + unpacking_offset + and inner_body_leaves[-1].type in CLOSING_BRACKETS + and inner_body_leaves[-1].opening_bracket + is inner_body_leaves[unpacking_offset] ): - head_leaves = head_leaves + body_leaves[: 1 + is_unpacking] - tail_leaves = body_leaves[-1:] + tail_leaves - body_leaves = body_leaves[1 + is_unpacking : -1] + if unpacking_offset: + hugged_opening_leaves.append(inner_body_leaves.pop(0)) + unpacking_offset = 0 + hugged_opening_leaves.append(inner_body_leaves.pop(0)) + hugged_closing_leaves.insert(0, inner_body_leaves.pop()) + + if hugged_opening_leaves and inner_body_leaves: + inner_body = bracket_split_build_line( + inner_body_leaves, + line, + hugged_opening_leaves[-1], + component=_BracketSplitComponent.body, + ) + if ( + line.mode.magic_trailing_comma + and inner_body_leaves[-1].type == token.COMMA + ): + should_hug = True + else: + line_length = line.mode.line_length - sum( + len(str(leaf)) + for leaf in hugged_opening_leaves + hugged_closing_leaves + ) + if is_line_short_enough( + inner_body, mode=replace(line.mode, line_length=line_length) + ): + # Do not hug if it fits on a single line. + should_hug = False + else: + should_hug = True + if should_hug: + body_leaves = inner_body_leaves + head_leaves.extend(hugged_opening_leaves) + tail_leaves = hugged_closing_leaves + tail_leaves + body = inner_body # No need to re-calculate the body again later. head = bracket_split_build_line( head_leaves, line, opening_bracket, component=_BracketSplitComponent.head ) - body = bracket_split_build_line( - body_leaves, line, opening_bracket, component=_BracketSplitComponent.body - ) + if body is None: + body = bracket_split_build_line( + body_leaves, line, opening_bracket, component=_BracketSplitComponent.body + ) tail = bracket_split_build_line( tail_leaves, line, opening_bracket, component=_BracketSplitComponent.tail ) diff --git a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py index 97b5b2e8dd1..9e5c9eb8546 100644 --- a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py +++ b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py @@ -128,6 +128,19 @@ def foo_square_brackets(request): func({"short line"}) func({"long line", "long long line", "long long long line", "long long long long line", "long long long long long line"}) func({{"long line", "long long line", "long long long line", "long long long long line", "long long long long long line"}}) +func(("long line", "long long line", "long long long line", "long long long long line", "long long long long long line")) +func((("long line", "long long line", "long long long line", "long long long long line", "long long long long long line"))) +func([["long line", "long long line", "long long long line", "long long long long line", "long long long long long line"]]) + +# Do not hug if the argument fits on a single line. +func({"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"}) +func(("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line")) +func(["fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"]) +func(**{"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit---"}) +func(*("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit----")) +array = [{"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"}] +array = [("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line")] +array = [["fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"]] foooooooooooooooooooo( [{c: n + 1 for c in range(256)} for n in range(100)] + [{}], {size} @@ -137,6 +150,13 @@ def foo_square_brackets(request): [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], {x}, "a string", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] ) +nested_mapping = {"key": [{"a very long key 1": "with a very long value", "a very long key 2": "with a very long value"}]} +nested_array = [[["long line", "long long line", "long long long line", "long long long long line", "long long long long long line"]]] +explicit_exploding = [[["short", "line",],],] +single_item_do_not_explode = Context({ + "version": get_docs_version(), +}) + foo(*["long long long long long line", "long long long long long line", "long long long long long line"]) foo(*[str(i) for i in range(100000000000000000000000000000000000000000000000000000000000)]) @@ -152,6 +172,9 @@ def foo_square_brackets(request): foo(**{x: y for x, y in enumerate(["long long long long line","long long long long line"])}) +# Edge case when deciding whether to hug the brackets without inner content. +very_very_very_long_variable = very_very_very_long_module.VeryVeryVeryVeryLongClassName([[]]) + for foo in ["a", "b"]: output.extend([ individual @@ -276,9 +299,9 @@ def foo_square_brackets(request): ) func([x for x in "short line"]) -func([ - x for x in "long line long line long line long line long line long line long line" -]) +func( + [x for x in "long line long line long line long line long line long line long line"] +) func([ x for x in [ @@ -295,15 +318,60 @@ def foo_square_brackets(request): "long long long long line", "long long long long long line", }) -func({ - { - "long line", - "long long line", - "long long long line", - "long long long long line", - "long long long long long line", - } -}) +func({{ + "long line", + "long long line", + "long long long line", + "long long long long line", + "long long long long long line", +}}) +func(( + "long line", + "long long line", + "long long long line", + "long long long long line", + "long long long long long line", +)) +func((( + "long line", + "long long line", + "long long long line", + "long long long long line", + "long long long long long line", +))) +func([[ + "long line", + "long long line", + "long long long line", + "long long long long line", + "long long long long long line", +]]) + +# Do not hug if the argument fits on a single line. +func( + {"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"} +) +func( + ("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line") +) +func( + ["fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"] +) +func( + **{"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit---"} +) +func( + *("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit----") +) +array = [ + {"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"} +] +array = [ + ("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line") +] +array = [ + ["fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"] +] foooooooooooooooooooo( [{c: n + 1 for c in range(256)} for n in range(100)] + [{}], {size} @@ -313,6 +381,31 @@ def foo_square_brackets(request): [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], {x}, "a string", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] ) +nested_mapping = { + "key": [{ + "a very long key 1": "with a very long value", + "a very long key 2": "with a very long value", + }] +} +nested_array = [[[ + "long line", + "long long line", + "long long long line", + "long long long long line", + "long long long long long line", +]]] +explicit_exploding = [ + [ + [ + "short", + "line", + ], + ], +] +single_item_do_not_explode = Context({ + "version": get_docs_version(), +}) + foo(*[ "long long long long long line", "long long long long long line", @@ -334,6 +427,11 @@ def foo_square_brackets(request): x: y for x, y in enumerate(["long long long long line", "long long long long line"]) }) +# Edge case when deciding whether to hug the brackets without inner content. +very_very_very_long_variable = very_very_very_long_module.VeryVeryVeryVeryLongClassName( + [[]] +) + for foo in ["a", "b"]: output.extend([ individual diff --git a/tests/data/cases/preview_long_strings__regression.py b/tests/data/cases/preview_long_strings__regression.py index 313d898cd83..5e76a8cf61c 100644 --- a/tests/data/cases/preview_long_strings__regression.py +++ b/tests/data/cases/preview_long_strings__regression.py @@ -611,14 +611,13 @@ def foo(): class A: def foo(): - XXXXXXXXXXXX.append( - ( - "xxx_xxxxxxxxxx(xxxxx={}, xxxx={}, xxxxx, xxxx_xxxx_xxxxxxxxxx={})" - .format(xxxxx, xxxx, xxxx_xxxx_xxxxxxxxxx), - my_var, - my_other_var, - ) - ) + XXXXXXXXXXXX.append(( + "xxx_xxxxxxxxxx(xxxxx={}, xxxx={}, xxxxx, xxxx_xxxx_xxxxxxxxxx={})".format( + xxxxx, xxxx, xxxx_xxxx_xxxxxxxxxx + ), + my_var, + my_other_var, + )) class A: From 80a166f2e115bda9f33d29a5ea313be2557dc7fd Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Sat, 18 Nov 2023 12:09:55 -0800 Subject: [PATCH 197/279] Make black[d] install + test run with 3.12 (#4035) * Make black[d] install + test run with 3.12 - With aiohttp >= 3.9.0 we can now install all dependencies with 3.12 - Add actions to run 3.12 - Lint still needs to be 3.11 Test: - `python3.12 -m venv /tmp/tb --upgrade-deps` - `/tmp/tb/bin/pip install tox` - `/tmp/tb/bin/pip install .[d]` - `/tmp/tb/bin/tox -e py312` ``` py312: OK (37.61=setup[3.98]+cmd[3.83,0.36,19.54,6.46,3.00,0.44] seconds) congratulations :) (37.63 seconds) ``` * Move to pypy-3.9 --------- Co-authored-by: Jelle Zijlstra --- .github/workflows/doc.yml | 2 +- .github/workflows/fuzz.yml | 2 +- .github/workflows/test.yml | 2 +- CHANGES.md | 2 +- pyproject.toml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 9a23e19cadd..fa3d87c70f5 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -26,7 +26,7 @@ jobs: - name: Set up latest Python uses: actions/setup-python@v4 with: - python-version: "3.11" + python-version: "*" - name: Install dependencies run: | diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 1b5a50c0e0b..48c26452c54 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1f33f2b814f..3f8928cc42a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,7 +31,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "pypy-3.8"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "pypy-3.9"] os: [ubuntu-latest, macOS-latest, windows-latest] steps: diff --git a/CHANGES.md b/CHANGES.md index 29f037b4767..9f7ab685afe 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -43,7 +43,7 @@ ### Integrations - +- Enable 3.12 CI (#4035) ### Documentation diff --git a/pyproject.toml b/pyproject.toml index f8f5155e898..e63e0aea3ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ [tool.black] line-length = 88 -target-version = ['py37', 'py38'] +target-version = ['py38'] include = '\.pyi?$' extend-exclude = ''' /( From 96faa3b469298573be9d3e2d55982328ee5feef9 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Sat, 18 Nov 2023 18:09:47 -0800 Subject: [PATCH 198/279] [docker] Build with 3.12 image (#4055) Test: ``` crl-m1:black cooper$ docker build --tag black_3_12 . ... => [stage-1 2/2] COPY --from=builder /opt/venv /opt/venv 0.2s => exporting to image 0.1s => => exporting layers 0.1s => => writing image sha256:bd66acc9d76d2c40d287b0684ce6601401631e0468204c4e6a81f8f1eebaf1dd 0.0s => => naming to docker.io/library/black_3_12 crl-m1:black cooper$ docker image ls | grep black_3_12 black_3_12 latest bd66acc9d76d 59 seconds ago 193MB ``` --- CHANGES.md | 1 + Dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9f7ab685afe..b3beefdd80e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -44,6 +44,7 @@ ### Integrations - Enable 3.12 CI (#4035) +- Build docker images with 3.12 (#4055) ### Documentation diff --git a/Dockerfile b/Dockerfile index bfd9acccb99..ab961a2f491 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim AS builder +FROM python:3.12-slim AS builder RUN mkdir /src COPY . /src/ @@ -12,7 +12,7 @@ RUN . /opt/venv/bin/activate && pip install --no-cache-dir --upgrade pip setupto && cd /src && hatch build -t wheel \ && pip install --no-cache-dir dist/*-cp*[colorama,d,uvloop] -FROM python:3.11-slim +FROM python:3.12-slim # copy only Python packages to limit the image size COPY --from=builder /opt/venv /opt/venv From f23b845a295f1422a1989f5ea1560ad2e0fadcdd Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Sat, 18 Nov 2023 18:11:50 -0800 Subject: [PATCH 199/279] [ci] Move 'lint' to 3.12 (#4053) - Add to run on MacOS + Windows too - Do not install [d] dependecies as blackd is not actually run / checked - Move to default GitHub action version - which is 3.12 today --- .github/workflows/lint.yml | 12 ++++++++---- tox.ini | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7fe1b04eb02..d1ad23bb2ab 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,4 +1,4 @@ -name: Lint +name: Lint + format ourselves on: [push, pull_request] @@ -11,7 +11,11 @@ jobs: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macOS-latest, windows-latest] steps: - uses: actions/checkout@v4 @@ -26,12 +30,12 @@ jobs: - name: Set up latest Python uses: actions/setup-python@v4 with: - python-version: "3.11" + python-version: "*" - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install -e '.[d]' + python -m pip install -e '.' python -m pip install tox - name: Run pre-commit hooks diff --git a/tox.ini b/tox.ini index 018cef993c0..8b4175d23fe 100644 --- a/tox.ini +++ b/tox.ini @@ -94,5 +94,5 @@ commands = setenv = PYTHONPATH = {toxinidir}/src skip_install = True commands = - pip install -e .[d] + pip install -e . black --check {toxinidir}/src {toxinidir}/tests From 30c6bb3651634ebf66177c85b27e7e277316eab7 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Sun, 19 Nov 2023 10:44:00 -0800 Subject: [PATCH 200/279] [docker ci] Split up amd64 (x86_64) and arm64 builds (#4054) * [docker ci] Split up amd64 (x86_64) and arm64 builds - Lets run them seperately to cut down total time - Will also more clearly show if either arch has specific problems - Kept amd64 (x86_64) using qemu actions so if GitHub ever offers arm64 boxes it could stay working too Fixes #3971 * Add CHANGES entry --------- Co-authored-by: Jelle Zijlstra --- .../{docker.yml => docker_amd64.yml} | 8 +-- .github/workflows/docker_arm64.yml | 69 +++++++++++++++++++ CHANGES.md | 1 + 3 files changed, 74 insertions(+), 4 deletions(-) rename .github/workflows/{docker.yml => docker_amd64.yml} (92%) create mode 100644 .github/workflows/docker_arm64.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker_amd64.yml similarity index 92% rename from .github/workflows/docker.yml rename to .github/workflows/docker_amd64.yml index ee858236fcf..846cd8c74e3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker_amd64.yml @@ -1,4 +1,4 @@ -name: docker +name: docker amd64 (x86_64) on: push: @@ -39,7 +39,7 @@ jobs: uses: docker/build-push-action@v5 with: context: . - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 push: true tags: pyfound/black:latest,pyfound/black:${{ env.GIT_TAG }} @@ -50,7 +50,7 @@ jobs: uses: docker/build-push-action@v5 with: context: . - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 push: true tags: pyfound/black:latest_release @@ -61,7 +61,7 @@ jobs: uses: docker/build-push-action@v5 with: context: . - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 push: true tags: pyfound/black:latest_prerelease diff --git a/.github/workflows/docker_arm64.yml b/.github/workflows/docker_arm64.yml new file mode 100644 index 00000000000..ddd554165af --- /dev/null +++ b/.github/workflows/docker_arm64.yml @@ -0,0 +1,69 @@ +name: docker arm64 + +on: + push: + branches: + - "main" + release: + types: [published] + +permissions: + contents: read + +jobs: + docker: + if: github.repository == 'psf/black' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Check + set version tag + run: + echo "GIT_TAG=$(git describe --candidates=0 --tags 2> /dev/null || echo + latest_non_release)" >> $GITHUB_ENV + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/arm64 + push: true + tags: pyfound/black:latest,pyfound/black:${{ env.GIT_TAG }} + + - name: Build and push latest_release tag + if: + ${{ github.event_name == 'release' && github.event.action == 'published' && + !github.event.release.prerelease }} + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/arm64 + push: true + tags: pyfound/black:latest_release + + - name: Build and push latest_prerelease tag + if: + ${{ github.event_name == 'release' && github.event.action == 'published' && + github.event.release.prerelease }} + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/arm64 + push: true + tags: pyfound/black:latest_prerelease + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/CHANGES.md b/CHANGES.md index b3beefdd80e..8d0f10a2f3a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -44,6 +44,7 @@ ### Integrations - Enable 3.12 CI (#4035) +- Build docker images in parallel (#4054) - Build docker images with 3.12 (#4055) ### Documentation From ec4a1525ee7f0daf05a2ab709123fbb0fe69e4e2 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Sun, 19 Nov 2023 11:28:00 -0800 Subject: [PATCH 201/279] [docker ci] Revert "parallel" builds in seperate actions (#4057) - Broke tagging images together - Saved only a few mins - x86_64 build is fast, time is all spent on cross compile of arm64 - Also remove evil copy pasta ... which is nice Was worth an attempt. --- .../{docker_amd64.yml => docker.yml} | 8 +-- .github/workflows/docker_arm64.yml | 69 ------------------- 2 files changed, 4 insertions(+), 73 deletions(-) rename .github/workflows/{docker_amd64.yml => docker.yml} (92%) delete mode 100644 .github/workflows/docker_arm64.yml diff --git a/.github/workflows/docker_amd64.yml b/.github/workflows/docker.yml similarity index 92% rename from .github/workflows/docker_amd64.yml rename to .github/workflows/docker.yml index 846cd8c74e3..ee858236fcf 100644 --- a/.github/workflows/docker_amd64.yml +++ b/.github/workflows/docker.yml @@ -1,4 +1,4 @@ -name: docker amd64 (x86_64) +name: docker on: push: @@ -39,7 +39,7 @@ jobs: uses: docker/build-push-action@v5 with: context: . - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 push: true tags: pyfound/black:latest,pyfound/black:${{ env.GIT_TAG }} @@ -50,7 +50,7 @@ jobs: uses: docker/build-push-action@v5 with: context: . - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 push: true tags: pyfound/black:latest_release @@ -61,7 +61,7 @@ jobs: uses: docker/build-push-action@v5 with: context: . - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 push: true tags: pyfound/black:latest_prerelease diff --git a/.github/workflows/docker_arm64.yml b/.github/workflows/docker_arm64.yml deleted file mode 100644 index ddd554165af..00000000000 --- a/.github/workflows/docker_arm64.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: docker arm64 - -on: - push: - branches: - - "main" - release: - types: [published] - -permissions: - contents: read - -jobs: - docker: - if: github.repository == 'psf/black' - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Check + set version tag - run: - echo "GIT_TAG=$(git describe --candidates=0 --tags 2> /dev/null || echo - latest_non_release)" >> $GITHUB_ENV - - - name: Build and push - uses: docker/build-push-action@v5 - with: - context: . - platforms: linux/arm64 - push: true - tags: pyfound/black:latest,pyfound/black:${{ env.GIT_TAG }} - - - name: Build and push latest_release tag - if: - ${{ github.event_name == 'release' && github.event.action == 'published' && - !github.event.release.prerelease }} - uses: docker/build-push-action@v5 - with: - context: . - platforms: linux/arm64 - push: true - tags: pyfound/black:latest_release - - - name: Build and push latest_prerelease tag - if: - ${{ github.event_name == 'release' && github.event.action == 'published' && - github.event.release.prerelease }} - uses: docker/build-push-action@v5 - with: - context: . - platforms: linux/arm64 - push: true - tags: pyfound/black:latest_prerelease - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} From 89e28ea66f50d4281cb9f624e31566aed9d5aab1 Mon Sep 17 00:00:00 2001 From: tungol Date: Mon, 20 Nov 2023 20:44:33 -0800 Subject: [PATCH 202/279] Permit standalone form feed characters at the module level (#4021) Co-authored-by: Stephen Morton Co-authored-by: Jelle Zijlstra --- CHANGES.md | 2 +- .../reference/reference_functions.rst | 4 +- docs/the_black_code_style/future_style.md | 11 + src/black/comments.py | 39 ++- src/black/linegen.py | 25 +- src/black/lines.py | 14 +- src/black/mode.py | 1 + src/black/nodes.py | 7 + src/black/output.py | 23 +- src/blib2to3/pgen2/driver.py | 2 + tests/data/cases/preview_form_feeds.py | 225 ++++++++++++++++++ 11 files changed, 318 insertions(+), 35 deletions(-) create mode 100644 tests/data/cases/preview_form_feeds.py diff --git a/CHANGES.md b/CHANGES.md index 8d0f10a2f3a..4c3fbf1afc8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,7 +12,7 @@ ### Preview style - +- Standalone form feed characters at the module level are no longer removed (#4021) - Additional cases of immediately nested tuples, lists, and dictionaries are now indented less (#4012) diff --git a/docs/contributing/reference/reference_functions.rst b/docs/contributing/reference/reference_functions.rst index dd92e37a7d4..ebadf6975a7 100644 --- a/docs/contributing/reference/reference_functions.rst +++ b/docs/contributing/reference/reference_functions.rst @@ -149,7 +149,7 @@ Utilities .. autofunction:: black.numerics.normalize_numeric_literal -.. autofunction:: black.linegen.normalize_prefix +.. autofunction:: black.comments.normalize_trailing_prefix .. autofunction:: black.strings.normalize_string_prefix @@ -168,3 +168,5 @@ Utilities .. autofunction:: black.strings.sub_twice .. autofunction:: black.nodes.whitespace + +.. autofunction:: black.nodes.make_simple_prefix diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index 428bd87ab50..f55ea5f60a9 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -296,3 +296,14 @@ s = ( # Top comment # Bottom comment ) ``` + +======= + +### Form feed characters + +_Black_ will now retain form feed characters on otherwise empty lines at the module +level. Only one form feed is retained for a group of consecutive empty lines. Where +there are two empty lines in a row, the form feed will be placed on the second line. + +_Black_ already retained form feed literals inside a comment or inside a string. This +remains the case. diff --git a/src/black/comments.py b/src/black/comments.py index 862fc7607cc..8a0e925fdc0 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -10,6 +10,7 @@ WHITESPACE, container_of, first_leaf_of, + make_simple_prefix, preceding_leaf, syms, ) @@ -44,6 +45,7 @@ class ProtoComment: value: str # content of the comment newlines: int # how many newlines before the comment consumed: int # how many characters of the original leaf's prefix did we consume + form_feed: bool # is there a form feed before the comment def generate_comments(leaf: LN) -> Iterator[Leaf]: @@ -65,8 +67,12 @@ def generate_comments(leaf: LN) -> Iterator[Leaf]: Inline comments are emitted as regular token.COMMENT leaves. Standalone are emitted with a fake STANDALONE_COMMENT token identifier. """ + total_consumed = 0 for pc in list_comments(leaf.prefix, is_endmarker=leaf.type == token.ENDMARKER): - yield Leaf(pc.type, pc.value, prefix="\n" * pc.newlines) + total_consumed = pc.consumed + prefix = make_simple_prefix(pc.newlines, pc.form_feed) + yield Leaf(pc.type, pc.value, prefix=prefix) + normalize_trailing_prefix(leaf, total_consumed) @lru_cache(maxsize=4096) @@ -79,11 +85,14 @@ def list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]: consumed = 0 nlines = 0 ignored_lines = 0 - for index, line in enumerate(re.split("\r?\n", prefix)): - consumed += len(line) + 1 # adding the length of the split '\n' - line = line.lstrip() + form_feed = False + for index, full_line in enumerate(re.split("\r?\n", prefix)): + consumed += len(full_line) + 1 # adding the length of the split '\n' + line = full_line.lstrip() if not line: nlines += 1 + if "\f" in full_line: + form_feed = True if not line.startswith("#"): # Escaped newlines outside of a comment are not really newlines at # all. We treat a single-line comment following an escaped newline @@ -99,13 +108,33 @@ def list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]: comment = make_comment(line) result.append( ProtoComment( - type=comment_type, value=comment, newlines=nlines, consumed=consumed + type=comment_type, + value=comment, + newlines=nlines, + consumed=consumed, + form_feed=form_feed, ) ) + form_feed = False nlines = 0 return result +def normalize_trailing_prefix(leaf: LN, total_consumed: int) -> None: + """Normalize the prefix that's left over after generating comments. + + Note: don't use backslashes for formatting or you'll lose your voting rights. + """ + remainder = leaf.prefix[total_consumed:] + if "\\" not in remainder: + nl_count = remainder.count("\n") + form_feed = "\f" in remainder and remainder.endswith("\n") + leaf.prefix = make_simple_prefix(nl_count, form_feed) + return + + leaf.prefix = "" + + def make_comment(content: str) -> str: """Return a consistently formatted comment from the given `content` string. diff --git a/src/black/linegen.py b/src/black/linegen.py index 8a2cd4710b9..7fbbe290d7e 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -149,7 +149,8 @@ def visit_default(self, node: LN) -> Iterator[Line]: self.current_line.append(comment) yield from self.line() - normalize_prefix(node, inside_brackets=any_open_brackets) + if any_open_brackets: + node.prefix = "" if self.mode.string_normalization and node.type == token.STRING: node.value = normalize_string_prefix(node.value) node.value = normalize_string_quotes(node.value) @@ -1035,8 +1036,6 @@ def bracket_split_build_line( result.inside_brackets = True result.depth += 1 if leaves: - # Since body is a new indent level, remove spurious leading whitespace. - normalize_prefix(leaves[0], inside_brackets=True) # Ensure a trailing comma for imports and standalone function arguments, but # be careful not to add one after any comments or within type annotations. no_commas = ( @@ -1106,7 +1105,7 @@ def split_wrapper( line: Line, features: Collection[Feature], mode: Mode ) -> Iterator[Line]: for split_line in split_func(line, features, mode): - normalize_prefix(split_line.leaves[0], inside_brackets=True) + split_line.leaves[0].prefix = "" yield split_line return split_wrapper @@ -1250,24 +1249,6 @@ def append_to_line(leaf: Leaf) -> Iterator[Line]: yield current_line -def normalize_prefix(leaf: Leaf, *, inside_brackets: bool) -> None: - """Leave existing extra newlines if not `inside_brackets`. Remove everything - else. - - Note: don't use backslashes for formatting or you'll lose your voting rights. - """ - if not inside_brackets: - spl = leaf.prefix.split("#") - if "\\" not in spl[0]: - nl_count = spl[-1].count("\n") - if len(spl) > 1: - nl_count -= 1 - leaf.prefix = "\n" * nl_count - return - - leaf.prefix = "" - - def normalize_invisible_parens( # noqa: C901 node: Node, parens_after: Set[str], *, mode: Mode, features: Collection[Feature] ) -> None: diff --git a/src/black/lines.py b/src/black/lines.py index 3ade0a5f4a5..ec6145ff848 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -31,6 +31,7 @@ is_type_comment, is_type_ignore_comment, is_with_or_async_with_stmt, + make_simple_prefix, replace_child, syms, whitespace, @@ -520,12 +521,12 @@ class LinesBlock: before: int = 0 content_lines: List[str] = field(default_factory=list) after: int = 0 + form_feed: bool = False def all_lines(self) -> List[str]: empty_line = str(Line(mode=self.mode)) - return ( - [empty_line * self.before] + self.content_lines + [empty_line * self.after] - ) + prefix = make_simple_prefix(self.before, self.form_feed, empty_line) + return [prefix] + self.content_lines + [empty_line * self.after] @dataclass @@ -550,6 +551,12 @@ def maybe_empty_lines(self, current_line: Line) -> LinesBlock: This is for separating `def`, `async def` and `class` with extra empty lines (two on module-level). """ + form_feed = ( + Preview.allow_form_feeds in self.mode + and current_line.depth == 0 + and bool(current_line.leaves) + and "\f\n" in current_line.leaves[0].prefix + ) before, after = self._maybe_empty_lines(current_line) previous_after = self.previous_block.after if self.previous_block else 0 before = ( @@ -575,6 +582,7 @@ def maybe_empty_lines(self, current_line: Line) -> LinesBlock: original_line=current_line, before=before, after=after, + form_feed=form_feed, ) # Maintain the semantic_leading_comment state. diff --git a/src/black/mode.py b/src/black/mode.py index 1aa5cbecc86..04038f49627 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -194,6 +194,7 @@ class Preview(Enum): allow_empty_first_line_before_new_block_or_comment = auto() single_line_format_skip_with_multiple_comments = auto() long_case_block_line_splitting = auto() + allow_form_feeds = auto() class Deprecated(UserWarning): diff --git a/src/black/nodes.py b/src/black/nodes.py index 9251b0defb0..de53f8e36a3 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -407,6 +407,13 @@ def whitespace(leaf: Leaf, *, complex_subscript: bool, mode: Mode) -> str: # no return SPACE +def make_simple_prefix(nl_count: int, form_feed: bool, empty_line: str = "\n") -> str: + """Generate a normalized prefix string.""" + if form_feed: + return (empty_line * (nl_count - 1)) + "\f" + empty_line + return empty_line * nl_count + + def preceding_leaf(node: Optional[LN]) -> Optional[Leaf]: """Return the first leaf that precedes `node`, if any.""" while node: diff --git a/src/black/output.py b/src/black/output.py index f4c17f28ea4..7c7dd0fe14e 100644 --- a/src/black/output.py +++ b/src/black/output.py @@ -4,8 +4,9 @@ """ import json +import re import tempfile -from typing import Any, Optional +from typing import Any, List, Optional from click import echo, style from mypy_extensions import mypyc_attr @@ -55,12 +56,28 @@ def ipynb_diff(a: str, b: str, a_name: str, b_name: str) -> str: return "".join(diff_lines) +_line_pattern = re.compile(r"(.*?(?:\r\n|\n|\r|$))") + + +def _splitlines_no_ff(source: str) -> List[str]: + """Split a string into lines ignoring form feed and other chars. + + This mimics how the Python parser splits source code. + + A simplified version of the function with the same name in Lib/ast.py + """ + result = [match[0] for match in _line_pattern.finditer(source)] + if result[-1] == "": + result.pop(-1) + return result + + def diff(a: str, b: str, a_name: str, b_name: str) -> str: """Return a unified diff string between strings `a` and `b`.""" import difflib - a_lines = a.splitlines(keepends=True) - b_lines = b.splitlines(keepends=True) + a_lines = _splitlines_no_ff(a) + b_lines = _splitlines_no_ff(b) diff_lines = [] for line in difflib.unified_diff( a_lines, b_lines, fromfile=a_name, tofile=b_name, n=5 diff --git a/src/blib2to3/pgen2/driver.py b/src/blib2to3/pgen2/driver.py index e629843f8b9..be3984437a8 100644 --- a/src/blib2to3/pgen2/driver.py +++ b/src/blib2to3/pgen2/driver.py @@ -222,6 +222,8 @@ def _partially_consume_prefix(self, prefix: str, column: int) -> Tuple[str, str] elif char == "\n": # unexpected empty line current_column = 0 + elif char == "\f": + current_column = 0 else: # indent is finished wait_for_nl = True diff --git a/tests/data/cases/preview_form_feeds.py b/tests/data/cases/preview_form_feeds.py new file mode 100644 index 00000000000..2d8653a1f04 --- /dev/null +++ b/tests/data/cases/preview_form_feeds.py @@ -0,0 +1,225 @@ +# flags: --preview + + +# Warning! This file contains form feeds (ASCII 0x0C, often represented by \f or ^L). +# These may be invisible in your editor: ensure you can see them before making changes here. + +# There's one at the start that'll get stripped + +# Comment and statement processing is different enough that we'll test variations of both +# contexts here + +# + + +# + + +# + + + +# + + + +# + + + +# + + +# + + + +# + +# + +# + + \ +# +pass + +pass + + +pass + + +pass + + + +pass + + + +pass + + + +pass + + +pass + + + +pass + +pass + +pass + + +# form feed after a dedent +def foo(): + pass + +pass + + +# form feeds are prohibited inside blocks, or on a line with nonwhitespace + def bar( a = 1 ,b : bool = False ) : + + + pass + + +class Baz: + + def __init__(self): + pass + + + def something(self): + pass + + + +# +pass +pass # +a = 1 + # + pass + a = 1 + +a = [ + +] + +# as internal whitespace of a comment is allowed but why +"form feed literal in a string is okay " + +# form feeds at the very end get removed. + + + +# output + +# Warning! This file contains form feeds (ASCII 0x0C, often represented by \f or ^L). +# These may be invisible in your editor: ensure you can see them before making changes here. + +# There's one at the start that'll get stripped + +# Comment and statement processing is different enough that we'll test variations of both +# contexts here + +# + + +# + + +# + + +# + + +# + + +# + + +# + + +# + +# + +# + +# +pass + +pass + + +pass + + +pass + + +pass + + +pass + + +pass + + +pass + + +pass + +pass + +pass + + +# form feed after a dedent +def foo(): + pass + + +pass + + +# form feeds are prohibited inside blocks, or on a line with nonwhitespace +def bar(a=1, b: bool = False): + pass + + +class Baz: + def __init__(self): + pass + + def something(self): + pass + + +# +pass +pass # +a = 1 +# +pass +a = 1 + +a = [] + +# as internal whitespace of a comment is allowed but why +"form feed literal in a string is okay " + +# form feeds at the very end get removed. From a8062983cd1f8ac8859297c870847906b10cf6a2 Mon Sep 17 00:00:00 2001 From: Yilei Yang Date: Mon, 20 Nov 2023 20:45:39 -0800 Subject: [PATCH 203/279] Disable the stability check with --line-ranges for now. (#4034) Co-authored-by: Jelle Zijlstra --- CHANGES.md | 3 ++ docs/usage_and_configuration/the_basics.md | 6 ++++ src/black/__init__.py | 9 ++++-- .../data/cases/line_ranges_diff_edge_case.py | 28 +++++++++++++++++++ 4 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 tests/data/cases/line_ranges_diff_edge_case.py diff --git a/CHANGES.md b/CHANGES.md index 4c3fbf1afc8..18bab5131e6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -21,6 +21,9 @@ +- `--line-ranges` now skips _Black_'s internal stability check in `--safe` mode. This + avoids a crash on rare inputs that have many unformatted same-content lines. (#4034) + ### Packaging - Upgrade to mypy 1.6.1 (#4049) diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 546fdc474e8..0c1a4d3b5a1 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -196,6 +196,12 @@ Example: `black --line-ranges=1-10 --line-ranges=21-30 test.py` will format line This option is mainly for editor integrations, such as "Format Selection". +```{note} +Due to [#4052](https://github.com/psf/black/issues/4052), `--line-ranges` might format +extra lines outside of the ranges when ther are unformatted lines with the exact +content. It also disables _Black_'s formatting stability check in `--safe` mode. +``` + #### `--color` / `--no-color` Show (or do not show) colored diff. Only applies when `--diff` is given. diff --git a/src/black/__init__.py b/src/black/__init__.py index 2455e8648fc..b33beeeeb23 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1465,11 +1465,16 @@ def assert_stable( src: str, dst: str, mode: Mode, *, lines: Collection[Tuple[int, int]] = () ) -> None: """Raise AssertionError if `dst` reformats differently the second time.""" + if lines: + # Formatting specified lines requires `adjusted_lines` to map original lines + # to the formatted lines before re-formatting the previously formatted result. + # Due to less-ideal diff algorithm, some edge cases produce incorrect new line + # ranges. Hence for now, we skip the stable check. + # See https://github.com/psf/black/issues/4033 for context. + return # We shouldn't call format_str() here, because that formats the string # twice and may hide a bug where we bounce back and forth between two # versions. - if lines: - lines = adjusted_lines(lines, src, dst) newdst = _format_str_once(dst, mode=mode, lines=lines) if dst != newdst: log = dump_to_file( diff --git a/tests/data/cases/line_ranges_diff_edge_case.py b/tests/data/cases/line_ranges_diff_edge_case.py new file mode 100644 index 00000000000..f5cb2d0bb5f --- /dev/null +++ b/tests/data/cases/line_ranges_diff_edge_case.py @@ -0,0 +1,28 @@ +# flags: --line-ranges=10-11 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# Reproducible example for https://github.com/psf/black/issues/4033. +# This can be fixed in the future if we use a better diffing algorithm, or make Black +# perform formatting in a single pass. + +print ( "format me" ) +print ( "format me" ) +print ( "format me" ) +print ( "format me" ) +print ( "format me" ) + +# output +# flags: --line-ranges=10-11 +# NOTE: If you need to modify this file, pay special attention to the --line-ranges= +# flag above as it's formatting specifically these lines. + +# Reproducible example for https://github.com/psf/black/issues/4033. +# This can be fixed in the future if we use a better diffing algorithm, or make Black +# perform formatting in a single pass. + +print ( "format me" ) +print("format me") +print("format me") +print("format me") +print("format me") From be336bb67fb6c12667836f7fba4993f9be9c61dd Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 20 Nov 2023 22:33:16 -0800 Subject: [PATCH 204/279] Run lint job on Ubuntu only (#4061) --- .github/workflows/lint.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d1ad23bb2ab..9c7aca8f869 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -11,11 +11,7 @@ jobs: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macOS-latest, windows-latest] + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From fb5e5d2be6367a0402a60da94f139dfb8943ed37 Mon Sep 17 00:00:00 2001 From: Henri Holopainen Date: Thu, 23 Nov 2023 05:11:49 +0200 Subject: [PATCH 205/279] Prefer more equal signs before a break when splitting chained assignments (#4010) Fixes #4007 --- CHANGES.md | 2 +- src/black/linegen.py | 64 ++++++++++++++------ src/black/lines.py | 5 ++ tests/data/cases/preview_prefer_rhs_split.py | 21 +++++++ 4 files changed, 71 insertions(+), 21 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 18bab5131e6..6a8b97c75eb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,8 +12,8 @@ ### Preview style +- Prefer more equal signs before a break when splitting chained assignments (#4010) - Standalone form feed characters at the module level are no longer removed (#4021) - - Additional cases of immediately nested tuples, lists, and dictionaries are now indented less (#4012) diff --git a/src/black/linegen.py b/src/black/linegen.py index 7fbbe290d7e..7152568783e 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -910,24 +910,32 @@ def _maybe_split_omitting_optional_parens( try: # The RHSResult Omitting Optional Parens. rhs_oop = _first_right_hand_split(line, omit=omit) - if not ( + prefer_splitting_rhs_mode = ( Preview.prefer_splitting_right_hand_side_of_assignments in line.mode - # the split is right after `=` - and len(rhs.head.leaves) >= 2 - and rhs.head.leaves[-2].type == token.EQUAL - # the left side of assignment contains brackets - and any(leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1]) - # the left side of assignment is short enough (the -1 is for the ending - # optional paren) - and is_line_short_enough( - rhs.head, mode=replace(mode, line_length=mode.line_length - 1) + ) + is_split_right_after_equal = ( + len(rhs.head.leaves) >= 2 and rhs.head.leaves[-2].type == token.EQUAL + ) + rhs_head_contains_brackets = any( + leaf.type in BRACKETS for leaf in rhs.head.leaves[:-1] + ) + # the -1 is for the ending optional paren + rhs_head_short_enough = is_line_short_enough( + rhs.head, mode=replace(mode, line_length=mode.line_length - 1) + ) + rhs_head_explode_blocked_by_magic_trailing_comma = ( + rhs.head.magic_trailing_comma is None + ) + if ( + not ( + prefer_splitting_rhs_mode + and is_split_right_after_equal + and rhs_head_contains_brackets + and rhs_head_short_enough + and rhs_head_explode_blocked_by_magic_trailing_comma ) - # the left side of assignment won't explode further because of magic - # trailing comma - and rhs.head.magic_trailing_comma is None - # the split by omitting optional parens isn't preferred by some other - # reason - and not _prefer_split_rhs_oop(rhs_oop, mode) + # the omit optional parens split is preferred by some other reason + or _prefer_split_rhs_oop_over_rhs(rhs_oop, rhs, mode) ): yield from _maybe_split_omitting_optional_parens( rhs_oop, line, mode, features=features, omit=omit @@ -935,8 +943,12 @@ def _maybe_split_omitting_optional_parens( return except CannotSplit as e: - if not ( - can_be_split(rhs.body) or is_line_short_enough(rhs.body, mode=mode) + # For chained assignments we want to use the previous successful split + if line.is_chained_assignment: + pass + + elif not can_be_split(rhs.body) and not is_line_short_enough( + rhs.body, mode=mode ): raise CannotSplit( "Splitting failed, body is still too long and can't be split." @@ -960,10 +972,22 @@ def _maybe_split_omitting_optional_parens( yield result -def _prefer_split_rhs_oop(rhs_oop: RHSResult, mode: Mode) -> bool: +def _prefer_split_rhs_oop_over_rhs( + rhs_oop: RHSResult, rhs: RHSResult, mode: Mode +) -> bool: """ - Returns whether we should prefer the result from a split omitting optional parens. + Returns whether we should prefer the result from a split omitting optional parens + (rhs_oop) over the original (rhs). """ + # If we have multiple targets, we prefer more `=`s on the head vs pushing them to + # the body + rhs_head_equal_count = [leaf.type for leaf in rhs.head.leaves].count(token.EQUAL) + rhs_oop_head_equal_count = [leaf.type for leaf in rhs_oop.head.leaves].count( + token.EQUAL + ) + if rhs_head_equal_count > 1 and rhs_head_equal_count > rhs_oop_head_equal_count: + return False + has_closing_bracket_after_assign = False for leaf in reversed(rhs_oop.head.leaves): if leaf.type == token.EQUAL: diff --git a/src/black/lines.py b/src/black/lines.py index ec6145ff848..6e33ee57eab 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -209,6 +209,11 @@ def is_triple_quoted_string(self) -> bool: return True return False + @property + def is_chained_assignment(self) -> bool: + """Is the line a chained assignment""" + return [leaf.type for leaf in self.leaves].count(token.EQUAL) > 1 + @property def opens_block(self) -> bool: """Does this line open a new level of indentation.""" diff --git a/tests/data/cases/preview_prefer_rhs_split.py b/tests/data/cases/preview_prefer_rhs_split.py index c732c33b53a..28d89c368c0 100644 --- a/tests/data/cases/preview_prefer_rhs_split.py +++ b/tests/data/cases/preview_prefer_rhs_split.py @@ -84,3 +84,24 @@ ) or ( isinstance(some_other_var, BaseClass) and table.something != table.some_other_thing ) + +# Multiple targets +a = b = ( + ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc +) + +a = b = c = d = e = f = g = ( + hhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh +) = i = j = ( + kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk +) + +a = ( + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +) = c + +a = ( + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +) = ( + cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc +) = ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd From 69d49c5a6fe064eb290dd6445745fdeb2643f54f Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 24 Nov 2023 14:19:54 +0000 Subject: [PATCH 206/279] Bump mypy to 1.7.1 (#4069) --- .pre-commit-config.yaml | 2 +- CHANGES.md | 2 +- pyproject.toml | 4 ++-- src/blib2to3/pgen2/tokenize.py | 7 ++----- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c153746b621..2896489d724 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: exclude: ^src/blib2to3/ - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.6.1 + rev: v1.7.1 hooks: - id: mypy exclude: ^docs/conf.py diff --git a/CHANGES.md b/CHANGES.md index 6a8b97c75eb..f6fe69bac68 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -26,7 +26,7 @@ ### Packaging -- Upgrade to mypy 1.6.1 (#4049) +- Upgrade to mypy 1.7.1 (#4049) (#4069) ### Parser diff --git a/pyproject.toml b/pyproject.toml index e63e0aea3ef..6b681e8226a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -121,7 +121,7 @@ macos-max-compat = true enable-by-default = false dependencies = [ "hatch-mypyc>=0.16.0", - "mypy==1.6.1", + "mypy==1.7.1", "click==8.1.3", # avoid https://github.com/pallets/click/issues/2558 ] require-runtime-dependencies = true @@ -187,7 +187,7 @@ CC = "clang" build-frontend = { name = "build", args = ["--no-isolation"] } # Unfortunately, hatch doesn't respect MACOSX_DEPLOYMENT_TARGET before-build = [ - "python -m pip install 'hatchling==1.18.0' hatch-vcs hatch-fancy-pypi-readme 'hatch-mypyc>=0.16.0' 'mypy==1.6.1' 'click==8.1.3'", + "python -m pip install 'hatchling==1.18.0' hatch-vcs hatch-fancy-pypi-readme 'hatch-mypyc>=0.16.0' 'mypy==1.7.1' 'click==8.1.3'", """sed -i '' -e "600,700s/'10_16'/os.environ['MACOSX_DEPLOYMENT_TARGET'].replace('.', '_')/" $(python -c 'import hatchling.builders.wheel as h; print(h.__file__)') """, ] diff --git a/src/blib2to3/pgen2/tokenize.py b/src/blib2to3/pgen2/tokenize.py index d0607f4b1e1..b04b18ba870 100644 --- a/src/blib2to3/pgen2/tokenize.py +++ b/src/blib2to3/pgen2/tokenize.py @@ -39,7 +39,6 @@ Set, Tuple, Union, - cast, ) from blib2to3.pgen2.grammar import Grammar @@ -262,11 +261,9 @@ def add_whitespace(self, start: Coord) -> None: def untokenize(self, iterable: Iterable[TokenInfo]) -> str: for t in iterable: if len(t) == 2: - self.compat(cast(Tuple[int, str], t), iterable) + self.compat(t, iterable) break - tok_type, token, start, end, line = cast( - Tuple[int, str, Coord, Coord, str], t - ) + tok_type, token, start, end, line = t self.add_whitespace(start) self.tokens.append(token) self.prev_row, self.prev_col = end From a0e270d0f246387202e676b25abbf7a02ddcbc71 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 24 Nov 2023 18:05:59 +0000 Subject: [PATCH 207/279] Build mypycified wheels for Python 3.12 (#4070) --- .github/workflows/pypi_upload.yml | 2 +- CHANGES.md | 1 + pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index 07273f09508..bbdcdf17a8f 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -73,7 +73,7 @@ jobs: | pyp 'json.dumps({"only": x, "os": "ubuntu-latest"})' } | pyp 'json.dumps(list(map(json.loads, lines)))' > /tmp/matrix env: - CIBW_BUILD: "cp38-* cp311-*" + CIBW_BUILD: "cp38-* cp312-*" CIBW_ARCHS_LINUX: x86_64 - id: set-matrix run: echo "include=$(cat /tmp/matrix)" | tee -a $GITHUB_OUTPUT diff --git a/CHANGES.md b/CHANGES.md index f6fe69bac68..e9ffd6bb9f5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -27,6 +27,7 @@ ### Packaging - Upgrade to mypy 1.7.1 (#4049) (#4069) +- Faster compiled wheels are now available for CPython 3.12 (#4070) ### Parser diff --git a/pyproject.toml b/pyproject.toml index 6b681e8226a..1098412981a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -150,7 +150,7 @@ build-verbosity = 1 # - Architecture (64-bit only): amd64 / x86_64, universal2, and arm64 # - OS: Linux (no musl), Windows, and macOS build = "cp3*" -skip = ["*-manylinux_i686", "*-musllinux_*", "*-win32", "pp*", "cp312-*"] +skip = ["*-manylinux_i686", "*-musllinux_*", "*-win32", "pp*"] # This is the bare minimum needed to run the test suite. Pulling in the full # test_requirements.txt would download a bunch of other packages not necessary # here and would slow down the testing step a fair bit. From 66ec056e39da957d3c82da5b7a86ef228606cfe6 Mon Sep 17 00:00:00 2001 From: exag Date: Mon, 4 Dec 2023 14:47:30 +0900 Subject: [PATCH 208/279] Fix minor typos in docstrings (#4085) --- src/black/numerics.py | 2 +- src/black/ranges.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/black/numerics.py b/src/black/numerics.py index 67ac8595fcc..3040de06fde 100644 --- a/src/black/numerics.py +++ b/src/black/numerics.py @@ -14,7 +14,7 @@ def format_hex(text: str) -> str: def format_scientific_notation(text: str) -> str: - """Formats a numeric string utilizing scentific notation""" + """Formats a numeric string utilizing scientific notation""" before, after = text.split("e") sign = "" if after.startswith("-"): diff --git a/src/black/ranges.py b/src/black/ranges.py index b0c312e6274..59e19242d47 100644 --- a/src/black/ranges.py +++ b/src/black/ranges.py @@ -172,7 +172,7 @@ class _TopLevelStatementsVisitor(Visitor[None]): A node visitor that converts unchanged top-level statements to STANDALONE_COMMENT. - This is used in addition to _convert_unchanged_lines_by_flatterning, to + This is used in addition to _convert_unchanged_line_by_line, to speed up formatting when there are unchanged top-level classes/functions/statements. """ @@ -302,7 +302,7 @@ def _convert_node_to_standalone_comment(node: LN) -> None: index = node.remove() if index is not None: # Remove the '\n', as STANDALONE_COMMENT will have '\n' appended when - # genearting the formatted code. + # generating the formatted code. value = str(node)[:-1] parent.insert_child( index, From 3416b2c82d51f27ce55c31ef0bfe4a9e21816611 Mon Sep 17 00:00:00 2001 From: Riyazuddin Khan Date: Mon, 4 Dec 2023 23:40:03 +0530 Subject: [PATCH 209/279] Fix: --line-ranges dedents a # fmt: off in the middle of a decorator (#4084) Fixes #4068 --- CHANGES.md | 3 ++- src/black/__init__.py | 2 +- src/black/comments.py | 23 ++++++++++++++---- .../cases/line_ranges_fmt_off_decorator.py | 24 +++++++++++++++++-- 4 files changed, 44 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e9ffd6bb9f5..f17cd7fdc9d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,7 +8,8 @@ ### Stable style - +- Fix bug where `# fmt: off` automatically dedents when used with the `--line-ranges` + option, even when it is not within the specified line range. (#4084) ### Preview style diff --git a/src/black/__init__.py b/src/black/__init__.py index b33beeeeb23..04f6d8c58de 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1180,7 +1180,7 @@ def _format_str_once( for feature in {Feature.PARENTHESIZED_CONTEXT_MANAGERS} if supports_feature(versions, feature) } - normalize_fmt_off(src_node, mode) + normalize_fmt_off(src_node, mode, lines) if lines: # This should be called after normalize_fmt_off. convert_unchanged_lines(src_node, lines) diff --git a/src/black/comments.py b/src/black/comments.py index 8a0e925fdc0..25413121199 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -1,7 +1,7 @@ import re from dataclasses import dataclass from functools import lru_cache -from typing import Final, Iterator, List, Optional, Union +from typing import Collection, Final, Iterator, List, Optional, Tuple, Union from black.mode import Mode, Preview from black.nodes import ( @@ -161,14 +161,18 @@ def make_comment(content: str) -> str: return "#" + content -def normalize_fmt_off(node: Node, mode: Mode) -> None: +def normalize_fmt_off( + node: Node, mode: Mode, lines: Collection[Tuple[int, int]] +) -> None: """Convert content between `# fmt: off`/`# fmt: on` into standalone comments.""" try_again = True while try_again: - try_again = convert_one_fmt_off_pair(node, mode) + try_again = convert_one_fmt_off_pair(node, mode, lines) -def convert_one_fmt_off_pair(node: Node, mode: Mode) -> bool: +def convert_one_fmt_off_pair( + node: Node, mode: Mode, lines: Collection[Tuple[int, int]] +) -> bool: """Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment. Returns True if a pair was converted. @@ -213,7 +217,18 @@ def convert_one_fmt_off_pair(node: Node, mode: Mode) -> bool: prefix[:previous_consumed] + "\n" * comment.newlines ) hidden_value = "".join(str(n) for n in ignored_nodes) + comment_lineno = leaf.lineno - comment.newlines if comment.value in FMT_OFF: + fmt_off_prefix = "" + if len(lines) > 0 and not any( + comment_lineno >= line[0] and comment_lineno <= line[1] + for line in lines + ): + # keeping indentation of comment by preserving original whitespaces. + fmt_off_prefix = prefix.split(comment.value)[0] + if "\n" in fmt_off_prefix: + fmt_off_prefix = fmt_off_prefix.split("\n")[-1] + standalone_comment_prefix += fmt_off_prefix hidden_value = comment.value + "\n" + hidden_value if _contains_fmt_skip_comment(comment.value, mode): hidden_value += " " + comment.value diff --git a/tests/data/cases/line_ranges_fmt_off_decorator.py b/tests/data/cases/line_ranges_fmt_off_decorator.py index 14aa1dda02d..065bf4328d7 100644 --- a/tests/data/cases/line_ranges_fmt_off_decorator.py +++ b/tests/data/cases/line_ranges_fmt_off_decorator.py @@ -1,4 +1,4 @@ -# flags: --line-ranges=12-12 +# flags: --line-ranges=12-12 --line-ranges=21-21 # NOTE: If you need to modify this file, pay special attention to the --line-ranges= # flag above as it's formatting specifically these lines. @@ -11,9 +11,19 @@ class MyClass: def method(): print ( "str" ) + @decor( + a=1, + # fmt: off + b=(2, 3), + # fmt: on + ) + def func(): + pass + + # output -# flags: --line-ranges=12-12 +# flags: --line-ranges=12-12 --line-ranges=21-21 # NOTE: If you need to modify this file, pay special attention to the --line-ranges= # flag above as it's formatting specifically these lines. @@ -25,3 +35,13 @@ class MyClass: # fmt: on def method(): print("str") + + @decor( + a=1, + # fmt: off + b=(2, 3), + # fmt: on + ) + def func(): + pass + From 50d5756e8e63b17e4523f096f312011273ce640f Mon Sep 17 00:00:00 2001 From: John Litborn <11260241+jakkdl@users.noreply.github.com> Date: Tue, 5 Dec 2023 19:19:24 +0100 Subject: [PATCH 210/279] fix crash in preview mode with --line-length=1 (#4086) --- CHANGES.md | 1 + src/black/linegen.py | 2 +- .../return_annotation_brackets_crash_line_length_1.py | 9 +++++++++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 tests/data/cases/return_annotation_brackets_crash_line_length_1.py diff --git a/CHANGES.md b/CHANGES.md index f17cd7fdc9d..8f0b75e7f10 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,7 @@ - Standalone form feed characters at the module level are no longer removed (#4021) - Additional cases of immediately nested tuples, lists, and dictionaries are now indented less (#4012) +- Fix crash in preview mode when using a short `--line-length` (#4086) ### Configuration diff --git a/src/black/linegen.py b/src/black/linegen.py index 7152568783e..073672a5ae7 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -744,7 +744,7 @@ def left_hand_split( if leaf.type in OPENING_BRACKETS: matching_bracket = leaf current_leaves = body_leaves - if not matching_bracket: + if not matching_bracket or not tail_leaves: raise CannotSplit("No brackets found") head = bracket_split_build_line( diff --git a/tests/data/cases/return_annotation_brackets_crash_line_length_1.py b/tests/data/cases/return_annotation_brackets_crash_line_length_1.py new file mode 100644 index 00000000000..9d96b4ab97a --- /dev/null +++ b/tests/data/cases/return_annotation_brackets_crash_line_length_1.py @@ -0,0 +1,9 @@ +# flags: --preview --minimum-version=3.10 --line-length=1 + +def foo() -> tuple[int, int,]: + ... +# output +def foo() -> tuple[ + int, + int, +]: ... From e4ae213f06050e7f76ebcf01578c002e412dafdc Mon Sep 17 00:00:00 2001 From: John Litborn <11260241+jakkdl@users.noreply.github.com> Date: Wed, 6 Dec 2023 16:17:33 +0100 Subject: [PATCH 211/279] test preview cases with line-length 1 unless explicitly skipped (#4087) * Add new flag for tests, --no-preview-line-length-1, to be used for test cases known to not work in preview mode with line-length=1. Also split out the problematic cases in three cases to separate files. Removed now redundant file which explicitly tested preview annotations with line-length=1 * mode.preview -> preview_mode, mark pep_572_remove_parens as failing with ll1 --- tests/data/cases/comment_type_hint.py | 3 + tests/data/cases/comments2.py | 4 - tests/data/cases/fmtskip2.py | 5 +- tests/data/cases/pep_572_remove_parens.py | 2 +- ..._parens_with_braces_and_square_brackets.py | 96 ---------------- ..._with_braces_and_square_brackets_no_ll1.py | 106 ++++++++++++++++++ ...annotation_brackets_crash_line_length_1.py | 9 -- tests/test_format.py | 2 + tests/util.py | 54 ++++++--- 9 files changed, 154 insertions(+), 127 deletions(-) create mode 100644 tests/data/cases/comment_type_hint.py create mode 100644 tests/data/cases/preview_hug_parens_with_braces_and_square_brackets_no_ll1.py delete mode 100644 tests/data/cases/return_annotation_brackets_crash_line_length_1.py diff --git a/tests/data/cases/comment_type_hint.py b/tests/data/cases/comment_type_hint.py new file mode 100644 index 00000000000..2992da88d90 --- /dev/null +++ b/tests/data/cases/comment_type_hint.py @@ -0,0 +1,3 @@ +# flags: --no-preview-line-length-1 +# split out from comments2 as it does not work with line-length=1, losing the comment +a = "type comment with trailing space" # type: str diff --git a/tests/data/cases/comments2.py b/tests/data/cases/comments2.py index 1487dc4b6e2..261c5e9f0a0 100644 --- a/tests/data/cases/comments2.py +++ b/tests/data/cases/comments2.py @@ -155,8 +155,6 @@ def _init_host(self, parsed) -> None: pass -a = "type comment with trailing space" # type: str - ####################### ### SECTION COMMENT ### ####################### @@ -335,8 +333,6 @@ def _init_host(self, parsed) -> None: pass -a = "type comment with trailing space" # type: str - ####################### ### SECTION COMMENT ### ####################### diff --git a/tests/data/cases/fmtskip2.py b/tests/data/cases/fmtskip2.py index e6248117aa9..0189d4e642d 100644 --- a/tests/data/cases/fmtskip2.py +++ b/tests/data/cases/fmtskip2.py @@ -1,9 +1,12 @@ +# flags: --no-preview-line-length-1 +# l2 loses the comment with line-length=1 in preview mode l1 = ["This list should be broken up", "into multiple lines", "because it is way too long"] l2 = ["But this list shouldn't", "even though it also has", "way too many characters in it"] # fmt: skip l3 = ["I have", "trailing comma", "so I should be braked",] # output +# l2 loses the comment with line-length=1 in preview mode l1 = [ "This list should be broken up", "into multiple lines", @@ -14,4 +17,4 @@ "I have", "trailing comma", "so I should be braked", -] \ No newline at end of file +] diff --git a/tests/data/cases/pep_572_remove_parens.py b/tests/data/cases/pep_572_remove_parens.py index 24f1ac29168..88774d81649 100644 --- a/tests/data/cases/pep_572_remove_parens.py +++ b/tests/data/cases/pep_572_remove_parens.py @@ -1,4 +1,4 @@ -# flags: --minimum-version=3.8 +# flags: --minimum-version=3.8 --no-preview-line-length-1 if (foo := 0): pass diff --git a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py index 9e5c9eb8546..47a6a0bcae6 100644 --- a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py +++ b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py @@ -125,23 +125,6 @@ def foo_square_brackets(request): func([x for x in "long line long line long line long line long line long line long line"]) func([x for x in [x for x in "long line long line long line long line long line long line long line"]]) -func({"short line"}) -func({"long line", "long long line", "long long long line", "long long long long line", "long long long long long line"}) -func({{"long line", "long long line", "long long long line", "long long long long line", "long long long long long line"}}) -func(("long line", "long long line", "long long long line", "long long long long line", "long long long long long line")) -func((("long line", "long long line", "long long long line", "long long long long line", "long long long long long line"))) -func([["long line", "long long line", "long long long line", "long long long long line", "long long long long long line"]]) - -# Do not hug if the argument fits on a single line. -func({"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"}) -func(("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line")) -func(["fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"]) -func(**{"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit---"}) -func(*("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit----")) -array = [{"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"}] -array = [("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line")] -array = [["fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"]] - foooooooooooooooooooo( [{c: n + 1 for c in range(256)} for n in range(100)] + [{}], {size} ) @@ -151,14 +134,11 @@ def foo_square_brackets(request): ) nested_mapping = {"key": [{"a very long key 1": "with a very long value", "a very long key 2": "with a very long value"}]} -nested_array = [[["long line", "long long line", "long long long line", "long long long long line", "long long long long long line"]]] explicit_exploding = [[["short", "line",],],] single_item_do_not_explode = Context({ "version": get_docs_version(), }) -foo(*["long long long long long line", "long long long long long line", "long long long long long line"]) - foo(*[str(i) for i in range(100000000000000000000000000000000000000000000000000000000000)]) foo( @@ -310,69 +290,6 @@ def foo_square_brackets(request): ] ]) -func({"short line"}) -func({ - "long line", - "long long line", - "long long long line", - "long long long long line", - "long long long long long line", -}) -func({{ - "long line", - "long long line", - "long long long line", - "long long long long line", - "long long long long long line", -}}) -func(( - "long line", - "long long line", - "long long long line", - "long long long long line", - "long long long long long line", -)) -func((( - "long line", - "long long line", - "long long long line", - "long long long long line", - "long long long long long line", -))) -func([[ - "long line", - "long long line", - "long long long line", - "long long long long line", - "long long long long long line", -]]) - -# Do not hug if the argument fits on a single line. -func( - {"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"} -) -func( - ("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line") -) -func( - ["fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"] -) -func( - **{"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit---"} -) -func( - *("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit----") -) -array = [ - {"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"} -] -array = [ - ("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line") -] -array = [ - ["fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"] -] - foooooooooooooooooooo( [{c: n + 1 for c in range(256)} for n in range(100)] + [{}], {size} ) @@ -387,13 +304,6 @@ def foo_square_brackets(request): "a very long key 2": "with a very long value", }] } -nested_array = [[[ - "long line", - "long long line", - "long long long line", - "long long long long line", - "long long long long long line", -]]] explicit_exploding = [ [ [ @@ -406,12 +316,6 @@ def foo_square_brackets(request): "version": get_docs_version(), }) -foo(*[ - "long long long long long line", - "long long long long long line", - "long long long long long line", -]) - foo(*[ str(i) for i in range(100000000000000000000000000000000000000000000000000000000000) ]) diff --git a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets_no_ll1.py b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets_no_ll1.py new file mode 100644 index 00000000000..fdebdf69c20 --- /dev/null +++ b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets_no_ll1.py @@ -0,0 +1,106 @@ +# flags: --preview --no-preview-line-length-1 +# split out from preview_hug_parens_with_brackes_and_square_brackets, as it produces +# different code on the second pass with line-length 1 in many cases. +# Seems to be about whether the last string in a sequence gets wrapped in parens or not. +foo(*["long long long long long line", "long long long long long line", "long long long long long line"]) +func({"short line"}) +func({"long line", "long long line", "long long long line", "long long long long line", "long long long long long line"}) +func({{"long line", "long long line", "long long long line", "long long long long line", "long long long long long line"}}) +func(("long line", "long long line", "long long long line", "long long long long line", "long long long long long line")) +func((("long line", "long long line", "long long long line", "long long long long line", "long long long long long line"))) +func([["long line", "long long line", "long long long line", "long long long long line", "long long long long long line"]]) + + +# Do not hug if the argument fits on a single line. +func({"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"}) +func(("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line")) +func(["fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"]) +func(**{"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit---"}) +func(*("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit----")) +array = [{"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"}] +array = [("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line")] +array = [["fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"]] + +nested_array = [[["long line", "long long line", "long long long line", "long long long long line", "long long long long long line"]]] + +# output + +# split out from preview_hug_parens_with_brackes_and_square_brackets, as it produces +# different code on the second pass with line-length 1 in many cases. +# Seems to be about whether the last string in a sequence gets wrapped in parens or not. +foo(*[ + "long long long long long line", + "long long long long long line", + "long long long long long line", +]) +func({"short line"}) +func({ + "long line", + "long long line", + "long long long line", + "long long long long line", + "long long long long long line", +}) +func({{ + "long line", + "long long line", + "long long long line", + "long long long long line", + "long long long long long line", +}}) +func(( + "long line", + "long long line", + "long long long line", + "long long long long line", + "long long long long long line", +)) +func((( + "long line", + "long long line", + "long long long line", + "long long long long line", + "long long long long long line", +))) +func([[ + "long line", + "long long line", + "long long long line", + "long long long long line", + "long long long long long line", +]]) + + +# Do not hug if the argument fits on a single line. +func( + {"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"} +) +func( + ("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line") +) +func( + ["fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"] +) +func( + **{"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit---"} +) +func( + *("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit----") +) +array = [ + {"fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"} +] +array = [ + ("fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line") +] +array = [ + ["fit line", "fit line", "fit line", "fit line", "fit line", "fit line", "fit line"] +] + +nested_array = [[[ + "long line", + "long long line", + "long long long line", + "long long long long line", + "long long long long long line", +]]] diff --git a/tests/data/cases/return_annotation_brackets_crash_line_length_1.py b/tests/data/cases/return_annotation_brackets_crash_line_length_1.py deleted file mode 100644 index 9d96b4ab97a..00000000000 --- a/tests/data/cases/return_annotation_brackets_crash_line_length_1.py +++ /dev/null @@ -1,9 +0,0 @@ -# flags: --preview --minimum-version=3.10 --line-length=1 - -def foo() -> tuple[int, int,]: - ... -# output -def foo() -> tuple[ - int, - int, -]: ... diff --git a/tests/test_format.py b/tests/test_format.py index 6c2eca8c618..9162c585c08 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -30,6 +30,7 @@ def check_file(subdir: str, filename: str, *, data: bool = True) -> None: fast=args.fast, minimum_version=args.minimum_version, lines=args.lines, + no_preview_line_length_1=args.no_preview_line_length_1, ) if args.minimum_version is not None: major, minor = args.minimum_version @@ -42,6 +43,7 @@ def check_file(subdir: str, filename: str, *, data: bool = True) -> None: fast=args.fast, minimum_version=args.minimum_version, lines=args.lines, + no_preview_line_length_1=args.no_preview_line_length_1, ) diff --git a/tests/util.py b/tests/util.py index c8699d335ab..9ea30e62fe3 100644 --- a/tests/util.py +++ b/tests/util.py @@ -46,6 +46,7 @@ class TestCaseArgs: fast: bool = False minimum_version: Optional[Tuple[int, int]] = None lines: Collection[Tuple[int, int]] = () + no_preview_line_length_1: bool = False def _assert_format_equal(expected: str, actual: str) -> None: @@ -96,6 +97,7 @@ def assert_format( fast: bool = False, minimum_version: Optional[Tuple[int, int]] = None, lines: Collection[Tuple[int, int]] = (), + no_preview_line_length_1: bool = False, ) -> None: """Convenience function to check that Black formats as expected. @@ -124,21 +126,28 @@ def assert_format( f"Black crashed formatting this case in {text} mode." ) from e # Similarly, setting line length to 1 is a good way to catch - # stability bugs. But only in non-preview mode because preview mode - # currently has a lot of line length 1 bugs. - try: - _assert_format_inner( - source, - None, - replace(mode, preview=False, line_length=1), - fast=fast, - minimum_version=minimum_version, - lines=lines, - ) - except Exception as e: - raise FormatFailure( - "Black crashed formatting this case with line-length set to 1." - ) from e + # stability bugs. Some tests are known to be broken in preview mode with line length + # of 1 though, and have marked that with a flag --no-preview-line-length-1 + preview_modes = [False] + if not no_preview_line_length_1: + preview_modes.append(True) + + for preview_mode in preview_modes: + + try: + _assert_format_inner( + source, + None, + replace(mode, preview=preview_mode, line_length=1), + fast=fast, + minimum_version=minimum_version, + lines=lines, + ) + except Exception as e: + text = "preview" if preview_mode else "non-preview" + raise FormatFailure( + f"Black crashed formatting this case in {text} mode with line-length=1." + ) from e def _assert_format_inner( @@ -246,6 +255,15 @@ def get_flags_parser() -> argparse.ArgumentParser: ), ) parser.add_argument("--line-ranges", action="append") + parser.add_argument( + "--no-preview-line-length-1", + default=False, + action="store_true", + help=( + "Don't run in preview mode with --line-length=1, as that's known to cause a" + " crash" + ), + ) return parser @@ -266,7 +284,11 @@ def parse_mode(flags_line: str) -> TestCaseArgs: else: lines = [] return TestCaseArgs( - mode=mode, fast=args.fast, minimum_version=args.minimum_version, lines=lines + mode=mode, + fast=args.fast, + minimum_version=args.minimum_version, + lines=lines, + no_preview_line_length_1=args.no_preview_line_length_1, ) From 50e287cecea41ee32bd66ab1eee4827f6b8312ce Mon Sep 17 00:00:00 2001 From: cobalt <61329810+RedGuy12@users.noreply.github.com> Date: Thu, 7 Dec 2023 10:38:57 -0600 Subject: [PATCH 212/279] docs: Clarify include/exclude documentation (#4072) --- docs/usage_and_configuration/the_basics.md | 22 +++++++-------- src/black/__init__.py | 33 +++++++++++----------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 0c1a4d3b5a1..3739bcaefa1 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -241,24 +241,17 @@ Because of our [stability policy](../the_black_code_style/index.md), this will g stable formatting, but still allow you to take advantage of improvements that do not affect formatting. -#### `--include` - -A regular expression that matches files and directories that should be included on -recursive searches. An empty value means all files are included regardless of the name. -Use forward slashes for directories on all platforms (Windows, too). Exclusions are -calculated first, inclusions later. - #### `--exclude` A regular expression that matches files and directories that should be excluded on recursive searches. An empty value means no paths are excluded. Use forward slashes for -directories on all platforms (Windows, too). Exclusions are calculated first, inclusions -later. +directories on all platforms (Windows, too). By default, Black also ignores all paths +listed in `.gitignore`. Changing this value will override all default exclusions. #### `--extend-exclude` -Like `--exclude`, but adds additional files and directories on top of the excluded ones. -Useful if you simply want to add to the default. +Like `--exclude`, but adds additional files and directories on top of the default values +instead of overriding them. #### `--force-exclude` @@ -271,6 +264,13 @@ programmatically on changed files, such as in a pre-commit hook or editor plugin The name of the file when passing it through stdin. Useful to make sure Black will respect the `--force-exclude` option on some editors that rely on using stdin. +#### `--include` + +A regular expression that matches files and directories that should be included on +recursive searches. An empty value means all files are included regardless of the name. +Use forward slashes for directories on all platforms (Windows, too). Overrides all +exclusions, including from `.gitignore` and command line options. + #### `-W`, `--workers` When _Black_ formats multiple files, it may use a process pool to speed up formatting. diff --git a/src/black/__init__.py b/src/black/__init__.py index 04f6d8c58de..e7dac895a6a 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -344,19 +344,6 @@ def validate_regex( " either a major version number or an exact version." ), ) -@click.option( - "--include", - type=str, - default=DEFAULT_INCLUDES, - callback=validate_regex, - help=( - "A regular expression that matches files and directories that should be" - " included on recursive searches. An empty value means all files are included" - " regardless of the name. Use forward slashes for directories on all platforms" - " (Windows, too). Exclusions are calculated first, inclusions later." - ), - show_default=True, -) @click.option( "--exclude", type=str, @@ -365,8 +352,8 @@ def validate_regex( "A regular expression that matches files and directories that should be" " excluded on recursive searches. An empty value means no paths are excluded." " Use forward slashes for directories on all platforms (Windows, too)." - " Exclusions are calculated first, inclusions later. [default:" - f" {DEFAULT_EXCLUDES}]" + " By default, Black also ignores all paths listed in .gitignore. Changing this" + f" value will override all default exclusions. [default: {DEFAULT_EXCLUDES}]" ), show_default=False, ) @@ -376,7 +363,7 @@ def validate_regex( callback=validate_regex, help=( "Like --exclude, but adds additional files and directories on top of the" - " excluded ones. (Useful if you simply want to add to the default)" + " default values instead of overriding them." ), ) @click.option( @@ -398,6 +385,20 @@ def validate_regex( "editors that rely on using stdin." ), ) +@click.option( + "--include", + type=str, + default=DEFAULT_INCLUDES, + callback=validate_regex, + help=( + "A regular expression that matches files and directories that should be" + " included on recursive searches. An empty value means all files are included" + " regardless of the name. Use forward slashes for directories on all platforms" + " (Windows, too). Overrides all exclusions, including from .gitignore and" + " command line options." + ), + show_default=True, +) @click.option( "-W", "--workers", From 432d9050c3d1e35a36ffc97d4a9e0e0c9e5e4ecc Mon Sep 17 00:00:00 2001 From: cobalt <61329810+RedGuy12@users.noreply.github.com> Date: Thu, 7 Dec 2023 11:32:06 -0600 Subject: [PATCH 213/279] docs: Unify option descriptions between `--help` and `the_basics.md` (#4076) --- docs/usage_and_configuration/the_basics.md | 45 +++++++------ src/black/__init__.py | 75 +++++++++++++--------- 2 files changed, 69 insertions(+), 51 deletions(-) diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 3739bcaefa1..73c0d1323e3 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -35,6 +35,10 @@ are deliberately limited and rarely added. Note that all command-line options listed above can also be configured using a `pyproject.toml` file (more on that below). +#### `-h`, `--help` + +Show available command-line options and exit. + #### `-c`, `--code` Format the code passed in as a string. @@ -109,6 +113,10 @@ useful when piping source on standard input. When processing Jupyter Notebooks, add the given magic to the list of known python- magics. Useful for formatting cells with custom python magics. +#### `-x, --skip-source-first-line` + +Skip the first line of the source code. + #### `-S, --skip-string-normalization` By default, _Black_ uses double quotes for all strings and normalizes string prefixes, @@ -132,7 +140,7 @@ functionality in the next major release. Read more about #### `--check` -Passing `--check` will make _Black_ exit with: +Don't write the files back, just return the status. _Black_ will exit with: - code 0 if nothing would change; - code 1 if some files would be reformatted; or @@ -162,8 +170,8 @@ $ echo $? #### `--diff` -Passing `--diff` will make _Black_ print out diffs that indicate what changes _Black_ -would've made. They are printed to stdout so capturing them is simple. +Don't write the files back, just output a diff to indicate what changes _Black_ would've +made. They are printed to stdout so capturing them is simple. If you'd like colored diffs, you can enable them with `--color`. @@ -179,6 +187,10 @@ All done! ✨ 🍰 ✨ 1 file would be reformatted. ``` +#### `--color` / `--no-color` + +Show (or do not show) colored diff. Only applies when `--diff` is given. + ### `--line-ranges` When specified, _Black_ will try its best to only format these lines. @@ -202,10 +214,6 @@ extra lines outside of the ranges when ther are unformatted lines with the exact content. It also disables _Black_'s formatting stability check in `--safe` mode. ``` -#### `--color` / `--no-color` - -Show (or do not show) colored diff. Only applies when `--diff` is given. - #### `--fast` / `--safe` By default, _Black_ performs [an AST safety check](labels/ast-changes) after formatting @@ -256,7 +264,7 @@ instead of overriding them. #### `--force-exclude` Like `--exclude`, but files and directories matching this regex will be excluded even -when they are passed explicitly as arguments. This is useful when invoking _Black_ +when they are passed explicitly as arguments. This is useful when invoking Black programmatically on changed files, such as in a pre-commit hook or editor plugin. #### `--stdin-filename` @@ -275,12 +283,12 @@ exclusions, including from `.gitignore` and command line options. When _Black_ formats multiple files, it may use a process pool to speed up formatting. This option controls the number of parallel workers. This can also be specified via the -`BLACK_NUM_WORKERS` environment variable. +`BLACK_NUM_WORKERS` environment variable. Defaults to the number of CPUs in the system. #### `-q`, `--quiet` -Passing `-q` / `--quiet` will cause _Black_ to stop emitting all non-critical output. -Error messages will still be emitted (which can silenced by `2>/dev/null`). +Stop emitting all non-critical output. Error messages will still be emitted (which can +silenced by `2>/dev/null`). ```console $ black src/ -q @@ -289,9 +297,9 @@ error: cannot format src/black_primer/cli.py: Cannot parse: 5:6: mport asyncio #### `-v`, `--verbose` -Passing `-v` / `--verbose` will cause _Black_ to also emit messages about files that -were not changed or were ignored due to exclusion patterns. If _Black_ is using a -configuration file, a blue message detailing which one it is using will be emitted. +Emit messages about files that were not changed or were ignored due to exclusion +patterns. If _Black_ is using a configuration file, a message detailing which one it is +using will be emitted. ```console $ black src/ -v @@ -321,10 +329,6 @@ black, 23.11.0 Read configuration options from a configuration file. See [below](#configuration-via-a-file) for more details on the configuration file. -#### `-h`, `--help` - -Show available command-line options and exit. - ### Environment variable options _Black_ supports the following configuration via environment variables. @@ -355,7 +359,7 @@ All done! ✨ 🍰 ✨ use `--stdin-filename`. Useful to make sure _Black_ will respect the `--force-exclude` option on some editors that rely on using stdin. -You can also pass code as a string using the `-c` / `--code` option. +You can also pass code as a string using the `--code` option. ### Writeback and reporting @@ -435,8 +439,7 @@ refers to the path to your home directory. On Windows, this will be something li You can also explicitly specify the path to a particular file that you want with `--config`. In this situation _Black_ will not look for any other file. -If you're running with `--verbose`, you will see a blue message if a file was found and -used. +If you're running with `--verbose`, you will see a message if a file was found and used. Please note `blackd` will not use `pyproject.toml` configuration. diff --git a/src/black/__init__.py b/src/black/__init__.py index e7dac895a6a..5073fa748d5 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -235,25 +235,26 @@ def validate_regex( callback=target_version_option_callback, multiple=True, help=( - "Python versions that should be supported by Black's output. By default, Black" - " will try to infer this from the project metadata in pyproject.toml. If this" - " does not yield conclusive results, Black will use per-file auto-detection." + "Python versions that should be supported by Black's output. You should" + " include all versions that your code supports. By default, Black will infer" + " target versions from the project metadata in pyproject.toml. If this does" + " not yield conclusive results, Black will use per-file auto-detection." ), ) @click.option( "--pyi", is_flag=True, help=( - "Format all input files like typing stubs regardless of file extension (useful" - " when piping source on standard input)." + "Format all input files like typing stubs regardless of file extension. This" + " is useful when piping source on standard input." ), ) @click.option( "--ipynb", is_flag=True, help=( - "Format all input files like Jupyter Notebooks regardless of file extension " - "(useful when piping source on standard input)." + "Format all input files like Jupyter Notebooks regardless of file extension." + "This is useful when piping source on standard input." ), ) @click.option( @@ -310,14 +311,22 @@ def validate_regex( @click.option( "--diff", is_flag=True, - help="Don't write the files back, just output a diff for each file on stdout.", + help=( + "Don't write the files back, just output a diff to indicate what changes" + " Black would've made. They are printed to stdout so capturing them is simple." + ), +) +@click.option( + "--color/--no-color", + is_flag=True, + help="Show (or do not show) colored diff. Only applies when --diff is given.", ) @click.option( "--line-ranges", multiple=True, metavar="START-END", help=( - "When specified, _Black_ will try its best to only format these lines. This" + "When specified, Black will try its best to only format these lines. This" " option can be specified multiple times, and a union of the lines will be" " formatted. Each range must be specified as two integers connected by a `-`:" " `-`. The `` and `` integer indices are 1-based and" @@ -325,23 +334,24 @@ def validate_regex( ), default=(), ) -@click.option( - "--color/--no-color", - is_flag=True, - help="Show colored diff. Only applies when `--diff` is given.", -) @click.option( "--fast/--safe", is_flag=True, - help="If --fast given, skip temporary sanity checks. [default: --safe]", + help=( + "By default, Black performs an AST safety check after formatting your code." + " The --fast flag turns off this check and the --safe flag explicitly enables" + " it. [default: --safe]" + ), ) @click.option( "--required-version", type=str, help=( - "Require a specific version of Black to be running (useful for unifying results" - " across many environments e.g. with a pyproject.toml file). It can be" - " either a major version number or an exact version." + "Require a specific version of Black to be running. This is useful for" + " ensuring that all contributors to your project are using the same" + " version, because different versions of Black may format code a little" + " differently. This option can be set in a configuration file for consistent" + " results across environments." ), ) @click.option( @@ -371,8 +381,10 @@ def validate_regex( type=str, callback=validate_regex, help=( - "Like --exclude, but files and directories matching this regex will be " - "excluded even when they are passed explicitly as arguments." + "Like --exclude, but files and directories matching this regex will be excluded" + " even when they are passed explicitly as arguments. This is useful when" + " invoking Black programmatically on changed files, such as in a pre-commit" + " hook or editor plugin." ), ) @click.option( @@ -380,9 +392,9 @@ def validate_regex( type=str, is_eager=True, help=( - "The name of the file when passing it through stdin. Useful to make " - "sure Black will respect --force-exclude option on some " - "editors that rely on using stdin." + "The name of the file when passing it through stdin. Useful to make sure Black" + " will respect the --force-exclude option on some editors that rely on using" + " stdin." ), ) @click.option( @@ -405,8 +417,10 @@ def validate_regex( type=click.IntRange(min=1), default=None, help=( - "Number of parallel workers [default: BLACK_NUM_WORKERS environment variable " - "or number of CPUs in the system]" + "When Black formats multiple files, it may use a process pool to speed up" + " formatting. This option controls the number of parallel workers. This can" + " also be specified via the BLACK_NUM_WORKERS environment variable. Defaults" + " to the number of CPUs in the system." ), ) @click.option( @@ -414,8 +428,8 @@ def validate_regex( "--quiet", is_flag=True, help=( - "Don't emit non-error messages to stderr. Errors are still emitted; silence" - " those with 2>/dev/null." + "Stop emitting all non-critical output. Error messages will still be emitted" + " (which can silenced by 2>/dev/null)." ), ) @click.option( @@ -423,8 +437,9 @@ def validate_regex( "--verbose", is_flag=True, help=( - "Also emit messages to stderr about files that were not changed or were ignored" - " due to exclusion patterns." + "Emit messages about files that were not changed or were ignored due to" + " exclusion patterns. If Black is using a configuration file, a message" + " detailing which one it is using will be emitted." ), ) @click.version_option( @@ -455,7 +470,7 @@ def validate_regex( ), is_eager=True, callback=read_pyproject_toml, - help="Read configuration from FILE path.", + help="Read configuration options from a configuration file.", ) @click.pass_context def main( # noqa: C901 From e7e122e9ff27fc040a6e8ecd92f0e7603c87f92d Mon Sep 17 00:00:00 2001 From: cobalt <61329810+RedGuy12@users.noreply.github.com> Date: Sat, 9 Dec 2023 19:44:15 -0600 Subject: [PATCH 214/279] docs: Move `fmt: off` docs (#4090) --- docs/the_black_code_style/current_style.md | 15 +++------------ docs/usage_and_configuration/the_basics.md | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/docs/the_black_code_style/current_style.md b/docs/the_black_code_style/current_style.md index 2a5e10162f2..00bd81416dc 100644 --- a/docs/the_black_code_style/current_style.md +++ b/docs/the_black_code_style/current_style.md @@ -8,18 +8,9 @@ deliberately limited and rarely added. Previous formatting is taken into account little as possible, with rare exceptions like the magic trailing comma. The coding style used by _Black_ can be viewed as a strict subset of PEP 8. -_Black_ reformats entire files in place. It doesn't reformat lines that contain -`# fmt: skip` or blocks that start with `# fmt: off` and end with `# fmt: on`. -`# fmt: skip` can be mixed with other pragmas/comments either with multiple comments -(e.g. `# fmt: skip # pylint # noqa`) or as a semicolon separated list (e.g. -`# fmt: skip; pylint; noqa`). `# fmt: on/off` must be on the same level of indentation -and in the same block, meaning no unindents beyond the initial indentation level between -them. It also recognizes [YAPF](https://github.com/google/yapf)'s block comments to the -same effect, as a courtesy for straddling code. - -The rest of this document describes the current formatting style. If you're interested -in trying out where the style is heading, see [future style](./future_style.md) and try -running `black --preview`. +This document describes the current formatting style. If you're interested in trying out +where the style is heading, see [future style](./future_style.md) and try running +`black --preview`. ### How _Black_ wraps lines diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 73c0d1323e3..eb92887f64f 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -12,7 +12,8 @@ _Black_ is a well-behaved Unix-style command-line tool: ## Usage -To get started right away with sensible defaults: +_Black_ will reformat entire files in place. To get started right away with sensible +defaults: ```sh black {source_file_or_directory} @@ -24,6 +25,17 @@ You can run _Black_ as a package if running it as a script doesn't work: python -m black {source_file_or_directory} ``` +### Ignoring sections + +Black will not reformat lines that contain `# fmt: skip` or blocks that start with +`# fmt: off` and end with `# fmt: on`. `# fmt: skip` can be mixed with other +pragmas/comments either with multiple comments (e.g. `# fmt: skip # pylint # noqa`) or +as a semicolon separated list (e.g. `# fmt: skip; pylint; noqa`). `# fmt: on/off` must +be on the same level of indentation and in the same block, meaning no unindents beyond +the initial indentation level between them. Black also recognizes +[YAPF](https://github.com/google/yapf)'s block comments to the same effect, as a +courtesy for straddling code. + ### Command line options The CLI options of _Black_ can be displayed by running `black --help`. All options are @@ -191,7 +203,7 @@ All done! ✨ 🍰 ✨ Show (or do not show) colored diff. Only applies when `--diff` is given. -### `--line-ranges` +#### `--line-ranges` When specified, _Black_ will try its best to only format these lines. From 61b529b7d15400309379f36104885a1dfcd2d026 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sat, 9 Dec 2023 18:29:09 -0800 Subject: [PATCH 215/279] Allow empty lines at beginning of blocks (again) (#4060) --- CHANGES.md | 2 ++ src/black/lines.py | 14 +++++------- src/black/mode.py | 2 +- ...s.py => preview_allow_empty_first_line.py} | 22 +++++++++++++++++++ tests/data/cases/preview_form_feeds.py | 1 + 5 files changed, 31 insertions(+), 10 deletions(-) rename tests/data/cases/{preview_allow_empty_first_line_in_special_cases.py => preview_allow_empty_first_line.py} (87%) diff --git a/CHANGES.md b/CHANGES.md index 8f0b75e7f10..fa0d2494f67 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,8 @@ - Standalone form feed characters at the module level are no longer removed (#4021) - Additional cases of immediately nested tuples, lists, and dictionaries are now indented less (#4012) +- Allow empty lines at the beginning of all blocks, except immediately before a + docstring (#4060) - Fix crash in preview mode when using a short `--line-length` (#4086) ### Configuration diff --git a/src/black/lines.py b/src/black/lines.py index 6e33ee57eab..4050f819757 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -689,18 +689,14 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: return 0, 1 return before, 1 + # In preview mode, always allow blank lines, except right before a function + # docstring is_empty_first_line_ok = ( - Preview.allow_empty_first_line_before_new_block_or_comment - in current_line.mode + Preview.allow_empty_first_line_in_block in current_line.mode and ( - # If it's a standalone comment - current_line.leaves[0].type == STANDALONE_COMMENT - # If it opens a new block - or current_line.opens_block - # If it's a triple quote comment (but not at the start of a funcdef) + not is_docstring(current_line.leaves[0]) or ( - is_docstring(current_line.leaves[0]) - and self.previous_line + self.previous_line and self.previous_line.leaves[0] and self.previous_line.leaves[0].parent and not is_funcdef(self.previous_line.leaves[0].parent) diff --git a/src/black/mode.py b/src/black/mode.py index 04038f49627..9df19618363 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -191,7 +191,7 @@ class Preview(Enum): accept_raw_docstrings = auto() fix_power_op_line_length = auto() hug_parens_with_braces_and_square_brackets = auto() - allow_empty_first_line_before_new_block_or_comment = auto() + allow_empty_first_line_in_block = auto() single_line_format_skip_with_multiple_comments = auto() long_case_block_line_splitting = auto() allow_form_feeds = auto() diff --git a/tests/data/cases/preview_allow_empty_first_line_in_special_cases.py b/tests/data/cases/preview_allow_empty_first_line.py similarity index 87% rename from tests/data/cases/preview_allow_empty_first_line_in_special_cases.py rename to tests/data/cases/preview_allow_empty_first_line.py index 96c1433c110..3e14fa15250 100644 --- a/tests/data/cases/preview_allow_empty_first_line_in_special_cases.py +++ b/tests/data/cases/preview_allow_empty_first_line.py @@ -51,6 +51,17 @@ def baz(): if x: a = 123 +def quux(): + + new_line = here + + +class Cls: + + def method(self): + + pass + # output def foo(): @@ -104,3 +115,14 @@ def baz(): # OK if x: a = 123 + + +def quux(): + + new_line = here + + +class Cls: + def method(self): + + pass diff --git a/tests/data/cases/preview_form_feeds.py b/tests/data/cases/preview_form_feeds.py index 2d8653a1f04..c236f177a95 100644 --- a/tests/data/cases/preview_form_feeds.py +++ b/tests/data/cases/preview_form_feeds.py @@ -198,6 +198,7 @@ def foo(): # form feeds are prohibited inside blocks, or on a line with nonwhitespace def bar(a=1, b: bool = False): + pass From ce28be2705ab29f184ec4a00aa3d23340630796d Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sat, 9 Dec 2023 21:14:25 -0800 Subject: [PATCH 216/279] Add dedicated preview feature for East Asian Width (#4097) --- src/black/lines.py | 2 +- src/black/mode.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/black/lines.py b/src/black/lines.py index 4050f819757..2a41db173d4 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -851,7 +851,7 @@ def is_line_short_enough( # noqa: C901 if not line_str: line_str = line_to_string(line) - width = str_width if mode.preview else len + width = str_width if Preview.respect_east_asian_width in mode else len if Preview.multiline_string_handling not in mode: return ( diff --git a/src/black/mode.py b/src/black/mode.py index 9df19618363..38b861e39ca 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -195,6 +195,7 @@ class Preview(Enum): single_line_format_skip_with_multiple_comments = auto() long_case_block_line_splitting = auto() allow_form_feeds = auto() + respect_east_asian_width = auto() class Deprecated(UserWarning): From 67b23d71854c19921cc6092c695d3301ab99229c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 11:32:04 -0800 Subject: [PATCH 217/279] Bump actions/setup-python from 4 to 5 (#4101) --- .github/workflows/diff_shades.yml | 4 ++-- .github/workflows/diff_shades_comment.yml | 2 +- .github/workflows/doc.yml | 2 +- .github/workflows/fuzz.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/pypi_upload.yml | 2 +- .github/workflows/release_tests.yml | 2 +- .github/workflows/test.yml | 4 ++-- .github/workflows/upload_binary.yml | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/diff_shades.yml b/.github/workflows/diff_shades.yml index 6bfc6ca9ed8..8d8be2550b0 100644 --- a/.github/workflows/diff_shades.yml +++ b/.github/workflows/diff_shades.yml @@ -20,7 +20,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.11" @@ -57,7 +57,7 @@ jobs: # The baseline revision could be rather old so a full clone is ideal. fetch-depth: 0 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.11" diff --git a/.github/workflows/diff_shades_comment.yml b/.github/workflows/diff_shades_comment.yml index 49fd376d85e..9b3b4b579da 100644 --- a/.github/workflows/diff_shades_comment.yml +++ b/.github/workflows/diff_shades_comment.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "*" diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index fa3d87c70f5..006991a16d8 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -24,7 +24,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up latest Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "*" diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 48c26452c54..42a399fd0aa 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -28,7 +28,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9c7aca8f869..2d016cef7a6 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -24,7 +24,7 @@ jobs: fi - name: Set up latest Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "*" diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index bbdcdf17a8f..8e3eb67a10d 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up latest Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "*" diff --git a/.github/workflows/release_tests.yml b/.github/workflows/release_tests.yml index 74729445052..192ba004f81 100644 --- a/.github/workflows/release_tests.yml +++ b/.github/workflows/release_tests.yml @@ -34,7 +34,7 @@ jobs: # Give us all history, branches and tags fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} allow-prereleases: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3f8928cc42a..55359a23303 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,7 +38,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -96,7 +96,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up latest Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "*" diff --git a/.github/workflows/upload_binary.yml b/.github/workflows/upload_binary.yml index bb19d48158c..06e55cfe93a 100644 --- a/.github/workflows/upload_binary.yml +++ b/.github/workflows/upload_binary.yml @@ -32,7 +32,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up latest Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "*" From 9aea9768cb60d23f2f4d331e94c4ee07ef1683a5 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 11 Dec 2023 13:19:02 -0800 Subject: [PATCH 218/279] Only use dummy implementation logic for functions and classes (#4066) Fixes #4063 --- CHANGES.md | 2 ++ src/black/linegen.py | 4 ++-- src/black/nodes.py | 9 +++++++- .../cases/preview_dummy_implementations.py | 22 +++++++++++++++++-- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index fa0d2494f67..62caea41c31 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,6 +20,8 @@ - Allow empty lines at the beginning of all blocks, except immediately before a docstring (#4060) - Fix crash in preview mode when using a short `--line-length` (#4086) +- Keep suites consisting of only an ellipsis on their own lines if they are not + functions or class definitions (#4066) ### Configuration diff --git a/src/black/linegen.py b/src/black/linegen.py index 073672a5ae7..6934823d340 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -286,7 +286,7 @@ def visit_suite(self, node: Node) -> Iterator[Line]: """Visit a suite.""" if ( self.mode.is_pyi or Preview.dummy_implementations in self.mode - ) and is_stub_suite(node): + ) and is_stub_suite(node, self.mode): yield from self.visit(node.children[2]) else: yield from self.visit_default(node) @@ -314,7 +314,7 @@ def visit_simple_stmt(self, node: Node) -> Iterator[Line]: if ( not (self.mode.is_pyi or Preview.dummy_implementations in self.mode) or not node.parent - or not is_stub_suite(node.parent) + or not is_stub_suite(node.parent, self.mode) ): yield from self.line() yield from self.visit_default(node) diff --git a/src/black/nodes.py b/src/black/nodes.py index de53f8e36a3..9b8d9a97835 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -736,8 +736,15 @@ def is_funcdef(node: Node) -> bool: return node.type == syms.funcdef -def is_stub_suite(node: Node) -> bool: +def is_stub_suite(node: Node, mode: Mode) -> bool: """Return True if `node` is a suite with a stub body.""" + if node.parent is not None: + if Preview.dummy_implementations in mode and node.parent.type not in ( + syms.funcdef, + syms.async_funcdef, + syms.classdef, + ): + return False # If there is a comment, we want to keep it. if node.prefix.strip(): diff --git a/tests/data/cases/preview_dummy_implementations.py b/tests/data/cases/preview_dummy_implementations.py index 98b69bf87b2..113ac36cdc5 100644 --- a/tests/data/cases/preview_dummy_implementations.py +++ b/tests/data/cases/preview_dummy_implementations.py @@ -1,9 +1,11 @@ # flags: --preview from typing import NoReturn, Protocol, Union, overload +class Empty: + ... def dummy(a): ... -def other(b): ... +async def other(b): ... @overload @@ -48,13 +50,22 @@ def b(arg: Union[int, str, object]) -> Union[int, str]: raise TypeError return arg +def has_comment(): + ... # still a dummy + +if some_condition: + ... + # output from typing import NoReturn, Protocol, Union, overload +class Empty: ... + + def dummy(a): ... -def other(b): ... +async def other(b): ... @overload @@ -98,3 +109,10 @@ def b(arg: Union[int, str, object]) -> Union[int, str]: if not isinstance(arg, (int, str)): raise TypeError return arg + + +def has_comment(): ... # still a dummy + + +if some_condition: + ... From 0c9899956d890a9dc9c3adbc80b478a47846ced9 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 11 Dec 2023 14:29:33 -0800 Subject: [PATCH 219/279] Fix path in test message (#4102) --- tests/test_black.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_black.py b/tests/test_black.py index 899cbeb111d..23815da9042 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -317,7 +317,7 @@ def test_expression_diff(self) -> None: msg = ( "Expected diff isn't equal to the actual. If you made changes to" " expression.py and this is an anticipated difference, overwrite" - f" tests/data/expression.diff with {dump}" + f" tests/data/cases/expression.diff with {dump}" ) self.assertEqual(expected, actual, msg) From eb7661f8ab9bff344835693c7c08789bb195137e Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 11 Dec 2023 14:41:41 -0800 Subject: [PATCH 220/279] Fix another case where we format dummy implementation for non-functions/classes (#4103) --- CHANGES.md | 2 +- src/black/linegen.py | 12 +++++++----- src/black/nodes.py | 17 ++++++++++------- .../data/cases/preview_dummy_implementations.py | 5 +++++ 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 62caea41c31..dcf6613b70c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -21,7 +21,7 @@ docstring (#4060) - Fix crash in preview mode when using a short `--line-length` (#4086) - Keep suites consisting of only an ellipsis on their own lines if they are not - functions or class definitions (#4066) + functions or class definitions (#4066) (#4103) ### Configuration diff --git a/src/black/linegen.py b/src/black/linegen.py index 6934823d340..245be235231 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -42,6 +42,7 @@ is_atom_with_invisible_parens, is_docstring, is_empty_tuple, + is_function_or_class, is_lpar_token, is_multiline_string, is_name_token, @@ -299,11 +300,12 @@ def visit_simple_stmt(self, node: Node) -> Iterator[Line]: wrap_in_parentheses(node, child, visible=False) prev_type = child.type - is_suite_like = node.parent and node.parent.type in STATEMENT - if is_suite_like: - if ( - self.mode.is_pyi or Preview.dummy_implementations in self.mode - ) and is_stub_body(node): + if node.parent and node.parent.type in STATEMENT: + if Preview.dummy_implementations in self.mode: + condition = is_function_or_class(node.parent) + else: + condition = self.mode.is_pyi + if condition and is_stub_body(node): yield from self.visit_default(node) else: yield from self.line(+1) diff --git a/src/black/nodes.py b/src/black/nodes.py index 9b8d9a97835..a4f555b4032 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -736,15 +736,18 @@ def is_funcdef(node: Node) -> bool: return node.type == syms.funcdef +def is_function_or_class(node: Node) -> bool: + return node.type in {syms.funcdef, syms.classdef, syms.async_funcdef} + + def is_stub_suite(node: Node, mode: Mode) -> bool: """Return True if `node` is a suite with a stub body.""" - if node.parent is not None: - if Preview.dummy_implementations in mode and node.parent.type not in ( - syms.funcdef, - syms.async_funcdef, - syms.classdef, - ): - return False + if ( + node.parent is not None + and Preview.dummy_implementations in mode + and not is_function_or_class(node.parent) + ): + return False # If there is a comment, we want to keep it. if node.prefix.strip(): diff --git a/tests/data/cases/preview_dummy_implementations.py b/tests/data/cases/preview_dummy_implementations.py index 113ac36cdc5..28b23bb8609 100644 --- a/tests/data/cases/preview_dummy_implementations.py +++ b/tests/data/cases/preview_dummy_implementations.py @@ -56,6 +56,8 @@ def has_comment(): if some_condition: ... +if already_dummy: ... + # output from typing import NoReturn, Protocol, Union, overload @@ -116,3 +118,6 @@ def has_comment(): ... # still a dummy if some_condition: ... + +if already_dummy: + ... From ebd543c0ac9b8a5f17636d0a42c425e5f693860e Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 11 Dec 2023 21:37:15 -0800 Subject: [PATCH 221/279] Fix feature detection for parenthesized context managers (#4104) --- CHANGES.md | 1 + src/black/__init__.py | 18 ++- tests/data/cases/pep_572_remove_parens.py | 2 +- tests/test_black.py | 130 ++++++++++++---------- 4 files changed, 93 insertions(+), 58 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index dcf6613b70c..e3b5b7392b9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,7 @@ - Fix bug where `# fmt: off` automatically dedents when used with the `--line-ranges` option, even when it is not within the specified line range. (#4084) +- Fix feature detection for parenthesized context managers (#4104) ### Preview style diff --git a/src/black/__init__.py b/src/black/__init__.py index 5073fa748d5..735ba713b8f 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -1351,7 +1351,7 @@ def get_features_used( # noqa: C901 if ( len(atom_children) == 3 and atom_children[0].type == token.LPAR - and atom_children[1].type == syms.testlist_gexp + and _contains_asexpr(atom_children[1]) and atom_children[2].type == token.RPAR ): features.add(Feature.PARENTHESIZED_CONTEXT_MANAGERS) @@ -1384,6 +1384,22 @@ def get_features_used( # noqa: C901 return features +def _contains_asexpr(node: Union[Node, Leaf]) -> bool: + """Return True if `node` contains an as-pattern.""" + if node.type == syms.asexpr_test: + return True + elif node.type == syms.atom: + if ( + len(node.children) == 3 + and node.children[0].type == token.LPAR + and node.children[2].type == token.RPAR + ): + return _contains_asexpr(node.children[1]) + elif node.type == syms.testlist_gexp: + return any(_contains_asexpr(child) for child in node.children) + return False + + def detect_target_versions( node: Node, *, future_imports: Optional[Set[str]] = None ) -> Set[TargetVersion]: diff --git a/tests/data/cases/pep_572_remove_parens.py b/tests/data/cases/pep_572_remove_parens.py index 88774d81649..24f1ac29168 100644 --- a/tests/data/cases/pep_572_remove_parens.py +++ b/tests/data/cases/pep_572_remove_parens.py @@ -1,4 +1,4 @@ -# flags: --minimum-version=3.8 --no-preview-line-length-1 +# flags: --minimum-version=3.8 if (foo := 0): pass diff --git a/tests/test_black.py b/tests/test_black.py index 23815da9042..0af5fd2a1f4 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -25,6 +25,7 @@ List, Optional, Sequence, + Set, Type, TypeVar, Union, @@ -874,71 +875,88 @@ def test_get_features_used_decorator(self) -> None: ) def test_get_features_used(self) -> None: - node = black.lib2to3_parse("def f(*, arg): ...\n") - self.assertEqual(black.get_features_used(node), set()) - node = black.lib2to3_parse("def f(*, arg,): ...\n") - self.assertEqual(black.get_features_used(node), {Feature.TRAILING_COMMA_IN_DEF}) - node = black.lib2to3_parse("f(*arg,)\n") - self.assertEqual( - black.get_features_used(node), {Feature.TRAILING_COMMA_IN_CALL} + self.check_features_used("def f(*, arg): ...\n", set()) + self.check_features_used( + "def f(*, arg,): ...\n", {Feature.TRAILING_COMMA_IN_DEF} ) - node = black.lib2to3_parse("def f(*, arg): f'string'\n") - self.assertEqual(black.get_features_used(node), {Feature.F_STRINGS}) - node = black.lib2to3_parse("123_456\n") - self.assertEqual(black.get_features_used(node), {Feature.NUMERIC_UNDERSCORES}) - node = black.lib2to3_parse("123456\n") - self.assertEqual(black.get_features_used(node), set()) + self.check_features_used("f(*arg,)\n", {Feature.TRAILING_COMMA_IN_CALL}) + self.check_features_used("def f(*, arg): f'string'\n", {Feature.F_STRINGS}) + self.check_features_used("123_456\n", {Feature.NUMERIC_UNDERSCORES}) + self.check_features_used("123456\n", set()) + source, expected = read_data("cases", "function") - node = black.lib2to3_parse(source) expected_features = { Feature.TRAILING_COMMA_IN_CALL, Feature.TRAILING_COMMA_IN_DEF, Feature.F_STRINGS, } - self.assertEqual(black.get_features_used(node), expected_features) - node = black.lib2to3_parse(expected) - self.assertEqual(black.get_features_used(node), expected_features) + self.check_features_used(source, expected_features) + self.check_features_used(expected, expected_features) + source, expected = read_data("cases", "expression") - node = black.lib2to3_parse(source) - self.assertEqual(black.get_features_used(node), set()) - node = black.lib2to3_parse(expected) - self.assertEqual(black.get_features_used(node), set()) - node = black.lib2to3_parse("lambda a, /, b: ...") - self.assertEqual(black.get_features_used(node), {Feature.POS_ONLY_ARGUMENTS}) - node = black.lib2to3_parse("def fn(a, /, b): ...") - self.assertEqual(black.get_features_used(node), {Feature.POS_ONLY_ARGUMENTS}) - node = black.lib2to3_parse("def fn(): yield a, b") - self.assertEqual(black.get_features_used(node), set()) - node = black.lib2to3_parse("def fn(): return a, b") - self.assertEqual(black.get_features_used(node), set()) - node = black.lib2to3_parse("def fn(): yield *b, c") - self.assertEqual(black.get_features_used(node), {Feature.UNPACKING_ON_FLOW}) - node = black.lib2to3_parse("def fn(): return a, *b, c") - self.assertEqual(black.get_features_used(node), {Feature.UNPACKING_ON_FLOW}) - node = black.lib2to3_parse("x = a, *b, c") - self.assertEqual(black.get_features_used(node), set()) - node = black.lib2to3_parse("x: Any = regular") - self.assertEqual(black.get_features_used(node), set()) - node = black.lib2to3_parse("x: Any = (regular, regular)") - self.assertEqual(black.get_features_used(node), set()) - node = black.lib2to3_parse("x: Any = Complex(Type(1))[something]") - self.assertEqual(black.get_features_used(node), set()) - node = black.lib2to3_parse("x: Tuple[int, ...] = a, b, c") - self.assertEqual( - black.get_features_used(node), {Feature.ANN_ASSIGN_EXTENDED_RHS} + self.check_features_used(source, set()) + self.check_features_used(expected, set()) + + self.check_features_used("lambda a, /, b: ...\n", {Feature.POS_ONLY_ARGUMENTS}) + self.check_features_used("def fn(a, /, b): ...", {Feature.POS_ONLY_ARGUMENTS}) + + self.check_features_used("def fn(): yield a, b", set()) + self.check_features_used("def fn(): return a, b", set()) + self.check_features_used("def fn(): yield *b, c", {Feature.UNPACKING_ON_FLOW}) + self.check_features_used( + "def fn(): return a, *b, c", {Feature.UNPACKING_ON_FLOW} ) - node = black.lib2to3_parse("try: pass\nexcept Something: pass") - self.assertEqual(black.get_features_used(node), set()) - node = black.lib2to3_parse("try: pass\nexcept (*Something,): pass") - self.assertEqual(black.get_features_used(node), set()) - node = black.lib2to3_parse("try: pass\nexcept *Group: pass") - self.assertEqual(black.get_features_used(node), {Feature.EXCEPT_STAR}) - node = black.lib2to3_parse("a[*b]") - self.assertEqual(black.get_features_used(node), {Feature.VARIADIC_GENERICS}) - node = black.lib2to3_parse("a[x, *y(), z] = t") - self.assertEqual(black.get_features_used(node), {Feature.VARIADIC_GENERICS}) - node = black.lib2to3_parse("def fn(*args: *T): pass") - self.assertEqual(black.get_features_used(node), {Feature.VARIADIC_GENERICS}) + self.check_features_used("x = a, *b, c", set()) + + self.check_features_used("x: Any = regular", set()) + self.check_features_used("x: Any = (regular, regular)", set()) + self.check_features_used("x: Any = Complex(Type(1))[something]", set()) + self.check_features_used( + "x: Tuple[int, ...] = a, b, c", {Feature.ANN_ASSIGN_EXTENDED_RHS} + ) + + self.check_features_used("try: pass\nexcept Something: pass", set()) + self.check_features_used("try: pass\nexcept (*Something,): pass", set()) + self.check_features_used( + "try: pass\nexcept *Group: pass", {Feature.EXCEPT_STAR} + ) + + self.check_features_used("a[*b]", {Feature.VARIADIC_GENERICS}) + self.check_features_used("a[x, *y(), z] = t", {Feature.VARIADIC_GENERICS}) + self.check_features_used("def fn(*args: *T): pass", {Feature.VARIADIC_GENERICS}) + + self.check_features_used("with a: pass", set()) + self.check_features_used("with a, b: pass", set()) + self.check_features_used("with a as b: pass", set()) + self.check_features_used("with a as b, c as d: pass", set()) + self.check_features_used("with (a): pass", set()) + self.check_features_used("with (a, b): pass", set()) + self.check_features_used("with (a, b) as (c, d): pass", set()) + self.check_features_used( + "with (a as b): pass", {Feature.PARENTHESIZED_CONTEXT_MANAGERS} + ) + self.check_features_used( + "with ((a as b)): pass", {Feature.PARENTHESIZED_CONTEXT_MANAGERS} + ) + self.check_features_used( + "with (a, b as c): pass", {Feature.PARENTHESIZED_CONTEXT_MANAGERS} + ) + self.check_features_used( + "with (a, (b as c)): pass", {Feature.PARENTHESIZED_CONTEXT_MANAGERS} + ) + self.check_features_used( + "with ((a, ((b as c)))): pass", {Feature.PARENTHESIZED_CONTEXT_MANAGERS} + ) + + def check_features_used(self, source: str, expected: Set[Feature]) -> None: + node = black.lib2to3_parse(source) + actual = black.get_features_used(node) + msg = f"Expected {expected} but got {actual} for {source!r}" + try: + self.assertEqual(actual, expected, msg=msg) + except AssertionError: + DebugVisitor.show(node) + raise def test_get_features_used_for_future_flags(self) -> None: for src, features in [ From d9ad09a32b0e0481bb4fef548d35b7a49cc03c5d Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 11 Dec 2023 21:55:28 -0800 Subject: [PATCH 222/279] Prepare release 23.12.0 (#4105) --- CHANGES.md | 33 +++++---------------- docs/integrations/source_version_control.md | 4 +-- docs/usage_and_configuration/the_basics.md | 6 ++-- 3 files changed, 13 insertions(+), 30 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e3b5b7392b9..223d7d2c819 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,10 +1,16 @@ # Change Log -## Unreleased +## 23.12.0 ### Highlights - +It's almost 2024, which means it's time for a new edition of _Black_'s stable style! +Together with this release, we'll put out an alpha release 24.1a1 showcasing the draft +2024 stable style, which we'll finalize in the January release. Please try it out and +[share your feedback](https://github.com/psf/black/issues/4042). + +This release (23.12.0) will still produce the 2023 style. Most but not all of the +changes in `--preview` mode will be in the 2024 stable style. ### Stable style @@ -26,8 +32,6 @@ ### Configuration - - - `--line-ranges` now skips _Black_'s internal stability check in `--safe` mode. This avoids a crash on rare inputs that have many unformatted same-content lines. (#4034) @@ -36,33 +40,12 @@ - Upgrade to mypy 1.7.1 (#4049) (#4069) - Faster compiled wheels are now available for CPython 3.12 (#4070) -### Parser - - - -### Performance - - - -### Output - - - -### _Blackd_ - - - ### Integrations - Enable 3.12 CI (#4035) - Build docker images in parallel (#4054) - Build docker images with 3.12 (#4055) -### Documentation - - - ## 23.11.0 ### Highlights diff --git a/docs/integrations/source_version_control.md b/docs/integrations/source_version_control.md index 3c7ef89918f..ca810f1d8f6 100644 --- a/docs/integrations/source_version_control.md +++ b/docs/integrations/source_version_control.md @@ -8,7 +8,7 @@ Use [pre-commit](https://pre-commit.com/). Once you repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.11.0 + rev: 23.12.0 hooks: - id: black # It is recommended to specify the latest version of Python @@ -35,7 +35,7 @@ include Jupyter Notebooks. To use this hook, simply replace the hook's `id: blac repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.11.0 + rev: 23.12.0 hooks: - id: black-jupyter # It is recommended to specify the latest version of Python diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index eb92887f64f..2dbb573803c 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -241,8 +241,8 @@ configuration file for consistent results across environments. ```console $ black --version -black, 23.11.0 (compiled: yes) -$ black --required-version 23.11.0 -c "format = 'this'" +black, 23.12.0 (compiled: yes) +$ black --required-version 23.12.0 -c "format = 'this'" format = "this" $ black --required-version 31.5b2 -c "still = 'beta?!'" Oh no! 💥 💔 💥 The required version does not match the running version! @@ -333,7 +333,7 @@ You can check the version of _Black_ you have installed using the `--version` fl ```console $ black --version -black, 23.11.0 +black, 23.12.0 ``` #### `--config` From 35ce37ded7bd8fdd3950af19e7c11f311ee7b8d8 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 11 Dec 2023 22:28:46 -0800 Subject: [PATCH 223/279] Add new changelog template --- CHANGES.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 223d7d2c819..9d79b0fb61a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,52 @@ # Change Log +## Unreleased + +### Highlights + + + +### Stable style + + + +### Preview style + + + +### Configuration + + + +### Packaging + + + +### Parser + + + +### Performance + + + +### Output + + + +### _Blackd_ + + + +### Integrations + + + +### Documentation + + + ## 23.12.0 ### Highlights From 8fec1c30855890cc9cfce5ae6d633a1c1a21d724 Mon Sep 17 00:00:00 2001 From: Bryce Willey Date: Thu, 14 Dec 2023 03:28:28 -0500 Subject: [PATCH 224/279] Adds paren to deps for hidden extra constraint (#4108) Fix #4107 --- CHANGES.md | 2 ++ pyproject.toml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9d79b0fb61a..69fe34a5052 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -22,6 +22,8 @@ +- Fixed a bug that included dependencies from the `d` extra by default (#4108) + ### Parser diff --git a/pyproject.toml b/pyproject.toml index 1098412981a..24b9c07674d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ preview = true # NOTE: You don't need this in your own Black configuration. [build-system] -requires = ["hatchling>=1.8.0", "hatch-vcs", "hatch-fancy-pypi-readme"] +requires = ["hatchling>=1.20.0", "hatch-vcs", "hatch-fancy-pypi-readme"] build-backend = "hatchling.build" [project] @@ -187,7 +187,7 @@ CC = "clang" build-frontend = { name = "build", args = ["--no-isolation"] } # Unfortunately, hatch doesn't respect MACOSX_DEPLOYMENT_TARGET before-build = [ - "python -m pip install 'hatchling==1.18.0' hatch-vcs hatch-fancy-pypi-readme 'hatch-mypyc>=0.16.0' 'mypy==1.7.1' 'click==8.1.3'", + "python -m pip install 'hatchling==1.20.0' hatch-vcs hatch-fancy-pypi-readme 'hatch-mypyc>=0.16.0' 'mypy==1.7.1' 'click==8.1.3'", """sed -i '' -e "600,700s/'10_16'/os.environ['MACOSX_DEPLOYMENT_TARGET'].replace('.', '_')/" $(python -c 'import hatchling.builders.wheel as h; print(h.__file__)') """, ] From ec91a2be3c44d88e1a3960a4937ad6ed3b63464e Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Fri, 22 Dec 2023 17:04:32 -0600 Subject: [PATCH 225/279] Prepare release 23.12.1 (#4124) --- CHANGES.md | 45 +-------------------- docs/integrations/source_version_control.md | 4 +- docs/usage_and_configuration/the_basics.md | 6 +-- 3 files changed, 6 insertions(+), 49 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 69fe34a5052..d0c9e567457 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,54 +1,11 @@ # Change Log -## Unreleased - -### Highlights - - - -### Stable style - - - -### Preview style - - - -### Configuration - - +## 23.12.1 ### Packaging - - - Fixed a bug that included dependencies from the `d` extra by default (#4108) -### Parser - - - -### Performance - - - -### Output - - - -### _Blackd_ - - - -### Integrations - - - -### Documentation - - - ## 23.12.0 ### Highlights diff --git a/docs/integrations/source_version_control.md b/docs/integrations/source_version_control.md index ca810f1d8f6..3b895193941 100644 --- a/docs/integrations/source_version_control.md +++ b/docs/integrations/source_version_control.md @@ -8,7 +8,7 @@ Use [pre-commit](https://pre-commit.com/). Once you repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.12.0 + rev: 23.12.1 hooks: - id: black # It is recommended to specify the latest version of Python @@ -35,7 +35,7 @@ include Jupyter Notebooks. To use this hook, simply replace the hook's `id: blac repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.12.0 + rev: 23.12.1 hooks: - id: black-jupyter # It is recommended to specify the latest version of Python diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 2dbb573803c..4f9856c6a47 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -241,8 +241,8 @@ configuration file for consistent results across environments. ```console $ black --version -black, 23.12.0 (compiled: yes) -$ black --required-version 23.12.0 -c "format = 'this'" +black, 23.12.1 (compiled: yes) +$ black --required-version 23.12.1 -c "format = 'this'" format = "this" $ black --required-version 31.5b2 -c "still = 'beta?!'" Oh no! 💥 💔 💥 The required version does not match the running version! @@ -333,7 +333,7 @@ You can check the version of _Black_ you have installed using the `--version` fl ```console $ black --version -black, 23.12.0 +black, 23.12.1 ``` #### `--config` From 1b831f214a111dfb45a571fc40f4404bb6b5b62c Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Fri, 22 Dec 2023 17:46:06 -0600 Subject: [PATCH 226/279] Add new changelog template (#4125) --- CHANGES.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index d0c9e567457..526cbd12123 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,52 @@ # Change Log +## Unreleased + +### Highlights + + + +### Stable style + + + +### Preview style + + + +### Configuration + + + +### Packaging + + + +### Parser + + + +### Performance + + + +### Output + + + +### _Blackd_ + + + +### Integrations + + + +### Documentation + + + ## 23.12.1 ### Packaging From 51786141cc4eb3c212be76638e66b91648d0e5f8 Mon Sep 17 00:00:00 2001 From: Anupya Pamidimukkala Date: Thu, 28 Dec 2023 01:23:42 -0500 Subject: [PATCH 227/279] Fix nits, chain comparisons, unused params, hyphens (#4114) --- src/black/brackets.py | 4 ++-- src/black/cache.py | 4 ++-- src/black/comments.py | 3 +-- src/black/linegen.py | 2 +- src/black/lines.py | 4 ++-- src/black/ranges.py | 5 +---- src/black/trans.py | 21 ++++++++++----------- 7 files changed, 19 insertions(+), 24 deletions(-) diff --git a/src/black/brackets.py b/src/black/brackets.py index 3020cc0d390..37e6b2590eb 100644 --- a/src/black/brackets.py +++ b/src/black/brackets.py @@ -115,7 +115,7 @@ def mark(self, leaf: Leaf) -> None: if delim and self.previous is not None: self.delimiters[id(self.previous)] = delim else: - delim = is_split_after_delimiter(leaf, self.previous) + delim = is_split_after_delimiter(leaf) if delim: self.delimiters[id(leaf)] = delim if leaf.type in OPENING_BRACKETS: @@ -215,7 +215,7 @@ def get_open_lsqb(self) -> Optional[Leaf]: return self.bracket_match.get((self.depth - 1, token.RSQB)) -def is_split_after_delimiter(leaf: Leaf, previous: Optional[Leaf] = None) -> Priority: +def is_split_after_delimiter(leaf: Leaf) -> Priority: """Return the priority of the `leaf` delimiter, given a line break after it. The delimiter priorities returned here are from those delimiters that would diff --git a/src/black/cache.py b/src/black/cache.py index 6baa096baca..c844c37b6f8 100644 --- a/src/black/cache.py +++ b/src/black/cache.py @@ -58,9 +58,9 @@ class Cache: @classmethod def read(cls, mode: Mode) -> Self: - """Read the cache if it exists and is well formed. + """Read the cache if it exists and is well-formed. - If it is not well formed, the call to write later should + If it is not well-formed, the call to write later should resolve the issue. """ cache_file = get_cache_file(mode) diff --git a/src/black/comments.py b/src/black/comments.py index 25413121199..52bb024a799 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -221,8 +221,7 @@ def convert_one_fmt_off_pair( if comment.value in FMT_OFF: fmt_off_prefix = "" if len(lines) > 0 and not any( - comment_lineno >= line[0] and comment_lineno <= line[1] - for line in lines + line[0] <= comment_lineno <= line[1] for line in lines ): # keeping indentation of comment by preserving original whitespaces. fmt_off_prefix = prefix.split(comment.value)[0] diff --git a/src/black/linegen.py b/src/black/linegen.py index 245be235231..0fd4a8d9c96 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -1635,7 +1635,7 @@ def generate_trailers_to_omit(line: Line, line_length: int) -> Iterator[Set[Leaf opening_bracket: Optional[Leaf] = None closing_bracket: Optional[Leaf] = None inner_brackets: Set[LeafID] = set() - for index, leaf, leaf_length in line.enumerate_with_length(reversed=True): + for index, leaf, leaf_length in line.enumerate_with_length(is_reversed=True): length += leaf_length if length > line_length: break diff --git a/src/black/lines.py b/src/black/lines.py index 2a41db173d4..0cd4189a778 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -451,7 +451,7 @@ def is_complex_subscript(self, leaf: Leaf) -> bool: ) def enumerate_with_length( - self, reversed: bool = False + self, is_reversed: bool = False ) -> Iterator[Tuple[Index, Leaf, int]]: """Return an enumeration of leaves with their length. @@ -459,7 +459,7 @@ def enumerate_with_length( """ op = cast( Callable[[Sequence[Leaf]], Iterator[Tuple[Index, Leaf]]], - enumerate_reversed if reversed else enumerate, + enumerate_reversed if is_reversed else enumerate, ) for index, leaf in op(self.leaves): length = len(leaf.prefix) + len(leaf.value) diff --git a/src/black/ranges.py b/src/black/ranges.py index 59e19242d47..06fa8790554 100644 --- a/src/black/ranges.py +++ b/src/black/ranges.py @@ -487,10 +487,7 @@ def _find_lines_mapping_index( index = start_index while index < len(lines_mappings): mapping = lines_mappings[index] - if ( - mapping.original_start <= original_line - and original_line <= mapping.original_end - ): + if mapping.original_start <= original_line <= mapping.original_end: return index index += 1 return index diff --git a/src/black/trans.py b/src/black/trans.py index ab3197fa6df..7c7335a005b 100644 --- a/src/black/trans.py +++ b/src/black/trans.py @@ -1273,7 +1273,7 @@ def iter_fexpr_spans(s: str) -> Iterator[Tuple[int, int]]: i += 1 continue - # if we're in an expression part of the f-string, fast forward through strings + # if we're in an expression part of the f-string, fast-forward through strings # note that backslashes are not legal in the expression portion of f-strings if stack: delim = None @@ -1740,7 +1740,7 @@ def passes_all_checks(i: Index) -> bool: """ Returns: True iff ALL of the conditions listed in the 'Transformations' - section of this classes' docstring would be be met by returning @i. + section of this classes' docstring would be met by returning @i. """ is_space = string[i] == " " is_split_safe = is_valid_index(i - 1) and string[i - 1] in SPLIT_SAFE_CHARS @@ -1932,7 +1932,7 @@ def _return_match(LL: List[Leaf]) -> Optional[int]: OR None, otherwise. """ - # If this line is apart of a return/yield statement and the first leaf + # If this line is a part of a return/yield statement and the first leaf # contains either the "return" or "yield" keywords... if parent_type(LL[0]) in [syms.return_stmt, syms.yield_expr] and LL[ 0 @@ -1957,7 +1957,7 @@ def _else_match(LL: List[Leaf]) -> Optional[int]: OR None, otherwise. """ - # If this line is apart of a ternary expression and the first leaf + # If this line is a part of a ternary expression and the first leaf # contains the "else" keyword... if ( parent_type(LL[0]) == syms.test @@ -1984,7 +1984,7 @@ def _assert_match(LL: List[Leaf]) -> Optional[int]: OR None, otherwise. """ - # If this line is apart of an assert statement and the first leaf + # If this line is a part of an assert statement and the first leaf # contains the "assert" keyword... if parent_type(LL[0]) == syms.assert_stmt and LL[0].value == "assert": is_valid_index = is_valid_index_factory(LL) @@ -2019,7 +2019,7 @@ def _assign_match(LL: List[Leaf]) -> Optional[int]: OR None, otherwise. """ - # If this line is apart of an expression statement or is a function + # If this line is a part of an expression statement or is a function # argument AND the first leaf contains a variable name... if ( parent_type(LL[0]) in [syms.expr_stmt, syms.argument, syms.power] @@ -2040,7 +2040,7 @@ def _assign_match(LL: List[Leaf]) -> Optional[int]: string_parser = StringParser() idx = string_parser.parse(LL, string_idx) - # The next leaf MAY be a comma iff this line is apart + # The next leaf MAY be a comma iff this line is a part # of a function argument... if ( parent_type(LL[0]) == syms.argument @@ -2187,8 +2187,7 @@ def do_transform( if opening_bracket is not None and opening_bracket in left_leaves: index = left_leaves.index(opening_bracket) if ( - index > 0 - and index < len(left_leaves) - 1 + 0 < index < len(left_leaves) - 1 and left_leaves[index - 1].type == token.COLON and left_leaves[index + 1].value == "lambda" ): @@ -2297,7 +2296,7 @@ def parse(self, leaves: List[Leaf], string_idx: int) -> int: * @leaves[@string_idx].type == token.STRING Returns: - The index directly after the last leaf which is apart of the string + The index directly after the last leaf which is a part of the string trailer, if a "trailer" exists. OR @string_idx + 1, if no string "trailer" exists. @@ -2320,7 +2319,7 @@ def _next_state(self, leaf: Leaf) -> bool: MUST be the leaf directly following @leaf. Returns: - True iff @leaf is apart of the string's trailer. + True iff @leaf is a part of the string's trailer. """ # We ignore empty LPAR or RPAR leaves. if is_empty_par(leaf): From c80685f36183f146f831a5737510cf105f947745 Mon Sep 17 00:00:00 2001 From: cobalt <61329810+RedGuy12@users.noreply.github.com> Date: Thu, 28 Dec 2023 00:24:25 -0600 Subject: [PATCH 228/279] Treat walruses like other binary operators in subscripts (#4109) Fixes #4078 --- CHANGES.md | 3 +++ src/black/lines.py | 9 ++++++++- tests/data/cases/preview_pep_572.py | 4 ++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 526cbd12123..1444463050f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,9 @@ +- Fix bug where spaces were not added around parenthesized walruses in subscripts, + unlike other binary operators (#4109) + ### Configuration diff --git a/src/black/lines.py b/src/black/lines.py index 0cd4189a778..d153b8c2e1b 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -446,8 +446,15 @@ def is_complex_subscript(self, leaf: Leaf) -> bool: if subscript_start.type == syms.subscriptlist: subscript_start = child_towards(subscript_start, leaf) + + # When this is moved out of preview, add syms.namedexpr_test directly to + # TEST_DESCENDANTS in nodes.py + if Preview.walrus_subscript in self.mode: + test_decendants = TEST_DESCENDANTS | {syms.namedexpr_test} + else: + test_decendants = TEST_DESCENDANTS return subscript_start is not None and any( - n.type in TEST_DESCENDANTS for n in subscript_start.pre_order() + n.type in test_decendants for n in subscript_start.pre_order() ) def enumerate_with_length( diff --git a/tests/data/cases/preview_pep_572.py b/tests/data/cases/preview_pep_572.py index 8e801ff6cdc..75ad0cc4176 100644 --- a/tests/data/cases/preview_pep_572.py +++ b/tests/data/cases/preview_pep_572.py @@ -3,5 +3,5 @@ x[:(a:=0)] # output -x[(a := 0):] -x[:(a := 0)] +x[(a := 0) :] +x[: (a := 0)] From bf6cabc8049cbdf4d0b8af33134317a0190a614f Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Wed, 27 Dec 2023 22:24:57 -0800 Subject: [PATCH 229/279] Do not round cache mtimes (#4128) Fixes #4116 This logic was introduced in #3821, I believe as a result of copying logic inside mypy that I think isn't relevant to Black --- CHANGES.md | 2 ++ src/black/cache.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 1444463050f..2389f6d39fd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -21,6 +21,8 @@ +- Fix cache mtime logic that resulted in false positive cache hits (#4128) + ### Packaging diff --git a/src/black/cache.py b/src/black/cache.py index c844c37b6f8..cfdbc21e92a 100644 --- a/src/black/cache.py +++ b/src/black/cache.py @@ -101,7 +101,7 @@ def is_changed(self, source: Path) -> bool: st = res_src.stat() if st.st_size != old.st_size: return True - if int(st.st_mtime) != int(old.st_mtime): + if st.st_mtime != old.st_mtime: new_hash = Cache.hash_digest(res_src) if new_hash != old.hash: return True From db9c592967b976a16eccd500f3e2676cfff7f29d Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 27 Dec 2023 22:59:30 -0800 Subject: [PATCH 230/279] Unify docstring detection (#4095) Co-authored-by: hauntsaninja --- CHANGES.md | 1 + src/black/linegen.py | 4 ++-- src/black/lines.py | 15 +++++++++++---- src/black/mode.py | 1 + src/black/nodes.py | 12 +++++++++++- src/black/strings.py | 2 ++ tests/data/cases/module_docstring_2.py | 2 ++ .../preview_no_blank_line_before_docstring.py | 7 +++++++ 8 files changed, 37 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2389f6d39fd..a6587cc5ceb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,7 @@ +- Format module docstrings the same as class and function docstrings (#4095) - Fix bug where spaces were not added around parenthesized walruses in subscripts, unlike other binary operators (#4109) diff --git a/src/black/linegen.py b/src/black/linegen.py index 0fd4a8d9c96..0972cf432e1 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -424,7 +424,7 @@ def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: if Preview.hex_codes_in_unicode_sequences in self.mode: normalize_unicode_escape_sequences(leaf) - if is_docstring(leaf) and not re.search(r"\\\s*\n", leaf.value): + if is_docstring(leaf, self.mode) and not re.search(r"\\\s*\n", leaf.value): # We're ignoring docstrings with backslash newline escapes because changing # indentation of those changes the AST representation of the code. if self.mode.string_normalization: @@ -477,7 +477,7 @@ def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: quote = quote_char * quote_len # It's invalid to put closing single-character quotes on a new line. - if self.mode and quote_len == 3: + if quote_len == 3: # We need to find the length of the last line of the docstring # to find if we can add the closing quotes to the line without # exceeding the maximum line length. diff --git a/src/black/lines.py b/src/black/lines.py index d153b8c2e1b..8d02267a85b 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -196,7 +196,7 @@ def is_class_paren_empty(self) -> bool: ) @property - def is_triple_quoted_string(self) -> bool: + def _is_triple_quoted_string(self) -> bool: """Is the line a triple quoted string?""" if not self or self.leaves[0].type != token.STRING: return False @@ -209,6 +209,13 @@ def is_triple_quoted_string(self) -> bool: return True return False + @property + def is_docstring(self) -> bool: + """Is the line a docstring?""" + if Preview.unify_docstring_detection not in self.mode: + return self._is_triple_quoted_string + return bool(self) and is_docstring(self.leaves[0], self.mode) + @property def is_chained_assignment(self) -> bool: """Is the line a chained assignment""" @@ -583,7 +590,7 @@ def maybe_empty_lines(self, current_line: Line) -> LinesBlock: and self.previous_block and self.previous_block.previous_block is None and len(self.previous_block.original_line.leaves) == 1 - and self.previous_block.original_line.is_triple_quoted_string + and self.previous_block.original_line.is_docstring and not (current_line.is_class or current_line.is_def) ): before = 1 @@ -690,7 +697,7 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: if ( self.previous_line and self.previous_line.is_class - and current_line.is_triple_quoted_string + and current_line.is_docstring ): if Preview.no_blank_line_before_class_docstring in current_line.mode: return 0, 1 @@ -701,7 +708,7 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: is_empty_first_line_ok = ( Preview.allow_empty_first_line_in_block in current_line.mode and ( - not is_docstring(current_line.leaves[0]) + not is_docstring(current_line.leaves[0], current_line.mode) or ( self.previous_line and self.previous_line.leaves[0] diff --git a/src/black/mode.py b/src/black/mode.py index 38b861e39ca..466b78228fc 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -195,6 +195,7 @@ class Preview(Enum): single_line_format_skip_with_multiple_comments = auto() long_case_block_line_splitting = auto() allow_form_feeds = auto() + unify_docstring_detection = auto() respect_east_asian_width = auto() diff --git a/src/black/nodes.py b/src/black/nodes.py index a4f555b4032..8e0f27e3ded 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -531,7 +531,7 @@ def is_arith_like(node: LN) -> bool: } -def is_docstring(leaf: Leaf) -> bool: +def is_docstring(leaf: Leaf, mode: Mode) -> bool: if leaf.type != token.STRING: return False @@ -539,6 +539,16 @@ def is_docstring(leaf: Leaf) -> bool: if set(prefix).intersection("bBfF"): return False + if ( + Preview.unify_docstring_detection in mode + and leaf.parent + and leaf.parent.type == syms.simple_stmt + and not leaf.parent.prev_sibling + and leaf.parent.parent + and leaf.parent.parent.type == syms.file_input + ): + return True + if prev_siblings_are( leaf.parent, [None, token.NEWLINE, token.INDENT, syms.simple_stmt] ): diff --git a/src/black/strings.py b/src/black/strings.py index 0d30f09ed11..0e0f968824b 100644 --- a/src/black/strings.py +++ b/src/black/strings.py @@ -63,6 +63,8 @@ def lines_with_leading_tabs_expanded(s: str) -> List[str]: ) else: lines.append(line) + if s.endswith("\n"): + lines.append("") return lines diff --git a/tests/data/cases/module_docstring_2.py b/tests/data/cases/module_docstring_2.py index e1f81b4d76b..1cc9aea9aea 100644 --- a/tests/data/cases/module_docstring_2.py +++ b/tests/data/cases/module_docstring_2.py @@ -1,6 +1,7 @@ # flags: --preview """I am a very helpful module docstring. +With trailing spaces: Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, @@ -38,6 +39,7 @@ # output """I am a very helpful module docstring. +With trailing spaces: Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, diff --git a/tests/data/cases/preview_no_blank_line_before_docstring.py b/tests/data/cases/preview_no_blank_line_before_docstring.py index 303035a7efb..faeaa1e46e4 100644 --- a/tests/data/cases/preview_no_blank_line_before_docstring.py +++ b/tests/data/cases/preview_no_blank_line_before_docstring.py @@ -29,6 +29,9 @@ class MultilineDocstringsAsWell: and on so many lines... """ +class SingleQuotedDocstring: + + "I'm a docstring but I don't even get triple quotes." # output @@ -57,3 +60,7 @@ class MultilineDocstringsAsWell: and on so many lines... """ + + +class SingleQuotedDocstring: + "I'm a docstring but I don't even get triple quotes." From c35924663cff4f696f9bb91ca9c7775487d95ac6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 Jan 2024 15:12:18 -0800 Subject: [PATCH 231/279] [pre-commit.ci] pre-commit autoupdate (#4139) --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2896489d724..13479565527 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,7 @@ repos: additional_dependencies: *version_check_dependencies - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort @@ -39,7 +39,7 @@ repos: exclude: ^src/blib2to3/ - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.7.1 + rev: v1.8.0 hooks: - id: mypy exclude: ^docs/conf.py @@ -58,13 +58,13 @@ repos: - hypothesmith - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.3 + rev: v4.0.0-alpha.8 hooks: - id: prettier exclude: \.github/workflows/diff_shades\.yml - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: end-of-file-fixer - id: trailing-whitespace From fe3376141c333271d3c64d7fa0e433652e2b48ff Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 1 Jan 2024 15:46:09 -0800 Subject: [PATCH 232/279] Allow empty lines at beginnings of more blocks (#4130) Fixes #4043, fixes #619 These include nested functions and methods. I think the nested function case quite clearly improves readability. I think the method case improves consistency, adherence to PEP 8 and resolves a point of contention. --- CHANGES.md | 2 ++ src/black/lines.py | 5 ++++- tests/data/cases/class_blank_parentheses.py | 1 + .../cases/preview_allow_empty_first_line.py | 19 +++++++++++++++++++ tests/data/cases/preview_form_feeds.py | 1 + 5 files changed, 27 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index a6587cc5ceb..fca88612afe 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,8 @@ - Format module docstrings the same as class and function docstrings (#4095) - Fix bug where spaces were not added around parenthesized walruses in subscripts, unlike other binary operators (#4109) +- Address a missing case in the change to allow empty lines at the beginning of all + blocks, except immediately before a docstring (#4130) ### Configuration diff --git a/src/black/lines.py b/src/black/lines.py index 8d02267a85b..4d4f47a44e8 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -745,7 +745,10 @@ def _maybe_empty_lines_for_class_or_def( # noqa: C901 if self.previous_line.depth < current_line.depth and ( self.previous_line.is_class or self.previous_line.is_def ): - return 0, 0 + if self.mode.is_pyi or not Preview.allow_empty_first_line_in_block: + return 0, 0 + else: + return 1 if user_had_newline else 0, 0 comment_to_add_newlines: Optional[LinesBlock] = None if ( diff --git a/tests/data/cases/class_blank_parentheses.py b/tests/data/cases/class_blank_parentheses.py index 1a5721a2889..3c460d9bd79 100644 --- a/tests/data/cases/class_blank_parentheses.py +++ b/tests/data/cases/class_blank_parentheses.py @@ -39,6 +39,7 @@ def test_func(self): class ClassWithEmptyFunc(object): + def func_with_blank_parentheses(): return 5 diff --git a/tests/data/cases/preview_allow_empty_first_line.py b/tests/data/cases/preview_allow_empty_first_line.py index 3e14fa15250..daf78344ad7 100644 --- a/tests/data/cases/preview_allow_empty_first_line.py +++ b/tests/data/cases/preview_allow_empty_first_line.py @@ -62,6 +62,15 @@ def method(self): pass + +def top_level( + a: int, + b: str, +) -> Whatever[Generic, Something]: + + def nested(x: int) -> int: + pass + # output def foo(): @@ -123,6 +132,16 @@ def quux(): class Cls: + def method(self): pass + + +def top_level( + a: int, + b: str, +) -> Whatever[Generic, Something]: + + def nested(x: int) -> int: + pass diff --git a/tests/data/cases/preview_form_feeds.py b/tests/data/cases/preview_form_feeds.py index c236f177a95..dc3bd6cfe2e 100644 --- a/tests/data/cases/preview_form_feeds.py +++ b/tests/data/cases/preview_form_feeds.py @@ -203,6 +203,7 @@ def bar(a=1, b: bool = False): class Baz: + def __init__(self): pass From b9ad4da2e81f6ec66d292b85f284889211e052b4 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 1 Jan 2024 16:55:25 -0800 Subject: [PATCH 233/279] Revert "confine pre-commit to stages (#3940)" (#4137) This reverts commit 7686989fc89aad5ea235a34977ebf8c81c26c4eb. --- .pre-commit-hooks.yaml | 2 -- CHANGES.md | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 54a03efe7a1..a1ff41fded8 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -4,7 +4,6 @@ name: black description: "Black: The uncompromising Python code formatter" entry: black - stages: [pre-commit, pre-merge-commit, pre-push, manual] language: python minimum_pre_commit_version: 2.9.2 require_serial: true @@ -14,7 +13,6 @@ description: "Black: The uncompromising Python code formatter (with Jupyter Notebook support)" entry: black - stages: [pre-commit, pre-merge-commit, pre-push, manual] language: python minimum_pre_commit_version: 2.9.2 require_serial: true diff --git a/CHANGES.md b/CHANGES.md index fca88612afe..360319ac964 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -50,6 +50,9 @@ +- Revert the change to run Black's pre-commit integration only on specific git hooks + (#3940) for better compatibility with older versions of pre-commit (#4137) + ### Documentation +- Fix comment handling when parenthesising conditional expressions (#4134) - Format module docstrings the same as class and function docstrings (#4095) - Fix bug where spaces were not added around parenthesized walruses in subscripts, unlike other binary operators (#4109) diff --git a/src/black/linegen.py b/src/black/linegen.py index 574c89b880c..4d468ce0f2e 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -170,8 +170,12 @@ def visit_test(self, node: Node) -> Iterator[Line]: ) if not already_parenthesized: + # Similar to logic in wrap_in_parentheses lpar = Leaf(token.LPAR, "") rpar = Leaf(token.RPAR, "") + prefix = node.prefix + node.prefix = "" + lpar.prefix = prefix node.insert_child(0, lpar) node.append_child(rpar) diff --git a/tests/data/cases/conditional_expression.py b/tests/data/cases/conditional_expression.py index c30cd76c791..76251bd9318 100644 --- a/tests/data/cases/conditional_expression.py +++ b/tests/data/cases/conditional_expression.py @@ -67,6 +67,28 @@ def something(): else ValuesListIterable ) + +def foo(wait: bool = True): + # This comment is two + # lines long + + # This is only one + time.sleep(1) if wait else None + time.sleep(1) if wait else None + + # With newline above + time.sleep(1) if wait else None + # Without newline above + time.sleep(1) if wait else None + + +a = "".join( + ( + "", # comment + "" if True else "", + ) +) + # output long_kwargs_single_line = my_function( @@ -159,3 +181,23 @@ def something(): if named else FlatValuesListIterable if flat else ValuesListIterable ) + + +def foo(wait: bool = True): + # This comment is two + # lines long + + # This is only one + time.sleep(1) if wait else None + time.sleep(1) if wait else None + + # With newline above + time.sleep(1) if wait else None + # Without newline above + time.sleep(1) if wait else None + + +a = "".join(( + "", # comment + "" if True else "", +)) From e11eaf2f44d3db5713fb99bdec966ba974b60c8c Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 1 Jan 2024 20:14:57 -0800 Subject: [PATCH 238/279] Make `blank_line_after_nested_stub_class` work for methods (#4141) Fixes #4113 Authored by dhruvmanila --- CHANGES.md | 1 + src/black/lines.py | 8 ++++---- tests/data/cases/nested_stub.py | 27 ++++++++++++++++++++++++++- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3dc0c87f89a..8fb8677dd77 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -21,6 +21,7 @@ - Remove empty lines before docstrings in async functions (#4132) - Address a missing case in the change to allow empty lines at the beginning of all blocks, except immediately before a docstring (#4130) +- For stubs, fix logic to enforce empty line after nested classes with bodies (#4141) ### Configuration diff --git a/src/black/lines.py b/src/black/lines.py index b544c5e0035..9eb5785da57 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -640,15 +640,15 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: if previous_def is not None: assert self.previous_line is not None if self.mode.is_pyi: - if depth and not current_line.is_def and self.previous_line.is_def: - # Empty lines between attributes and methods should be preserved. - before = 1 if user_had_newline else 0 - elif ( + if ( Preview.blank_line_after_nested_stub_class in self.mode and previous_def.is_class and not previous_def.is_stub_class ): before = 1 + elif depth and not current_line.is_def and self.previous_line.is_def: + # Empty lines between attributes and methods should be preserved. + before = 1 if user_had_newline else 0 elif depth: before = 0 else: diff --git a/tests/data/cases/nested_stub.py b/tests/data/cases/nested_stub.py index b81549ec115..ef13c588ce6 100644 --- a/tests/data/cases/nested_stub.py +++ b/tests/data/cases/nested_stub.py @@ -18,6 +18,18 @@ def function_definition(self): ... assignment = 1 def f2(self) -> str: ... + +class TopLevel: + class Nested1: + foo: int + def bar(self): ... + field = 1 + + class Nested2: + def bar(self): ... + foo: int + field = 1 + # output import sys @@ -41,4 +53,17 @@ def f1(self) -> str: ... def function_definition(self): ... assignment = 1 - def f2(self) -> str: ... \ No newline at end of file + def f2(self) -> str: ... + +class TopLevel: + class Nested1: + foo: int + def bar(self): ... + + field = 1 + + class Nested2: + def bar(self): ... + foo: int + + field = 1 From b7c3a9fedd4cfcc6a6a88aacc7b0f599b63d4716 Mon Sep 17 00:00:00 2001 From: Dragorn421 Date: Thu, 11 Jan 2024 16:46:17 +0100 Subject: [PATCH 239/279] Docs: Add note on `--exclude` about possibly verbose regex (#4145) Co-authored-by: Jelle Zijlstra --- docs/usage_and_configuration/the_basics.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 4f9856c6a47..b541f07907c 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -268,6 +268,11 @@ recursive searches. An empty value means no paths are excluded. Use forward slas directories on all platforms (Windows, too). By default, Black also ignores all paths listed in `.gitignore`. Changing this value will override all default exclusions. +If the regular expression contains newlines, it is treated as a +[verbose regular expression](https://docs.python.org/3/library/re.html#re.VERBOSE). This +is typically useful when setting these options in a `pyproject.toml` configuration file; +see [Configuration format](#configuration-format) for more information. + #### `--extend-exclude` Like `--exclude`, but adds additional files and directories on top of the default values From 9a331d606f3fd60cac19bfbfc3f98cbe8be2517d Mon Sep 17 00:00:00 2001 From: cobalt <61329810+RedGuy12@users.noreply.github.com> Date: Wed, 17 Jan 2024 13:04:15 -0600 Subject: [PATCH 240/279] fix: Don't allow unparenthesizing walruses (#4155) Signed-off-by: RedGuy12 <61329810+RedGuy12@users.noreply.github.com> Signed-off-by: RedGuy12 --- CHANGES.md | 1 + src/black/linegen.py | 6 +++++- tests/data/cases/walrus_in_dict.py | 7 +++++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 tests/data/cases/walrus_in_dict.py diff --git a/CHANGES.md b/CHANGES.md index 8fb8677dd77..2bd58ed49ff 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -22,6 +22,7 @@ - Address a missing case in the change to allow empty lines at the beginning of all blocks, except immediately before a docstring (#4130) - For stubs, fix logic to enforce empty line after nested classes with bodies (#4141) +- Fix crash when using a walrus in a dictionary (#4155) ### Configuration diff --git a/src/black/linegen.py b/src/black/linegen.py index 4d468ce0f2e..9a3eb0ce73f 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -242,7 +242,11 @@ def visit_dictsetmaker(self, node: Node) -> Iterator[Line]: if i == 0: continue if node.children[i - 1].type == token.COLON: - if child.type == syms.atom and child.children[0].type == token.LPAR: + if ( + child.type == syms.atom + and child.children[0].type == token.LPAR + and not is_walrus_assignment(child) + ): if maybe_make_parens_invisible_in_atom( child, parent=node, diff --git a/tests/data/cases/walrus_in_dict.py b/tests/data/cases/walrus_in_dict.py new file mode 100644 index 00000000000..c33eecd84a6 --- /dev/null +++ b/tests/data/cases/walrus_in_dict.py @@ -0,0 +1,7 @@ +# flags: --preview +{ + "is_update": (up := commit.hash in update_hashes) +} + +# output +{"is_update": (up := commit.hash in update_hashes)} From 7f60f3dbd7d2d36011fbae6c140b35802932952b Mon Sep 17 00:00:00 2001 From: Kevin Paulson Date: Fri, 19 Jan 2024 18:54:32 -0500 Subject: [PATCH 241/279] Update using_black_with_other_tools.md to ensure flake8 configuration examples are consistant (#4157) --- docs/guides/using_black_with_other_tools.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/guides/using_black_with_other_tools.md b/docs/guides/using_black_with_other_tools.md index 22c641a7420..e642a1aef33 100644 --- a/docs/guides/using_black_with_other_tools.md +++ b/docs/guides/using_black_with_other_tools.md @@ -145,7 +145,7 @@ There are a few deviations that cause incompatibilities with _Black_. ``` max-line-length = 88 -extend-ignore = E203 +extend-ignore = E203, E704 ``` #### Why those options above? @@ -184,7 +184,7 @@ extend-ignore = E203, E704 ```ini [flake8] max-line-length = 88 -extend-ignore = E203 +extend-ignore = E203, E704 ``` @@ -195,7 +195,7 @@ extend-ignore = E203 ```ini [flake8] max-line-length = 88 -extend-ignore = E203 +extend-ignore = E203, E704 ``` From 995e4ada14d63a9bec39c5fc83275d0e49742618 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 19 Jan 2024 17:13:26 -0800 Subject: [PATCH 242/279] Fix unnecessary nesting when wrapping long dict (#4135) Fixes #4129 --- CHANGES.md | 1 + src/black/linegen.py | 7 ++-- tests/data/cases/preview_long_dict_values.py | 38 ++++++++++++++++++++ 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2bd58ed49ff..1e75fb58563 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -23,6 +23,7 @@ blocks, except immediately before a docstring (#4130) - For stubs, fix logic to enforce empty line after nested classes with bodies (#4141) - Fix crash when using a walrus in a dictionary (#4155) +- Fix unnecessary parentheses when wrapping long dicts (#4135) ### Configuration diff --git a/src/black/linegen.py b/src/black/linegen.py index 9a3eb0ce73f..dd296eb801d 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -244,15 +244,14 @@ def visit_dictsetmaker(self, node: Node) -> Iterator[Line]: if node.children[i - 1].type == token.COLON: if ( child.type == syms.atom - and child.children[0].type == token.LPAR + and child.children[0].type in OPENING_BRACKETS and not is_walrus_assignment(child) ): - if maybe_make_parens_invisible_in_atom( + maybe_make_parens_invisible_in_atom( child, parent=node, remove_brackets_around_comma=False, - ): - wrap_in_parentheses(node, child, visible=False) + ) else: wrap_in_parentheses(node, child, visible=False) yield from self.visit_default(node) diff --git a/tests/data/cases/preview_long_dict_values.py b/tests/data/cases/preview_long_dict_values.py index fbbacd13d1d..54da76038dc 100644 --- a/tests/data/cases/preview_long_dict_values.py +++ b/tests/data/cases/preview_long_dict_values.py @@ -37,6 +37,26 @@ } +class Random: + def func(): + random_service.status.active_states.inactive = ( + make_new_top_level_state_from_dict( + { + "topLevelBase": { + "secondaryBase": { + "timestamp": 1234, + "latitude": 1, + "longitude": 2, + "actionTimestamp": Timestamp( + seconds=1530584000, nanos=0 + ).ToJsonString(), + } + }, + } + ) + ) + + # output @@ -89,3 +109,21 @@ } ), } + + +class Random: + def func(): + random_service.status.active_states.inactive = ( + make_new_top_level_state_from_dict({ + "topLevelBase": { + "secondaryBase": { + "timestamp": 1234, + "latitude": 1, + "longitude": 2, + "actionTimestamp": ( + Timestamp(seconds=1530584000, nanos=0).ToJsonString() + ), + } + }, + }) + ) From 6f3fb78444655f883780dcc19349226833c677c1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jan 2024 09:22:56 -0800 Subject: [PATCH 243/279] Bump actions/cache from 3 to 4 (#4162) Bumps [actions/cache](https://github.com/actions/cache) from 3 to 4. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/diff_shades.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/diff_shades.yml b/.github/workflows/diff_shades.yml index 8d8be2550b0..0e1aab00e34 100644 --- a/.github/workflows/diff_shades.yml +++ b/.github/workflows/diff_shades.yml @@ -72,7 +72,7 @@ jobs: - name: Attempt to use cached baseline analysis id: baseline-cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ matrix.baseline-analysis }} key: ${{ matrix.baseline-cache-key }} From 8fe602b1fa91dc6db682d1dba79a8a7341597271 Mon Sep 17 00:00:00 2001 From: Daniel Krzeminski Date: Mon, 22 Jan 2024 11:46:57 -0600 Subject: [PATCH 244/279] fix pathlib exception handling with symlinks (#4161) Fixes #4077 --- CHANGES.md | 2 ++ src/black/__init__.py | 6 +++++- src/black/files.py | 24 ++++++++++++++++-------- tests/test_black.py | 14 ++++++++++++++ 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1e75fb58563..f29834a3f7f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -29,6 +29,8 @@ +- Fix symlink handling, properly catch and ignore symlinks that point outside of root + (#4161) - Fix cache mtime logic that resulted in false positive cache hits (#4128) ### Packaging diff --git a/src/black/__init__.py b/src/black/__init__.py index 735ba713b8f..e3cbaab5f1d 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -49,6 +49,7 @@ find_user_pyproject_toml, gen_python_files, get_gitignore, + get_root_relative_path, normalize_path_maybe_ignore, parse_pyproject_toml, path_is_excluded, @@ -700,7 +701,10 @@ def get_sources( # Compare the logic here to the logic in `gen_python_files`. if is_stdin or path.is_file(): - root_relative_path = path.absolute().relative_to(root).as_posix() + root_relative_path = get_root_relative_path(path, root, report) + + if root_relative_path is None: + continue root_relative_path = "/" + root_relative_path diff --git a/src/black/files.py b/src/black/files.py index 858303ca1a3..65951efdbe8 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -259,14 +259,7 @@ def normalize_path_maybe_ignore( try: abspath = path if path.is_absolute() else Path.cwd() / path normalized_path = abspath.resolve() - try: - root_relative_path = normalized_path.relative_to(root).as_posix() - except ValueError: - if report: - report.path_ignored( - path, f"is a symbolic link that points outside {root}" - ) - return None + root_relative_path = get_root_relative_path(normalized_path, root, report) except OSError as e: if report: @@ -276,6 +269,21 @@ def normalize_path_maybe_ignore( return root_relative_path +def get_root_relative_path( + path: Path, + root: Path, + report: Optional[Report] = None, +) -> Optional[str]: + """Returns the file path relative to the 'root' directory""" + try: + root_relative_path = path.absolute().relative_to(root).as_posix() + except ValueError: + if report: + report.path_ignored(path, f"is a symbolic link that points outside {root}") + return None + return root_relative_path + + def _path_is_ignored( root_relative_path: str, root: Path, diff --git a/tests/test_black.py b/tests/test_black.py index 0af5fd2a1f4..2b5fab5d28d 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -2592,6 +2592,20 @@ def test_symlinks(self) -> None: outside_root_symlink.resolve.assert_called_once() ignored_symlink.resolve.assert_not_called() + def test_get_sources_with_stdin_symlink_outside_root( + self, + ) -> None: + path = THIS_DIR / "data" / "include_exclude_tests" + stdin_filename = str(path / "b/exclude/a.py") + outside_root_symlink = Path("/target_directory/a.py") + with patch("pathlib.Path.resolve", return_value=outside_root_symlink): + assert_collected_sources( + root=Path("target_directory/"), + src=["-"], + expected=[], + stdin_filename=stdin_filename, + ) + @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None)) def test_get_sources_with_stdin(self) -> None: src = ["-"] From 59b9d858a30de56801e84c31f57b53337c61647c Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 24 Jan 2024 17:06:14 -0800 Subject: [PATCH 245/279] Create the 2024 stable style (#4106) --- CHANGES.md | 40 +++++++- src/black/comments.py | 32 +++---- src/black/linegen.py | 95 +++++-------------- src/black/lines.py | 63 +++--------- src/black/mode.py | 25 +---- src/black/nodes.py | 14 +-- ...irst_line.py => allow_empty_first_line.py} | 1 - ...{preview_async_stmts.py => async_stmts.py} | 1 - tests/data/cases/comments5.py | 9 +- tests/data/cases/conditional_expression.py | 11 ++- ..._managers_38.py => context_managers_38.py} | 2 +- ..._managers_39.py => context_managers_39.py} | 2 +- ....py => context_managers_autodetect_310.py} | 2 +- ....py => context_managers_autodetect_311.py} | 2 +- ...8.py => context_managers_autodetect_38.py} | 1 - ...9.py => context_managers_autodetect_39.py} | 2 +- ...mentations.py => dummy_implementations.py} | 1 - tests/data/cases/empty_lines.py | 1 + tests/data/cases/fmtonoff.py | 8 +- tests/data/cases/fmtonoff5.py | 3 +- .../{preview_form_feeds.py => form_feeds.py} | 1 - tests/data/cases/function.py | 5 +- ...r_match.py => keep_newline_after_match.py} | 6 ++ .../data/cases/long_strings_flag_disabled.py | 16 ++-- tests/data/cases/module_docstring_1.py | 1 - tests/data/cases/module_docstring_2.py | 4 +- tests/data/cases/module_docstring_3.py | 1 - tests/data/cases/module_docstring_4.py | 1 - .../module_docstring_followed_by_class.py | 1 - .../module_docstring_followed_by_function.py | 1 - tests/data/cases/nested_stub.py | 2 +- ...g.py => no_blank_line_before_docstring.py} | 2 +- ...ching_long.py => pattern_matching_long.py} | 2 +- ....py => pattern_matching_trailing_comma.py} | 2 +- .../cases/pep604_union_types_line_breaks.py | 2 +- tests/data/cases/pep_572_py310.py | 6 +- tests/data/cases/pep_572_remove_parens.py | 8 +- .../{preview_pep_572.py => pep_572_slices.py} | 1 - ...nt_precedence.py => percent_precedence.py} | 5 +- ...op_spacing.py => power_op_spacing_long.py} | 1 - ...refer_rhs_split.py => prefer_rhs_split.py} | 1 - tests/data/cases/py310_pep572.py | 2 +- tests/data/cases/python39.py | 9 +- tests/data/cases/raw_docstring.py | 2 +- ... raw_docstring_no_string_normalization.py} | 2 +- .../remove_newline_after_code_block_open.py | 79 +++++++++------ .../data/cases/return_annotation_brackets.py | 34 +++---- ...ine_format_skip_with_multiple_comments.py} | 1 - ...ew_trailing_comma.py => trailing_comma.py} | 1 - tests/data/cases/walrus_in_dict.py | 2 + 50 files changed, 222 insertions(+), 294 deletions(-) rename tests/data/cases/{preview_allow_empty_first_line.py => allow_empty_first_line.py} (98%) rename tests/data/cases/{preview_async_stmts.py => async_stmts.py} (93%) rename tests/data/cases/{preview_context_managers_38.py => context_managers_38.py} (96%) rename tests/data/cases/{preview_context_managers_39.py => context_managers_39.py} (98%) rename tests/data/cases/{preview_context_managers_autodetect_310.py => context_managers_autodetect_310.py} (93%) rename tests/data/cases/{preview_context_managers_autodetect_311.py => context_managers_autodetect_311.py} (92%) rename tests/data/cases/{preview_context_managers_autodetect_38.py => context_managers_autodetect_38.py} (98%) rename tests/data/cases/{preview_context_managers_autodetect_39.py => context_managers_autodetect_39.py} (93%) rename tests/data/cases/{preview_dummy_implementations.py => dummy_implementations.py} (99%) rename tests/data/cases/{preview_form_feeds.py => form_feeds.py} (99%) rename tests/data/cases/{remove_newline_after_match.py => keep_newline_after_match.py} (98%) rename tests/data/cases/{preview_no_blank_line_before_docstring.py => no_blank_line_before_docstring.py} (98%) rename tests/data/cases/{preview_pattern_matching_long.py => pattern_matching_long.py} (94%) rename tests/data/cases/{preview_pattern_matching_trailing_comma.py => pattern_matching_trailing_comma.py} (92%) rename tests/data/cases/{preview_pep_572.py => pep_572_slices.py} (75%) rename tests/data/cases/{preview_percent_precedence.py => percent_precedence.py} (91%) rename tests/data/cases/{preview_power_op_spacing.py => power_op_spacing_long.py} (99%) rename tests/data/cases/{preview_prefer_rhs_split.py => prefer_rhs_split.py} (99%) rename tests/data/cases/{preview_docstring_no_string_normalization.py => raw_docstring_no_string_normalization.py} (88%) rename tests/data/cases/{preview_single_line_format_skip_with_multiple_comments.py => single_line_format_skip_with_multiple_comments.py} (97%) rename tests/data/cases/{preview_trailing_comma.py => trailing_comma.py} (97%) diff --git a/CHANGES.md b/CHANGES.md index f29834a3f7f..0e2974d706e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,22 +6,54 @@ +This release introduces the new 2024 stable style (#4106), stabilizing the following +changes: + +- Add parentheses around `if`-`else` expressions (#2278) +- Dummy class and function implementations consisting only of `...` are formatted more + compactly (#3796) +- If an assignment statement is too long, we now prefer splitting on the right-hand side + (#3368) +- Hex codes in Unicode escape sequences are now standardized to lowercase (#2916) +- Allow empty first lines at the beginning of most blocks (#3967, #4061) +- Add parentheses around long type annotations (#3899) +- Standardize on a single newline after module docstrings (#3932) +- Fix incorrect magic trailing comma handling in return types (#3916) +- Remove blank lines before class docstrings (#3692) +- Wrap multiple context managers in parentheses if combined in a single `with` statement + (#3489) +- Fix bug in line length calculations for power operations (#3942) +- Add trailing commas to collection literals even if there's a comment after the last + entry (#3393) +- When using `--skip-magic-trailing-comma` or `-C`, trailing commas are stripped from + subscript expressions with more than 1 element (#3209) +- Add extra blank lines in stubs in a few cases (#3564, #3862) +- Accept raw strings as docstrings (#3947) +- Split long lines in case blocks (#4024) +- Stop removing spaces from walrus operators within subscripts (#3823) +- Fix incorrect formatting of certain async statements (#3609) +- Allow combining `# fmt: skip` with other comments (#3959) + ### Stable style -### Preview style - - +Several bug fixes were made in features that are moved to the stable style in this +release: - Fix comment handling when parenthesising conditional expressions (#4134) -- Format module docstrings the same as class and function docstrings (#4095) - Fix bug where spaces were not added around parenthesized walruses in subscripts, unlike other binary operators (#4109) - Remove empty lines before docstrings in async functions (#4132) - Address a missing case in the change to allow empty lines at the beginning of all blocks, except immediately before a docstring (#4130) - For stubs, fix logic to enforce empty line after nested classes with bodies (#4141) + +### Preview style + + + +- Format module docstrings the same as class and function docstrings (#4095) - Fix crash when using a walrus in a dictionary (#4155) - Fix unnecessary parentheses when wrapping long dicts (#4135) diff --git a/src/black/comments.py b/src/black/comments.py index 52bb024a799..910e1b760f0 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -3,7 +3,7 @@ from functools import lru_cache from typing import Collection, Final, Iterator, List, Optional, Tuple, Union -from black.mode import Mode, Preview +from black.mode import Mode from black.nodes import ( CLOSING_BRACKETS, STANDALONE_COMMENT, @@ -390,22 +390,18 @@ def _contains_fmt_skip_comment(comment_line: str, mode: Mode) -> bool: # noqa:XXX # fmt:skip # a nice line <-- multiple comments (Preview) # pylint:XXX; fmt:skip <-- list of comments (; separated, Preview) """ - semantic_comment_blocks = ( - [ - comment_line, - *[ - _COMMENT_PREFIX + comment.strip() - for comment in comment_line.split(_COMMENT_PREFIX)[1:] - ], - *[ - _COMMENT_PREFIX + comment.strip() - for comment in comment_line.strip(_COMMENT_PREFIX).split( - _COMMENT_LIST_SEPARATOR - ) - ], - ] - if Preview.single_line_format_skip_with_multiple_comments in mode - else [comment_line] - ) + semantic_comment_blocks = [ + comment_line, + *[ + _COMMENT_PREFIX + comment.strip() + for comment in comment_line.split(_COMMENT_PREFIX)[1:] + ], + *[ + _COMMENT_PREFIX + comment.strip() + for comment in comment_line.strip(_COMMENT_PREFIX).split( + _COMMENT_LIST_SEPARATOR + ) + ], + ] return any(comment in FMT_SKIP for comment in semantic_comment_blocks) diff --git a/src/black/linegen.py b/src/black/linegen.py index dd296eb801d..a276805f2fe 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -115,10 +115,8 @@ def line(self, indent: int = 0) -> Iterator[Line]: self.current_line.depth += indent return # Line is empty, don't emit. Creating a new one unnecessary. - if ( - Preview.improved_async_statements_handling in self.mode - and len(self.current_line.leaves) == 1 - and is_async_stmt_or_funcdef(self.current_line.leaves[0]) + if len(self.current_line.leaves) == 1 and is_async_stmt_or_funcdef( + self.current_line.leaves[0] ): # Special case for async def/for/with statements. `visit_async_stmt` # adds an `ASYNC` leaf then visits the child def/for/with statement @@ -164,20 +162,19 @@ def visit_default(self, node: LN) -> Iterator[Line]: def visit_test(self, node: Node) -> Iterator[Line]: """Visit an `x if y else z` test""" - if Preview.parenthesize_conditional_expressions in self.mode: - already_parenthesized = ( - node.prev_sibling and node.prev_sibling.type == token.LPAR - ) + already_parenthesized = ( + node.prev_sibling and node.prev_sibling.type == token.LPAR + ) - if not already_parenthesized: - # Similar to logic in wrap_in_parentheses - lpar = Leaf(token.LPAR, "") - rpar = Leaf(token.RPAR, "") - prefix = node.prefix - node.prefix = "" - lpar.prefix = prefix - node.insert_child(0, lpar) - node.append_child(rpar) + if not already_parenthesized: + # Similar to logic in wrap_in_parentheses + lpar = Leaf(token.LPAR, "") + rpar = Leaf(token.RPAR, "") + prefix = node.prefix + node.prefix = "" + lpar.prefix = prefix + node.insert_child(0, lpar) + node.append_child(rpar) yield from self.visit_default(node) @@ -292,9 +289,7 @@ def visit_match_case(self, node: Node) -> Iterator[Line]: def visit_suite(self, node: Node) -> Iterator[Line]: """Visit a suite.""" - if ( - self.mode.is_pyi or Preview.dummy_implementations in self.mode - ) and is_stub_suite(node, self.mode): + if is_stub_suite(node): yield from self.visit(node.children[2]) else: yield from self.visit_default(node) @@ -308,11 +303,7 @@ def visit_simple_stmt(self, node: Node) -> Iterator[Line]: prev_type = child.type if node.parent and node.parent.type in STATEMENT: - if Preview.dummy_implementations in self.mode: - condition = is_parent_function_or_class(node) - else: - condition = self.mode.is_pyi - if condition and is_stub_body(node): + if is_parent_function_or_class(node) and is_stub_body(node): yield from self.visit_default(node) else: yield from self.line(+1) @@ -320,11 +311,7 @@ def visit_simple_stmt(self, node: Node) -> Iterator[Line]: yield from self.line(-1) else: - if ( - not (self.mode.is_pyi or Preview.dummy_implementations in self.mode) - or not node.parent - or not is_stub_suite(node.parent, self.mode) - ): + if not node.parent or not is_stub_suite(node.parent): yield from self.line() yield from self.visit_default(node) @@ -342,11 +329,7 @@ def visit_async_stmt(self, node: Node) -> Iterator[Line]: break internal_stmt = next(children) - if Preview.improved_async_statements_handling in self.mode: - yield from self.visit(internal_stmt) - else: - for child in internal_stmt.children: - yield from self.visit(child) + yield from self.visit(internal_stmt) def visit_decorators(self, node: Node) -> Iterator[Line]: """Visit decorators.""" @@ -420,10 +403,9 @@ def foo(a: int, b: float = 7): ... def foo(a: (int), b: (float) = 7): ... """ - if Preview.parenthesize_long_type_hints in self.mode: - assert len(node.children) == 3 - if maybe_make_parens_invisible_in_atom(node.children[2], parent=node): - wrap_in_parentheses(node, node.children[2], visible=False) + assert len(node.children) == 3 + if maybe_make_parens_invisible_in_atom(node.children[2], parent=node): + wrap_in_parentheses(node, node.children[2], visible=False) yield from self.visit_default(node) @@ -529,13 +511,7 @@ def __post_init__(self) -> None: self.visit_with_stmt = partial(v, keywords={"with"}, parens={"with"}) self.visit_classdef = partial(v, keywords={"class"}, parens=Ø) - # When this is moved out of preview, add ":" directly to ASSIGNMENTS in nodes.py - if Preview.parenthesize_long_type_hints in self.mode: - assignments = ASSIGNMENTS | {":"} - else: - assignments = ASSIGNMENTS - self.visit_expr_stmt = partial(v, keywords=Ø, parens=assignments) - + self.visit_expr_stmt = partial(v, keywords=Ø, parens=ASSIGNMENTS) self.visit_return_stmt = partial(v, keywords={"return"}, parens={"return"}) self.visit_import_from = partial(v, keywords=Ø, parens={"import"}) self.visit_del_stmt = partial(v, keywords=Ø, parens={"del"}) @@ -576,9 +552,7 @@ def transform_line( # We need the line string when power operators are hugging to determine if we should # split the line. Default to line_str, if no power operator are present on the line. line_str_hugging_power_ops = ( - (_hugging_power_ops_line_to_string(line, features, mode) or line_str) - if Preview.fix_power_op_line_length in mode - else line_str + _hugging_power_ops_line_to_string(line, features, mode) or line_str ) ll = mode.line_length @@ -688,9 +662,6 @@ def should_split_funcdef_with_rhs(line: Line, mode: Mode) -> bool: """If a funcdef has a magic trailing comma in the return type, then we should first split the line with rhs to respect the comma. """ - if Preview.respect_magic_trailing_comma_in_return_type not in mode: - return False - return_type_leaves: List[Leaf] = [] in_return_type = False @@ -919,9 +890,6 @@ def _maybe_split_omitting_optional_parens( try: # The RHSResult Omitting Optional Parens. rhs_oop = _first_right_hand_split(line, omit=omit) - prefer_splitting_rhs_mode = ( - Preview.prefer_splitting_right_hand_side_of_assignments in line.mode - ) is_split_right_after_equal = ( len(rhs.head.leaves) >= 2 and rhs.head.leaves[-2].type == token.EQUAL ) @@ -937,8 +905,7 @@ def _maybe_split_omitting_optional_parens( ) if ( not ( - prefer_splitting_rhs_mode - and is_split_right_after_equal + is_split_right_after_equal and rhs_head_contains_brackets and rhs_head_short_enough and rhs_head_explode_blocked_by_magic_trailing_comma @@ -1224,11 +1191,7 @@ def append_to_line(leaf: Leaf) -> Iterator[Line]: trailing_comma_safe and Feature.TRAILING_COMMA_IN_CALL in features ) - if ( - Preview.add_trailing_comma_consistently in mode - and last_leaf.type == STANDALONE_COMMENT - and leaf_idx == last_non_comment_leaf - ): + if last_leaf.type == STANDALONE_COMMENT and leaf_idx == last_non_comment_leaf: current_line = _safe_add_trailing_comma( trailing_comma_safe, delimiter_priority, current_line ) @@ -1315,11 +1278,7 @@ def normalize_invisible_parens( # noqa: C901 # Fixes a bug where invisible parens are not properly wrapped around # case blocks. - if ( - isinstance(child, Node) - and child.type == syms.case_block - and Preview.long_case_block_line_splitting in mode - ): + if isinstance(child, Node) and child.type == syms.case_block: normalize_invisible_parens( child, parens_after={"case"}, mode=mode, features=features ) @@ -1374,7 +1333,6 @@ def normalize_invisible_parens( # noqa: C901 and child.next_sibling is not None and child.next_sibling.type == token.COLON and child.value == "case" - and Preview.long_case_block_line_splitting in mode ): # A special patch for "case case:" scenario, the second occurrence # of case will be not parsed as a Python keyword. @@ -1448,7 +1406,6 @@ def _maybe_wrap_cms_in_parens( """ if ( Feature.PARENTHESIZED_CONTEXT_MANAGERS not in features - or Preview.wrap_multiple_context_managers_in_parens not in mode or len(node.children) <= 2 # If it's an atom, it's already wrapped in parens. or node.children[1].type == syms.atom diff --git a/src/black/lines.py b/src/black/lines.py index 9eb5785da57..29f87137614 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -202,9 +202,7 @@ def _is_triple_quoted_string(self) -> bool: value = self.leaves[0].value if value.startswith(('"""', "'''")): return True - if Preview.accept_raw_docstrings in self.mode and value.startswith( - ("r'''", 'r"""', "R'''", 'R"""') - ): + if value.startswith(("r'''", 'r"""', "R'''", 'R"""')): return True return False @@ -450,14 +448,8 @@ def is_complex_subscript(self, leaf: Leaf) -> bool: if subscript_start.type == syms.subscriptlist: subscript_start = child_towards(subscript_start, leaf) - # When this is moved out of preview, add syms.namedexpr_test directly to - # TEST_DESCENDANTS in nodes.py - if Preview.walrus_subscript in self.mode: - test_decendants = TEST_DESCENDANTS | {syms.namedexpr_test} - else: - test_decendants = TEST_DESCENDANTS return subscript_start is not None and any( - n.type in test_decendants for n in subscript_start.pre_order() + n.type in TEST_DESCENDANTS for n in subscript_start.pre_order() ) def enumerate_with_length( @@ -567,8 +559,7 @@ def maybe_empty_lines(self, current_line: Line) -> LinesBlock: lines (two on module-level). """ form_feed = ( - Preview.allow_form_feeds in self.mode - and current_line.depth == 0 + current_line.depth == 0 and bool(current_line.leaves) and "\f\n" in current_line.leaves[0].prefix ) @@ -582,8 +573,7 @@ def maybe_empty_lines(self, current_line: Line) -> LinesBlock: else before - previous_after ) if ( - Preview.module_docstring_newlines in current_line.mode - and self.previous_block + self.previous_block and self.previous_block.previous_block is None and len(self.previous_block.original_line.leaves) == 1 and self.previous_block.original_line.is_docstring @@ -640,11 +630,7 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: if previous_def is not None: assert self.previous_line is not None if self.mode.is_pyi: - if ( - Preview.blank_line_after_nested_stub_class in self.mode - and previous_def.is_class - and not previous_def.is_stub_class - ): + if previous_def.is_class and not previous_def.is_stub_class: before = 1 elif depth and not current_line.is_def and self.previous_line.is_def: # Empty lines between attributes and methods should be preserved. @@ -695,18 +681,12 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: and self.previous_line.is_class and current_line.is_docstring ): - if Preview.no_blank_line_before_class_docstring in current_line.mode: - return 0, 1 - return before, 1 + return 0, 1 # In preview mode, always allow blank lines, except right before a function # docstring - is_empty_first_line_ok = ( - Preview.allow_empty_first_line_in_block in current_line.mode - and ( - not current_line.is_docstring - or (self.previous_line and not self.previous_line.is_def) - ) + is_empty_first_line_ok = not current_line.is_docstring or ( + self.previous_line and not self.previous_line.is_def ) if ( @@ -736,7 +716,7 @@ def _maybe_empty_lines_for_class_or_def( # noqa: C901 if self.previous_line.depth < current_line.depth and ( self.previous_line.is_class or self.previous_line.is_def ): - if self.mode.is_pyi or not Preview.allow_empty_first_line_in_block: + if self.mode.is_pyi: return 0, 0 else: return 1 if user_had_newline else 0, 0 @@ -776,10 +756,7 @@ def _maybe_empty_lines_for_class_or_def( # noqa: C901 # Don't inspect the previous line if it's part of the body of the previous # statement in the same level, we always want a blank line if there's # something with a body preceding. - elif ( - Preview.blank_line_between_nested_and_def_stub_file in current_line.mode - and self.previous_line.depth > current_line.depth - ): + elif self.previous_line.depth > current_line.depth: newlines = 1 elif ( current_line.is_def or current_line.is_decorator @@ -800,11 +777,7 @@ def _maybe_empty_lines_for_class_or_def( # noqa: C901 newlines = 1 if current_line.depth else 2 # If a user has left no space after a dummy implementation, don't insert # new lines. This is useful for instance for @overload or Protocols. - if ( - Preview.dummy_implementations in self.mode - and self.previous_line.is_stub_def - and not user_had_newline - ): + if self.previous_line.is_stub_def and not user_had_newline: newlines = 0 if comment_to_add_newlines is not None: previous_block = comment_to_add_newlines.previous_block @@ -859,11 +832,9 @@ def is_line_short_enough( # noqa: C901 if not line_str: line_str = line_to_string(line) - width = str_width if Preview.respect_east_asian_width in mode else len - if Preview.multiline_string_handling not in mode: return ( - width(line_str) <= mode.line_length + str_width(line_str) <= mode.line_length and "\n" not in line_str # multiline strings and not line.contains_standalone_comments() ) @@ -872,10 +843,10 @@ def is_line_short_enough( # noqa: C901 return False if "\n" not in line_str: # No multiline strings (MLS) present - return width(line_str) <= mode.line_length + return str_width(line_str) <= mode.line_length first, *_, last = line_str.split("\n") - if width(first) > mode.line_length or width(last) > mode.line_length: + if str_width(first) > mode.line_length or str_width(last) > mode.line_length: return False # Traverse the AST to examine the context of the multiline string (MLS), @@ -1015,11 +986,7 @@ def can_omit_invisible_parens( return False if delimiter_count == 1: - if ( - Preview.wrap_multiple_context_managers_in_parens in line.mode - and max_priority == COMMA_PRIORITY - and rhs.head.is_with_or_async_with_stmt - ): + if max_priority == COMMA_PRIORITY and rhs.head.is_with_or_async_with_stmt: # For two context manager with statements, the optional parentheses read # better. In this case, `rhs.body` is the context managers part of # the with statement. `rhs.head` is the `with (` part on the previous diff --git a/src/black/mode.py b/src/black/mode.py index 466b78228fc..1b97f3508ee 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -168,35 +168,14 @@ def supports_feature(target_versions: Set[TargetVersion], feature: Feature) -> b class Preview(Enum): """Individual preview style features.""" - add_trailing_comma_consistently = auto() - blank_line_after_nested_stub_class = auto() - blank_line_between_nested_and_def_stub_file = auto() hex_codes_in_unicode_sequences = auto() - improved_async_statements_handling = auto() - multiline_string_handling = auto() - no_blank_line_before_class_docstring = auto() - prefer_splitting_right_hand_side_of_assignments = auto() # NOTE: string_processing requires wrap_long_dict_values_in_parens # for https://github.com/psf/black/issues/3117 to be fixed. string_processing = auto() - parenthesize_conditional_expressions = auto() - parenthesize_long_type_hints = auto() - respect_magic_trailing_comma_in_return_type = auto() - skip_magic_trailing_comma_in_subscript = auto() - wrap_long_dict_values_in_parens = auto() - wrap_multiple_context_managers_in_parens = auto() - dummy_implementations = auto() - walrus_subscript = auto() - module_docstring_newlines = auto() - accept_raw_docstrings = auto() - fix_power_op_line_length = auto() hug_parens_with_braces_and_square_brackets = auto() - allow_empty_first_line_in_block = auto() - single_line_format_skip_with_multiple_comments = auto() - long_case_block_line_splitting = auto() - allow_form_feeds = auto() unify_docstring_detection = auto() - respect_east_asian_width = auto() + wrap_long_dict_values_in_parens = auto() + multiline_string_handling = auto() class Deprecated(UserWarning): diff --git a/src/black/nodes.py b/src/black/nodes.py index 7ee2df2e061..a8869cba234 100644 --- a/src/black/nodes.py +++ b/src/black/nodes.py @@ -104,6 +104,7 @@ syms.trailer, syms.term, syms.power, + syms.namedexpr_test, } TYPED_NAMES: Final = {syms.tname, syms.tname_star} ASSIGNMENTS: Final = { @@ -121,6 +122,7 @@ ">>=", "**=", "//=", + ":", } IMPLICIT_TUPLE: Final = {syms.testlist, syms.testlist_star_expr, syms.exprlist} @@ -346,9 +348,7 @@ def whitespace(leaf: Leaf, *, complex_subscript: bool, mode: Mode) -> str: # no return NO - elif Preview.walrus_subscript in mode and ( - t == token.COLONEQUAL or prev.type == token.COLONEQUAL - ): + elif t == token.COLONEQUAL or prev.type == token.COLONEQUAL: return SPACE elif not complex_subscript: @@ -753,13 +753,9 @@ def is_function_or_class(node: Node) -> bool: return node.type in {syms.funcdef, syms.classdef, syms.async_funcdef} -def is_stub_suite(node: Node, mode: Mode) -> bool: +def is_stub_suite(node: Node) -> bool: """Return True if `node` is a suite with a stub body.""" - if ( - node.parent is not None - and Preview.dummy_implementations in mode - and not is_parent_function_or_class(node) - ): + if node.parent is not None and not is_parent_function_or_class(node): return False # If there is a comment, we want to keep it. diff --git a/tests/data/cases/preview_allow_empty_first_line.py b/tests/data/cases/allow_empty_first_line.py similarity index 98% rename from tests/data/cases/preview_allow_empty_first_line.py rename to tests/data/cases/allow_empty_first_line.py index 4269987305d..32a170a97d0 100644 --- a/tests/data/cases/preview_allow_empty_first_line.py +++ b/tests/data/cases/allow_empty_first_line.py @@ -1,4 +1,3 @@ -# flags: --preview def foo(): """ Docstring diff --git a/tests/data/cases/preview_async_stmts.py b/tests/data/cases/async_stmts.py similarity index 93% rename from tests/data/cases/preview_async_stmts.py rename to tests/data/cases/async_stmts.py index 0a7671be5a6..fe9594b2164 100644 --- a/tests/data/cases/preview_async_stmts.py +++ b/tests/data/cases/async_stmts.py @@ -1,4 +1,3 @@ -# flags: --preview async def func() -> (int): return 0 diff --git a/tests/data/cases/comments5.py b/tests/data/cases/comments5.py index bda40619f62..4270d3a09a2 100644 --- a/tests/data/cases/comments5.py +++ b/tests/data/cases/comments5.py @@ -45,8 +45,7 @@ def wat(): @deco2(with_args=True) # leading 3 @deco3 -def decorated1(): - ... +def decorated1(): ... # leading 1 @@ -54,8 +53,7 @@ def decorated1(): # leading 2 @deco2(with_args=True) # leading function comment -def decorated1(): - ... +def decorated1(): ... # Note: this is fixed in @@ -65,8 +63,7 @@ def decorated1(): # This comment should be split from `some_instruction` by two lines but isn't. -def g(): - ... +def g(): ... if __name__ == "__main__": diff --git a/tests/data/cases/conditional_expression.py b/tests/data/cases/conditional_expression.py index 76251bd9318..f65d6fb00e7 100644 --- a/tests/data/cases/conditional_expression.py +++ b/tests/data/cases/conditional_expression.py @@ -1,4 +1,3 @@ -# flags: --preview long_kwargs_single_line = my_function( foo="test, this is a sample value", bar=some_long_value_name_foo_bar_baz if some_boolean_variable else some_fallback_value_foo_bar_baz, @@ -197,7 +196,9 @@ def foo(wait: bool = True): time.sleep(1) if wait else None -a = "".join(( - "", # comment - "" if True else "", -)) +a = "".join( + ( + "", # comment + "" if True else "", + ) +) diff --git a/tests/data/cases/preview_context_managers_38.py b/tests/data/cases/context_managers_38.py similarity index 96% rename from tests/data/cases/preview_context_managers_38.py rename to tests/data/cases/context_managers_38.py index 719d94fdcc5..54fb97c708b 100644 --- a/tests/data/cases/preview_context_managers_38.py +++ b/tests/data/cases/context_managers_38.py @@ -1,4 +1,4 @@ -# flags: --preview --minimum-version=3.8 +# flags: --minimum-version=3.8 with \ make_context_manager1() as cm1, \ make_context_manager2() as cm2, \ diff --git a/tests/data/cases/preview_context_managers_39.py b/tests/data/cases/context_managers_39.py similarity index 98% rename from tests/data/cases/preview_context_managers_39.py rename to tests/data/cases/context_managers_39.py index 589e00ad187..60fd1a56409 100644 --- a/tests/data/cases/preview_context_managers_39.py +++ b/tests/data/cases/context_managers_39.py @@ -1,4 +1,4 @@ -# flags: --preview --minimum-version=3.9 +# flags: --minimum-version=3.9 with \ make_context_manager1() as cm1, \ make_context_manager2() as cm2, \ diff --git a/tests/data/cases/preview_context_managers_autodetect_310.py b/tests/data/cases/context_managers_autodetect_310.py similarity index 93% rename from tests/data/cases/preview_context_managers_autodetect_310.py rename to tests/data/cases/context_managers_autodetect_310.py index a9e31076f03..80f211032e5 100644 --- a/tests/data/cases/preview_context_managers_autodetect_310.py +++ b/tests/data/cases/context_managers_autodetect_310.py @@ -1,4 +1,4 @@ -# flags: --preview --minimum-version=3.10 +# flags: --minimum-version=3.10 # This file uses pattern matching introduced in Python 3.10. diff --git a/tests/data/cases/preview_context_managers_autodetect_311.py b/tests/data/cases/context_managers_autodetect_311.py similarity index 92% rename from tests/data/cases/preview_context_managers_autodetect_311.py rename to tests/data/cases/context_managers_autodetect_311.py index af1e83fe74c..020c4cea967 100644 --- a/tests/data/cases/preview_context_managers_autodetect_311.py +++ b/tests/data/cases/context_managers_autodetect_311.py @@ -1,4 +1,4 @@ -# flags: --preview --minimum-version=3.11 +# flags: --minimum-version=3.11 # This file uses except* clause in Python 3.11. diff --git a/tests/data/cases/preview_context_managers_autodetect_38.py b/tests/data/cases/context_managers_autodetect_38.py similarity index 98% rename from tests/data/cases/preview_context_managers_autodetect_38.py rename to tests/data/cases/context_managers_autodetect_38.py index 25217a40604..79e438b995e 100644 --- a/tests/data/cases/preview_context_managers_autodetect_38.py +++ b/tests/data/cases/context_managers_autodetect_38.py @@ -1,4 +1,3 @@ -# flags: --preview # This file doesn't use any Python 3.9+ only grammars. diff --git a/tests/data/cases/preview_context_managers_autodetect_39.py b/tests/data/cases/context_managers_autodetect_39.py similarity index 93% rename from tests/data/cases/preview_context_managers_autodetect_39.py rename to tests/data/cases/context_managers_autodetect_39.py index 3f72e48db9d..98e674b2f9d 100644 --- a/tests/data/cases/preview_context_managers_autodetect_39.py +++ b/tests/data/cases/context_managers_autodetect_39.py @@ -1,4 +1,4 @@ -# flags: --preview --minimum-version=3.9 +# flags: --minimum-version=3.9 # This file uses parenthesized context managers introduced in Python 3.9. diff --git a/tests/data/cases/preview_dummy_implementations.py b/tests/data/cases/dummy_implementations.py similarity index 99% rename from tests/data/cases/preview_dummy_implementations.py rename to tests/data/cases/dummy_implementations.py index 3cd392c9587..0a52c081bcc 100644 --- a/tests/data/cases/preview_dummy_implementations.py +++ b/tests/data/cases/dummy_implementations.py @@ -1,4 +1,3 @@ -# flags: --preview from typing import NoReturn, Protocol, Union, overload class Empty: diff --git a/tests/data/cases/empty_lines.py b/tests/data/cases/empty_lines.py index 4fd47b93dca..4c03e432383 100644 --- a/tests/data/cases/empty_lines.py +++ b/tests/data/cases/empty_lines.py @@ -119,6 +119,7 @@ def f(): if not prev: prevp = preceding_leaf(p) if not prevp or prevp.type in OPENING_BRACKETS: + return NO if prevp.type == token.EQUAL: diff --git a/tests/data/cases/fmtonoff.py b/tests/data/cases/fmtonoff.py index d1f15cd5c8b..8af94563af8 100644 --- a/tests/data/cases/fmtonoff.py +++ b/tests/data/cases/fmtonoff.py @@ -243,12 +243,8 @@ def spaces_types( g: int = 1 if False else 2, h: str = "", i: str = r"", -): - ... - - -def spaces2(result=_core.Value(None)): - ... +): ... +def spaces2(result=_core.Value(None)): ... something = { diff --git a/tests/data/cases/fmtonoff5.py b/tests/data/cases/fmtonoff5.py index 181151b6bd6..4c134a9eea3 100644 --- a/tests/data/cases/fmtonoff5.py +++ b/tests/data/cases/fmtonoff5.py @@ -161,8 +161,7 @@ def this_wont_be_formatted ( self ) -> str: ... class Factory(t.Protocol): - def this_will_be_formatted(self, **kwargs) -> Named: - ... + def this_will_be_formatted(self, **kwargs) -> Named: ... # fmt: on diff --git a/tests/data/cases/preview_form_feeds.py b/tests/data/cases/form_feeds.py similarity index 99% rename from tests/data/cases/preview_form_feeds.py rename to tests/data/cases/form_feeds.py index dc3bd6cfe2e..48ffc98106b 100644 --- a/tests/data/cases/preview_form_feeds.py +++ b/tests/data/cases/form_feeds.py @@ -1,4 +1,3 @@ -# flags: --preview # Warning! This file contains form feeds (ASCII 0x0C, often represented by \f or ^L). diff --git a/tests/data/cases/function.py b/tests/data/cases/function.py index 2d642c8731b..4e3f91fd8b1 100644 --- a/tests/data/cases/function.py +++ b/tests/data/cases/function.py @@ -158,10 +158,7 @@ def spaces_types( g: int = 1 if False else 2, h: str = "", i: str = r"", -): - ... - - +): ... def spaces2(result=_core.Value(None)): assert fut is self._read_fut, (fut, self._read_fut) diff --git a/tests/data/cases/remove_newline_after_match.py b/tests/data/cases/keep_newline_after_match.py similarity index 98% rename from tests/data/cases/remove_newline_after_match.py rename to tests/data/cases/keep_newline_after_match.py index fe6592b664d..dbeccce6264 100644 --- a/tests/data/cases/remove_newline_after_match.py +++ b/tests/data/cases/keep_newline_after_match.py @@ -21,15 +21,21 @@ def http_status(status): # output def http_status(status): + match status: + case 400: + return "Bad request" case 401: + return "Unauthorized" case 403: + return "Forbidden" case 404: + return "Not found" \ No newline at end of file diff --git a/tests/data/cases/long_strings_flag_disabled.py b/tests/data/cases/long_strings_flag_disabled.py index db3954e3abd..d81c331cab2 100644 --- a/tests/data/cases/long_strings_flag_disabled.py +++ b/tests/data/cases/long_strings_flag_disabled.py @@ -43,8 +43,10 @@ % ( "formatted", "string", - ): "This is a really really really long string that has to go inside of a dictionary. It is %s bad (#%d)." - % ("soooo", 2), + ): ( + "This is a really really really long string that has to go inside of a dictionary. It is %s bad (#%d)." + % ("soooo", 2) + ), } func_with_keywords( @@ -254,10 +256,12 @@ + CONCATENATED + "using the '+' operator." ) -annotated_variable: Final = "This is a large string that has a type annotation attached to it. A type annotation should NOT stop a long string from being wrapped." -annotated_variable: Literal[ - "fakse_literal" -] = "This is a large string that has a type annotation attached to it. A type annotation should NOT stop a long string from being wrapped." +annotated_variable: Final = ( + "This is a large string that has a type annotation attached to it. A type annotation should NOT stop a long string from being wrapped." +) +annotated_variable: Literal["fakse_literal"] = ( + "This is a large string that has a type annotation attached to it. A type annotation should NOT stop a long string from being wrapped." +) backslashes = "This is a really long string with \"embedded\" double quotes and 'single' quotes that also handles checking for an even number of backslashes \\" backslashes = "This is a really long string with \"embedded\" double quotes and 'single' quotes that also handles checking for an even number of backslashes \\\\" diff --git a/tests/data/cases/module_docstring_1.py b/tests/data/cases/module_docstring_1.py index d5897b4db60..5751154f7f0 100644 --- a/tests/data/cases/module_docstring_1.py +++ b/tests/data/cases/module_docstring_1.py @@ -1,4 +1,3 @@ -# flags: --preview """Single line module-level docstring should be followed by single newline.""" diff --git a/tests/data/cases/module_docstring_2.py b/tests/data/cases/module_docstring_2.py index 1cc9aea9aea..ac486096c02 100644 --- a/tests/data/cases/module_docstring_2.py +++ b/tests/data/cases/module_docstring_2.py @@ -1,7 +1,7 @@ # flags: --preview """I am a very helpful module docstring. -With trailing spaces: +With trailing spaces (only removed with unify_docstring_detection on): Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, @@ -39,7 +39,7 @@ # output """I am a very helpful module docstring. -With trailing spaces: +With trailing spaces (only removed with unify_docstring_detection on): Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, diff --git a/tests/data/cases/module_docstring_3.py b/tests/data/cases/module_docstring_3.py index 0631e136a3d..3d0058dd554 100644 --- a/tests/data/cases/module_docstring_3.py +++ b/tests/data/cases/module_docstring_3.py @@ -1,4 +1,3 @@ -# flags: --preview """Single line module-level docstring should be followed by single newline.""" a = 1 diff --git a/tests/data/cases/module_docstring_4.py b/tests/data/cases/module_docstring_4.py index 515174dcc04..b1720078f71 100644 --- a/tests/data/cases/module_docstring_4.py +++ b/tests/data/cases/module_docstring_4.py @@ -1,4 +1,3 @@ -# flags: --preview """Single line module-level docstring should be followed by single newline.""" a = 1 diff --git a/tests/data/cases/module_docstring_followed_by_class.py b/tests/data/cases/module_docstring_followed_by_class.py index 6fdbfc8c240..c291e61b960 100644 --- a/tests/data/cases/module_docstring_followed_by_class.py +++ b/tests/data/cases/module_docstring_followed_by_class.py @@ -1,4 +1,3 @@ -# flags: --preview """Two blank lines between module docstring and a class.""" class MyClass: pass diff --git a/tests/data/cases/module_docstring_followed_by_function.py b/tests/data/cases/module_docstring_followed_by_function.py index 5913a59e1fe..fd29b98da8e 100644 --- a/tests/data/cases/module_docstring_followed_by_function.py +++ b/tests/data/cases/module_docstring_followed_by_function.py @@ -1,4 +1,3 @@ -# flags: --preview """Two blank lines between module docstring and a function def.""" def function(): pass diff --git a/tests/data/cases/nested_stub.py b/tests/data/cases/nested_stub.py index ef13c588ce6..40ca11e9330 100644 --- a/tests/data/cases/nested_stub.py +++ b/tests/data/cases/nested_stub.py @@ -1,4 +1,4 @@ -# flags: --pyi --preview +# flags: --pyi import sys class Outer: diff --git a/tests/data/cases/preview_no_blank_line_before_docstring.py b/tests/data/cases/no_blank_line_before_docstring.py similarity index 98% rename from tests/data/cases/preview_no_blank_line_before_docstring.py rename to tests/data/cases/no_blank_line_before_docstring.py index faeaa1e46e4..ced125fef78 100644 --- a/tests/data/cases/preview_no_blank_line_before_docstring.py +++ b/tests/data/cases/no_blank_line_before_docstring.py @@ -1,4 +1,3 @@ -# flags: --preview def line_before_docstring(): """Please move me up""" @@ -63,4 +62,5 @@ class MultilineDocstringsAsWell: class SingleQuotedDocstring: + "I'm a docstring but I don't even get triple quotes." diff --git a/tests/data/cases/preview_pattern_matching_long.py b/tests/data/cases/pattern_matching_long.py similarity index 94% rename from tests/data/cases/preview_pattern_matching_long.py rename to tests/data/cases/pattern_matching_long.py index df849fdc4f2..9a944c9d0c9 100644 --- a/tests/data/cases/preview_pattern_matching_long.py +++ b/tests/data/cases/pattern_matching_long.py @@ -1,4 +1,4 @@ -# flags: --preview --minimum-version=3.10 +# flags: --minimum-version=3.10 match x: case "abcd" | "abcd" | "abcd" : pass diff --git a/tests/data/cases/preview_pattern_matching_trailing_comma.py b/tests/data/cases/pattern_matching_trailing_comma.py similarity index 92% rename from tests/data/cases/preview_pattern_matching_trailing_comma.py rename to tests/data/cases/pattern_matching_trailing_comma.py index e6c0d88bb80..5660b0f6a14 100644 --- a/tests/data/cases/preview_pattern_matching_trailing_comma.py +++ b/tests/data/cases/pattern_matching_trailing_comma.py @@ -1,4 +1,4 @@ -# flags: --preview --minimum-version=3.10 +# flags: --minimum-version=3.10 match maybe, multiple: case perhaps, 5: pass diff --git a/tests/data/cases/pep604_union_types_line_breaks.py b/tests/data/cases/pep604_union_types_line_breaks.py index fee2b840494..745bc9e8b02 100644 --- a/tests/data/cases/pep604_union_types_line_breaks.py +++ b/tests/data/cases/pep604_union_types_line_breaks.py @@ -1,4 +1,4 @@ -# flags: --preview --minimum-version=3.10 +# flags: --minimum-version=3.10 # This has always worked z= Loooooooooooooooooooooooong | Loooooooooooooooooooooooong | Loooooooooooooooooooooooong | Loooooooooooooooooooooooong diff --git a/tests/data/cases/pep_572_py310.py b/tests/data/cases/pep_572_py310.py index 9f999deeb89..ba488d4741c 100644 --- a/tests/data/cases/pep_572_py310.py +++ b/tests/data/cases/pep_572_py310.py @@ -1,8 +1,8 @@ # flags: --minimum-version=3.10 # Unparenthesized walruses are now allowed in indices since Python 3.10. -x[a:=0] -x[a:=0, b:=1] -x[5, b:=0] +x[a := 0] +x[a := 0, b := 1] +x[5, b := 0] # Walruses are allowed inside generator expressions on function calls since 3.10. if any(match := pattern_error.match(s) for s in buffer): diff --git a/tests/data/cases/pep_572_remove_parens.py b/tests/data/cases/pep_572_remove_parens.py index 24f1ac29168..f0026ceb032 100644 --- a/tests/data/cases/pep_572_remove_parens.py +++ b/tests/data/cases/pep_572_remove_parens.py @@ -96,18 +96,16 @@ async def await_the_walrus(): foo(x=(y := f(x))) -def foo(answer=(p := 42)): - ... +def foo(answer=(p := 42)): ... -def foo2(answer: (p := 42) = 5): - ... +def foo2(answer: (p := 42) = 5): ... lambda: (x := 1) a[(x := 12)] -a[:(x := 13)] +a[: (x := 13)] # we don't touch expressions in f-strings but if we do one day, don't break 'em f"{(x:=10)}" diff --git a/tests/data/cases/preview_pep_572.py b/tests/data/cases/pep_572_slices.py similarity index 75% rename from tests/data/cases/preview_pep_572.py rename to tests/data/cases/pep_572_slices.py index 75ad0cc4176..aa772b1f1f5 100644 --- a/tests/data/cases/preview_pep_572.py +++ b/tests/data/cases/pep_572_slices.py @@ -1,4 +1,3 @@ -# flags: --preview x[(a:=0):] x[:(a:=0)] diff --git a/tests/data/cases/preview_percent_precedence.py b/tests/data/cases/percent_precedence.py similarity index 91% rename from tests/data/cases/preview_percent_precedence.py rename to tests/data/cases/percent_precedence.py index aeaf450ff5e..7822e42c69d 100644 --- a/tests/data/cases/preview_percent_precedence.py +++ b/tests/data/cases/percent_precedence.py @@ -1,4 +1,3 @@ -# flags: --preview ("" % a) ** 2 ("" % a)[0] ("" % a)() @@ -31,9 +30,9 @@ 2 // ("" % a) 2 % ("" % a) +("" % a) -b + "" % a +b + ("" % a) -("" % a) -b - "" % a +b - ("" % a) b + -("" % a) ~("" % a) 2 ** ("" % a) diff --git a/tests/data/cases/preview_power_op_spacing.py b/tests/data/cases/power_op_spacing_long.py similarity index 99% rename from tests/data/cases/preview_power_op_spacing.py rename to tests/data/cases/power_op_spacing_long.py index 650c6fecb20..30e6eb788b3 100644 --- a/tests/data/cases/preview_power_op_spacing.py +++ b/tests/data/cases/power_op_spacing_long.py @@ -1,4 +1,3 @@ -# flags: --preview a = 1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1 b = 1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1**1 c = 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 ** 1 diff --git a/tests/data/cases/preview_prefer_rhs_split.py b/tests/data/cases/prefer_rhs_split.py similarity index 99% rename from tests/data/cases/preview_prefer_rhs_split.py rename to tests/data/cases/prefer_rhs_split.py index 28d89c368c0..f3d9fd67251 100644 --- a/tests/data/cases/preview_prefer_rhs_split.py +++ b/tests/data/cases/prefer_rhs_split.py @@ -1,4 +1,3 @@ -# flags: --preview first_item, second_item = ( some_looooooooong_module.some_looooooooooooooong_function_name( first_argument, second_argument, third_argument diff --git a/tests/data/cases/py310_pep572.py b/tests/data/cases/py310_pep572.py index 172be3898d6..73fbe44d42c 100644 --- a/tests/data/cases/py310_pep572.py +++ b/tests/data/cases/py310_pep572.py @@ -1,4 +1,4 @@ -# flags: --preview --minimum-version=3.10 +# flags: --minimum-version=3.10 x[a:=0] x[a := 0] x[a := 0, b := 1] diff --git a/tests/data/cases/python39.py b/tests/data/cases/python39.py index 1b9536c1529..85eddc38e00 100644 --- a/tests/data/cases/python39.py +++ b/tests/data/cases/python39.py @@ -15,19 +15,16 @@ def f(): # output @relaxed_decorator[0] -def f(): - ... +def f(): ... @relaxed_decorator[ extremely_long_name_that_definitely_will_not_fit_on_one_line_of_standard_length ] -def f(): - ... +def f(): ... @extremely_long_variable_name_that_doesnt_fit := complex.expression( with_long="arguments_value_that_wont_fit_at_the_end_of_the_line" ) -def f(): - ... \ No newline at end of file +def f(): ... \ No newline at end of file diff --git a/tests/data/cases/raw_docstring.py b/tests/data/cases/raw_docstring.py index 751fd3201df..7f88bb2de86 100644 --- a/tests/data/cases/raw_docstring.py +++ b/tests/data/cases/raw_docstring.py @@ -1,4 +1,4 @@ -# flags: --preview --skip-string-normalization +# flags: --skip-string-normalization class C: r"""Raw""" diff --git a/tests/data/cases/preview_docstring_no_string_normalization.py b/tests/data/cases/raw_docstring_no_string_normalization.py similarity index 88% rename from tests/data/cases/preview_docstring_no_string_normalization.py rename to tests/data/cases/raw_docstring_no_string_normalization.py index 712c7364f51..a201c1e8fae 100644 --- a/tests/data/cases/preview_docstring_no_string_normalization.py +++ b/tests/data/cases/raw_docstring_no_string_normalization.py @@ -1,4 +1,4 @@ -# flags: --preview --skip-string-normalization +# flags: --skip-string-normalization def do_not_touch_this_prefix(): R"""There was a bug where docstring prefixes would be normalized even with -S.""" diff --git a/tests/data/cases/remove_newline_after_code_block_open.py b/tests/data/cases/remove_newline_after_code_block_open.py index ef2e5c2f6f5..6622e8afb7d 100644 --- a/tests/data/cases/remove_newline_after_code_block_open.py +++ b/tests/data/cases/remove_newline_after_code_block_open.py @@ -3,14 +3,14 @@ def foo1(): - print("The newline above me should be deleted!") + print("The newline above me should be kept!") def foo2(): - print("All the newlines above me should be deleted!") + print("All the newlines above me should be kept!") def foo3(): @@ -30,31 +30,31 @@ def foo4(): class Foo: def bar(self): - print("The newline above me should be deleted!") + print("The newline above me should be kept!") for i in range(5): - print(f"{i}) The line above me should be removed!") + print(f"{i}) The line above me should be kept!") for i in range(5): - print(f"{i}) The lines above me should be removed!") + print(f"{i}) The lines above me should be kept!") for i in range(5): for j in range(7): - print(f"{i}) The lines above me should be removed!") + print(f"{i}) The lines above me should be kept!") if random.randint(0, 3) == 0: - print("The new line above me is about to be removed!") + print("The new line above me will be kept!") if random.randint(0, 3) == 0: @@ -62,43 +62,45 @@ def bar(self): - print("The new lines above me is about to be removed!") + print("The new lines above me will be kept!") if random.randint(0, 3) == 0: + if random.uniform(0, 1) > 0.5: - print("Two lines above me are about to be removed!") + + print("Two lines above me will be kept!") while True: - print("The newline above me should be deleted!") + print("The newline above me should be kept!") while True: - print("The newlines above me should be deleted!") + print("The newlines above me should be kept!") while True: while False: - print("The newlines above me should be deleted!") + print("The newlines above me should be kept!") with open("/path/to/file.txt", mode="w") as file: - file.write("The new line above me is about to be removed!") + file.write("The new line above me will be kept!") with open("/path/to/file.txt", mode="w") as file: - file.write("The new lines above me is about to be removed!") + file.write("The new lines above me will be kept!") with open("/path/to/file.txt", mode="r") as read_file: @@ -113,20 +115,24 @@ def bar(self): def foo1(): - print("The newline above me should be deleted!") + + print("The newline above me should be kept!") def foo2(): - print("All the newlines above me should be deleted!") + + print("All the newlines above me should be kept!") def foo3(): + print("No newline above me!") print("There is a newline above me, and that's OK!") def foo4(): + # There is a comment here print("The newline above me should not be deleted!") @@ -134,56 +140,73 @@ def foo4(): class Foo: def bar(self): - print("The newline above me should be deleted!") + + print("The newline above me should be kept!") for i in range(5): - print(f"{i}) The line above me should be removed!") + + print(f"{i}) The line above me should be kept!") for i in range(5): - print(f"{i}) The lines above me should be removed!") + + print(f"{i}) The lines above me should be kept!") for i in range(5): + for j in range(7): - print(f"{i}) The lines above me should be removed!") + + print(f"{i}) The lines above me should be kept!") if random.randint(0, 3) == 0: - print("The new line above me is about to be removed!") + + print("The new line above me will be kept!") if random.randint(0, 3) == 0: - print("The new lines above me is about to be removed!") + + print("The new lines above me will be kept!") if random.randint(0, 3) == 0: + if random.uniform(0, 1) > 0.5: - print("Two lines above me are about to be removed!") + + print("Two lines above me will be kept!") while True: - print("The newline above me should be deleted!") + + print("The newline above me should be kept!") while True: - print("The newlines above me should be deleted!") + + print("The newlines above me should be kept!") while True: + while False: - print("The newlines above me should be deleted!") + + print("The newlines above me should be kept!") with open("/path/to/file.txt", mode="w") as file: - file.write("The new line above me is about to be removed!") + + file.write("The new line above me will be kept!") with open("/path/to/file.txt", mode="w") as file: - file.write("The new lines above me is about to be removed!") + + file.write("The new lines above me will be kept!") with open("/path/to/file.txt", mode="r") as read_file: + with open("/path/to/output_file.txt", mode="w") as write_file: + write_file.writelines(read_file.readlines()) diff --git a/tests/data/cases/return_annotation_brackets.py b/tests/data/cases/return_annotation_brackets.py index 8509ecdb92c..ed05bed61f4 100644 --- a/tests/data/cases/return_annotation_brackets.py +++ b/tests/data/cases/return_annotation_brackets.py @@ -88,7 +88,6 @@ def foo() -> tuple[int, int, int,]: return 2 # Magic trailing comma example, with params -# this is broken - the trailing comma is transferred to the param list. Fixed in preview def foo(a,b) -> tuple[int, int, int,]: return 2 @@ -194,30 +193,27 @@ def foo() -> tuple[int, int, int]: return 2 -def foo() -> ( - tuple[ - loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong, - loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong, - loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong, - ] -): +def foo() -> tuple[ + loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong, + loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong, + loooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong, +]: return 2 # Magic trailing comma example -def foo() -> ( - tuple[ - int, - int, - int, - ] -): +def foo() -> tuple[ + int, + int, + int, +]: return 2 # Magic trailing comma example, with params -# this is broken - the trailing comma is transferred to the param list. Fixed in preview -def foo( - a, b -) -> tuple[int, int, int,]: +def foo(a, b) -> tuple[ + int, + int, + int, +]: return 2 diff --git a/tests/data/cases/preview_single_line_format_skip_with_multiple_comments.py b/tests/data/cases/single_line_format_skip_with_multiple_comments.py similarity index 97% rename from tests/data/cases/preview_single_line_format_skip_with_multiple_comments.py rename to tests/data/cases/single_line_format_skip_with_multiple_comments.py index efde662baa8..7212740fc42 100644 --- a/tests/data/cases/preview_single_line_format_skip_with_multiple_comments.py +++ b/tests/data/cases/single_line_format_skip_with_multiple_comments.py @@ -1,4 +1,3 @@ -# flags: --preview foo = 123 # fmt: skip # noqa: E501 # pylint bar = ( 123 , diff --git a/tests/data/cases/preview_trailing_comma.py b/tests/data/cases/trailing_comma.py similarity index 97% rename from tests/data/cases/preview_trailing_comma.py rename to tests/data/cases/trailing_comma.py index bba7e7ad16d..5b09c664606 100644 --- a/tests/data/cases/preview_trailing_comma.py +++ b/tests/data/cases/trailing_comma.py @@ -1,4 +1,3 @@ -# flags: --preview e = { "a": fun(msg, "ts"), "longggggggggggggggid": ..., diff --git a/tests/data/cases/walrus_in_dict.py b/tests/data/cases/walrus_in_dict.py index c33eecd84a6..c91ad9e8611 100644 --- a/tests/data/cases/walrus_in_dict.py +++ b/tests/data/cases/walrus_in_dict.py @@ -1,7 +1,9 @@ # flags: --preview +# This is testing an issue that is specific to the preview style { "is_update": (up := commit.hash in update_hashes) } # output +# This is testing an issue that is specific to the preview style {"is_update": (up := commit.hash in update_hashes)} From a5196e6f1f450e4c8da0e514e01873a0cc1e1a3c Mon Sep 17 00:00:00 2001 From: cobalt <61329810+RedGuy12@users.noreply.github.com> Date: Thu, 25 Jan 2024 03:31:49 -0600 Subject: [PATCH 246/279] fix: Don't normalize whitespace before fmt:skip comments (#4146) Signed-off-by: RedGuy12 --- CHANGES.md | 1 + src/black/comments.py | 14 +++++++++++--- src/black/mode.py | 1 + tests/data/cases/fmtskip9.py | 9 +++++++++ 4 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 tests/data/cases/fmtskip9.py diff --git a/CHANGES.md b/CHANGES.md index 0e2974d706e..2fe14cd5246 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -56,6 +56,7 @@ release: - Format module docstrings the same as class and function docstrings (#4095) - Fix crash when using a walrus in a dictionary (#4155) - Fix unnecessary parentheses when wrapping long dicts (#4135) +- Stop normalizing spaces before `# fmt: skip` comments (#4146) ### Configuration diff --git a/src/black/comments.py b/src/black/comments.py index 910e1b760f0..ea54e2468c9 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -3,7 +3,7 @@ from functools import lru_cache from typing import Collection, Final, Iterator, List, Optional, Tuple, Union -from black.mode import Mode +from black.mode import Mode, Preview from black.nodes import ( CLOSING_BRACKETS, STANDALONE_COMMENT, @@ -46,6 +46,7 @@ class ProtoComment: newlines: int # how many newlines before the comment consumed: int # how many characters of the original leaf's prefix did we consume form_feed: bool # is there a form feed before the comment + leading_whitespace: str # leading whitespace before the comment, if any def generate_comments(leaf: LN) -> Iterator[Leaf]: @@ -88,7 +89,9 @@ def list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]: form_feed = False for index, full_line in enumerate(re.split("\r?\n", prefix)): consumed += len(full_line) + 1 # adding the length of the split '\n' - line = full_line.lstrip() + match = re.match(r"^(\s*)(\S.*|)$", full_line) + assert match + whitespace, line = match.groups() if not line: nlines += 1 if "\f" in full_line: @@ -113,6 +116,7 @@ def list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]: newlines=nlines, consumed=consumed, form_feed=form_feed, + leading_whitespace=whitespace, ) ) form_feed = False @@ -230,7 +234,11 @@ def convert_one_fmt_off_pair( standalone_comment_prefix += fmt_off_prefix hidden_value = comment.value + "\n" + hidden_value if _contains_fmt_skip_comment(comment.value, mode): - hidden_value += " " + comment.value + hidden_value += ( + comment.leading_whitespace + if Preview.no_normalize_fmt_skip_whitespace in mode + else " " + ) + comment.value if hidden_value.endswith("\n"): # That happens when one of the `ignored_nodes` ended with a NEWLINE # leaf (possibly followed by a DEDENT). diff --git a/src/black/mode.py b/src/black/mode.py index 1b97f3508ee..a1519f17bcc 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -174,6 +174,7 @@ class Preview(Enum): string_processing = auto() hug_parens_with_braces_and_square_brackets = auto() unify_docstring_detection = auto() + no_normalize_fmt_skip_whitespace = auto() wrap_long_dict_values_in_parens = auto() multiline_string_handling = auto() diff --git a/tests/data/cases/fmtskip9.py b/tests/data/cases/fmtskip9.py new file mode 100644 index 00000000000..30085bdd973 --- /dev/null +++ b/tests/data/cases/fmtskip9.py @@ -0,0 +1,9 @@ +# flags: --preview +print () # fmt: skip +print () # fmt:skip + + +# output + +print () # fmt: skip +print () # fmt:skip From f7d552d9b7bd33f43edd0867757c27b1aa36c651 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 25 Jan 2024 13:11:26 -0800 Subject: [PATCH 247/279] Remove reference (#4169) This is out-of-date and just a chore. I don't think this is useful to contributors and Black doesn't even have a public API. --- docs/contributing/index.md | 5 - .../reference/reference_classes.rst | 234 ------------------ .../reference/reference_exceptions.rst | 18 -- .../reference/reference_functions.rst | 172 ------------- .../reference/reference_summary.rst | 19 -- 5 files changed, 448 deletions(-) delete mode 100644 docs/contributing/reference/reference_classes.rst delete mode 100644 docs/contributing/reference/reference_exceptions.rst delete mode 100644 docs/contributing/reference/reference_functions.rst delete mode 100644 docs/contributing/reference/reference_summary.rst diff --git a/docs/contributing/index.md b/docs/contributing/index.md index 3314c8eaa39..f15afa318d4 100644 --- a/docs/contributing/index.md +++ b/docs/contributing/index.md @@ -9,7 +9,6 @@ the_basics gauging_changes issue_triage release_process -reference/reference_summary ``` Welcome! Happy to see you willing to make the project better. Have you read the entire @@ -42,9 +41,5 @@ This section covers the following topics: - {doc}`the_basics` - {doc}`gauging_changes` - {doc}`release_process` -- {doc}`reference/reference_summary` For an overview on contributing to the _Black_, please checkout {doc}`the_basics`. - -If you need a reference of the functions, classes, etc. available to you while -developing _Black_, there's the {doc}`reference/reference_summary` docs. diff --git a/docs/contributing/reference/reference_classes.rst b/docs/contributing/reference/reference_classes.rst deleted file mode 100644 index dc615579e30..00000000000 --- a/docs/contributing/reference/reference_classes.rst +++ /dev/null @@ -1,234 +0,0 @@ -*Black* classes -=============== - -*Contents are subject to change.* - -Black Classes -~~~~~~~~~~~~~~ - -.. currentmodule:: black - -:class:`BracketTracker` ------------------------ - -.. autoclass:: black.brackets.BracketTracker - :members: - -:class:`Line` -------------- - -.. autoclass:: black.lines.Line - :members: - :special-members: __str__, __bool__ - -:class:`RHSResult` -------------------------- - -.. autoclass:: black.lines.RHSResult - :members: - -:class:`LinesBlock` -------------------------- - -.. autoclass:: black.lines.LinesBlock - :members: - -:class:`EmptyLineTracker` -------------------------- - -.. autoclass:: black.lines.EmptyLineTracker - :members: - -:class:`LineGenerator` ----------------------- - -.. autoclass:: black.linegen.LineGenerator - :show-inheritance: - :members: - -:class:`ProtoComment` ---------------------- - -.. autoclass:: black.comments.ProtoComment - :members: - -:class:`Mode` ---------------------- - -.. autoclass:: black.mode.Mode - :members: - -:class:`Report` ---------------- - -.. autoclass:: black.report.Report - :members: - :special-members: __str__ - -:class:`Ok` ---------------- - -.. autoclass:: black.rusty.Ok - :show-inheritance: - :members: - -:class:`Err` ---------------- - -.. autoclass:: black.rusty.Err - :show-inheritance: - :members: - -:class:`Visitor` ----------------- - -.. autoclass:: black.nodes.Visitor - :show-inheritance: - :members: - -:class:`StringTransformer` ----------------------------- - -.. autoclass:: black.trans.StringTransformer - :show-inheritance: - :members: - -:class:`CustomSplit` ----------------------------- - -.. autoclass:: black.trans.CustomSplit - :members: - -:class:`CustomSplitMapMixin` ------------------------------ - -.. autoclass:: black.trans.CustomSplitMapMixin - :show-inheritance: - :members: - -:class:`StringMerger` ----------------------- - -.. autoclass:: black.trans.StringMerger - :show-inheritance: - :members: - -:class:`StringParenStripper` ------------------------------ - -.. autoclass:: black.trans.StringParenStripper - :show-inheritance: - :members: - -:class:`BaseStringSplitter` ------------------------------ - -.. autoclass:: black.trans.BaseStringSplitter - :show-inheritance: - :members: - -:class:`StringSplitter` ------------------------------ - -.. autoclass:: black.trans.StringSplitter - :show-inheritance: - :members: - -:class:`StringParenWrapper` ------------------------------ - -.. autoclass:: black.trans.StringParenWrapper - :show-inheritance: - :members: - -:class:`StringParser` ------------------------------ - -.. autoclass:: black.trans.StringParser - :members: - -:class:`DebugVisitor` ------------------------- - -.. autoclass:: black.debug.DebugVisitor - :show-inheritance: - :members: - -:class:`Replacement` ------------------------- - -.. autoclass:: black.handle_ipynb_magics.Replacement - :members: - -:class:`CellMagic` ------------------------- - -.. autoclass:: black.handle_ipynb_magics.CellMagic - :members: - -:class:`CellMagicFinder` ------------------------- - -.. autoclass:: black.handle_ipynb_magics.CellMagicFinder - :show-inheritance: - :members: - -:class:`OffsetAndMagic` ------------------------- - -.. autoclass:: black.handle_ipynb_magics.OffsetAndMagic - :members: - -:class:`MagicFinder` ------------------------- - -.. autoclass:: black.handle_ipynb_magics.MagicFinder - :show-inheritance: - :members: - -:class:`Cache` ------------------------- - -.. autoclass:: black.cache.Cache - :show-inheritance: - :members: - -Enum Classes -~~~~~~~~~~~~~ - -Classes inherited from Python `Enum `_ class. - -:class:`Changed` ----------------- - -.. autoclass:: black.report.Changed - :show-inheritance: - :members: - -:class:`WriteBack` ------------------- - -.. autoclass:: black.WriteBack - :show-inheritance: - :members: - -:class:`TargetVersion` ----------------------- - -.. autoclass:: black.mode.TargetVersion - :show-inheritance: - :members: - -:class:`Feature` ------------------- - -.. autoclass:: black.mode.Feature - :show-inheritance: - :members: - -:class:`Preview` ------------------- - -.. autoclass:: black.mode.Preview - :show-inheritance: - :members: diff --git a/docs/contributing/reference/reference_exceptions.rst b/docs/contributing/reference/reference_exceptions.rst deleted file mode 100644 index ab46ebdb628..00000000000 --- a/docs/contributing/reference/reference_exceptions.rst +++ /dev/null @@ -1,18 +0,0 @@ -*Black* exceptions -================== - -*Contents are subject to change.* - -.. currentmodule:: black - -.. autoexception:: black.trans.CannotTransform - -.. autoexception:: black.linegen.CannotSplit - -.. autoexception:: black.brackets.BracketMatchError - -.. autoexception:: black.report.NothingChanged - -.. autoexception:: black.parsing.InvalidInput - -.. autoexception:: black.mode.Deprecated diff --git a/docs/contributing/reference/reference_functions.rst b/docs/contributing/reference/reference_functions.rst deleted file mode 100644 index ebadf6975a7..00000000000 --- a/docs/contributing/reference/reference_functions.rst +++ /dev/null @@ -1,172 +0,0 @@ -*Black* functions -================= - -*Contents are subject to change.* - -.. currentmodule:: black - -Assertions and checks ---------------------- - -.. autofunction:: black.assert_equivalent - -.. autofunction:: black.assert_stable - -.. autofunction:: black.lines.can_be_split - -.. autofunction:: black.lines.can_omit_invisible_parens - -.. autofunction:: black.nodes.is_empty_tuple - -.. autofunction:: black.nodes.is_import - -.. autofunction:: black.lines.is_line_short_enough - -.. autofunction:: black.nodes.is_multiline_string - -.. autofunction:: black.nodes.is_one_tuple - -.. autofunction:: black.brackets.is_split_after_delimiter - -.. autofunction:: black.brackets.is_split_before_delimiter - -.. autofunction:: black.nodes.is_stub_body - -.. autofunction:: black.nodes.is_stub_suite - -.. autofunction:: black.nodes.is_vararg - -.. autofunction:: black.nodes.is_yield - - -Formatting ----------- - -.. autofunction:: black.format_file_contents - -.. autofunction:: black.format_file_in_place - -.. autofunction:: black.format_stdin_to_stdout - -.. autofunction:: black.format_str - -.. autofunction:: black.reformat_one - -.. autofunction:: black.concurrency.schedule_formatting - -File operations ---------------- - -.. autofunction:: black.dump_to_file - -.. autofunction:: black.find_project_root - -.. autofunction:: black.gen_python_files - -.. autofunction:: black.read_pyproject_toml - -Parsing -------- - -.. autofunction:: black.decode_bytes - -.. autofunction:: black.parsing.lib2to3_parse - -.. autofunction:: black.parsing.lib2to3_unparse - -Split functions ---------------- - -.. autofunction:: black.linegen.bracket_split_build_line - -.. autofunction:: black.linegen.bracket_split_succeeded_or_raise - -.. autofunction:: black.linegen.delimiter_split - -.. autofunction:: black.linegen.left_hand_split - -.. autofunction:: black.linegen.right_hand_split - -.. autofunction:: black.linegen.standalone_comment_split - -.. autofunction:: black.linegen.transform_line - -Caching -------- - -.. autofunction:: black.cache.get_cache_dir - -.. autofunction:: black.cache.get_cache_file - -Utilities ---------- - -.. py:function:: black.debug.DebugVisitor.show(code: str) -> None - - Pretty-print the lib2to3 AST of a given string of `code`. - -.. autofunction:: black.concurrency.cancel - -.. autofunction:: black.nodes.child_towards - -.. autofunction:: black.nodes.container_of - -.. autofunction:: black.comments.convert_one_fmt_off_pair - -.. autofunction:: black.diff - -.. autofunction:: black.linegen.dont_increase_indentation - -.. autofunction:: black.numerics.format_float_or_int_string - -.. autofunction:: black.nodes.ensure_visible - -.. autofunction:: black.lines.enumerate_reversed - -.. autofunction:: black.comments.generate_comments - -.. autofunction:: black.comments.generate_ignored_nodes - -.. autofunction:: black.comments.is_fmt_on - -.. autofunction:: black.comments.children_contains_fmt_on - -.. autofunction:: black.nodes.first_leaf_of - -.. autofunction:: black.linegen.generate_trailers_to_omit - -.. autofunction:: black.get_future_imports - -.. autofunction:: black.comments.list_comments - -.. autofunction:: black.comments.make_comment - -.. autofunction:: black.linegen.maybe_make_parens_invisible_in_atom - -.. autofunction:: black.brackets.max_delimiter_priority_in_atom - -.. autofunction:: black.normalize_fmt_off - -.. autofunction:: black.numerics.normalize_numeric_literal - -.. autofunction:: black.comments.normalize_trailing_prefix - -.. autofunction:: black.strings.normalize_string_prefix - -.. autofunction:: black.strings.normalize_string_quotes - -.. autofunction:: black.linegen.normalize_invisible_parens - -.. autofunction:: black.nodes.preceding_leaf - -.. autofunction:: black.re_compile_maybe_verbose - -.. autofunction:: black.linegen.should_split_line - -.. autofunction:: black.concurrency.shutdown - -.. autofunction:: black.strings.sub_twice - -.. autofunction:: black.nodes.whitespace - -.. autofunction:: black.nodes.make_simple_prefix diff --git a/docs/contributing/reference/reference_summary.rst b/docs/contributing/reference/reference_summary.rst deleted file mode 100644 index c6163d897b6..00000000000 --- a/docs/contributing/reference/reference_summary.rst +++ /dev/null @@ -1,19 +0,0 @@ -Developer reference -=================== - -.. note:: - - As of June 2023, the documentation of *Black classes* and *Black exceptions* - has been updated to the latest available version. - - The documentation of *Black functions* is quite outdated and has been neglected. Many - functions worthy of inclusion aren't documented. Contributions are appreciated! - -*Contents are subject to change.* - -.. toctree:: - :maxdepth: 2 - - reference_classes - reference_functions - reference_exceptions From 17f7f297efd29a9b4187af8420e88ca156c1d221 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 25 Jan 2024 13:41:45 -0800 Subject: [PATCH 248/279] Simplify code in lines.py (#4167) This has been getting a little messy. These changes neaten things up, we don't have to keep guarding against `self.previous_line is not None`, we make it clearer what logic has side effects, we reduce the amount of code that tricky `before` could touch, etc --- src/black/lines.py | 62 +++++++++++++++------------------------------- 1 file changed, 20 insertions(+), 42 deletions(-) diff --git a/src/black/lines.py b/src/black/lines.py index 29f87137614..72634fd36d2 100644 --- a/src/black/lines.py +++ b/src/black/lines.py @@ -565,14 +565,9 @@ def maybe_empty_lines(self, current_line: Line) -> LinesBlock: ) before, after = self._maybe_empty_lines(current_line) previous_after = self.previous_block.after if self.previous_block else 0 - before = ( - # Black should not insert empty lines at the beginning - # of the file - 0 - if self.previous_line is None - else before - previous_after - ) + before = max(0, before - previous_after) if ( + # Always have one empty line after a module docstring self.previous_block and self.previous_block.previous_block is None and len(self.previous_block.original_line.leaves) == 1 @@ -607,10 +602,11 @@ def maybe_empty_lines(self, current_line: Line) -> LinesBlock: self.previous_block = block return block - def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: + def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: # noqa: C901 max_allowed = 1 if current_line.depth == 0: max_allowed = 1 if self.mode.is_pyi else 2 + if current_line.leaves: # Consume the first leaf's extra newlines. first_leaf = current_line.leaves[0] @@ -623,9 +619,22 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: user_had_newline = bool(before) depth = current_line.depth + # Mutate self.previous_defs, remainder of this function should be pure previous_def = None while self.previous_defs and self.previous_defs[-1].depth >= depth: previous_def = self.previous_defs.pop() + if current_line.is_def or current_line.is_class: + self.previous_defs.append(current_line) + + if self.previous_line is None: + # Don't insert empty lines before the first line in the file. + return 0, 0 + + if current_line.is_docstring: + if self.previous_line.is_class: + return 0, 1 + if self.previous_line.opens_block and self.previous_line.is_def: + return 0, 0 if previous_def is not None: assert self.previous_line is not None @@ -668,49 +677,24 @@ def _maybe_empty_lines(self, current_line: Line) -> Tuple[int, int]: ) if ( - self.previous_line - and self.previous_line.is_import + self.previous_line.is_import and not current_line.is_import and not current_line.is_fmt_pass_converted(first_leaf_matches=is_import) and depth == self.previous_line.depth ): return (before or 1), 0 - if ( - self.previous_line - and self.previous_line.is_class - and current_line.is_docstring - ): - return 0, 1 - - # In preview mode, always allow blank lines, except right before a function - # docstring - is_empty_first_line_ok = not current_line.is_docstring or ( - self.previous_line and not self.previous_line.is_def - ) - - if ( - self.previous_line - and self.previous_line.opens_block - and not is_empty_first_line_ok - ): - return 0, 0 return before, 0 def _maybe_empty_lines_for_class_or_def( # noqa: C901 self, current_line: Line, before: int, user_had_newline: bool ) -> Tuple[int, int]: - if not current_line.is_decorator: - self.previous_defs.append(current_line) - if self.previous_line is None: - # Don't insert empty lines before the first line in the file. - return 0, 0 + assert self.previous_line is not None if self.previous_line.is_decorator: if self.mode.is_pyi and current_line.is_stub_class: # Insert an empty line after a decorated stub class return 0, 1 - return 0, 0 if self.previous_line.depth < current_line.depth and ( @@ -718,8 +702,7 @@ def _maybe_empty_lines_for_class_or_def( # noqa: C901 ): if self.mode.is_pyi: return 0, 0 - else: - return 1 if user_had_newline else 0, 0 + return 1 if user_had_newline else 0, 0 comment_to_add_newlines: Optional[LinesBlock] = None if ( @@ -750,9 +733,6 @@ def _maybe_empty_lines_for_class_or_def( # noqa: C901 newlines = 0 else: newlines = 1 - # Remove case `self.previous_line.depth > current_line.depth` below when - # this becomes stable. - # # Don't inspect the previous line if it's part of the body of the previous # statement in the same level, we always want a blank line if there's # something with a body preceding. @@ -769,8 +749,6 @@ def _maybe_empty_lines_for_class_or_def( # noqa: C901 # Blank line between a block of functions (maybe with preceding # decorators) and a block of non-functions newlines = 1 - elif self.previous_line.depth > current_line.depth: - newlines = 1 else: newlines = 0 else: From 7d789469ed947022f183962b823f5862511272ac Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 25 Jan 2024 16:15:18 -0800 Subject: [PATCH 249/279] Describe 2024 module docstring more accurately (#4168) --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 2fe14cd5246..95ef0095102 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,7 +17,7 @@ changes: - Hex codes in Unicode escape sequences are now standardized to lowercase (#2916) - Allow empty first lines at the beginning of most blocks (#3967, #4061) - Add parentheses around long type annotations (#3899) -- Standardize on a single newline after module docstrings (#3932) +- Enforce newline after module docstrings (#3932, #4028) - Fix incorrect magic trailing comma handling in return types (#3916) - Remove blank lines before class docstrings (#3692) - Wrap multiple context managers in parentheses if combined in a single `with` statement From bccec8adfbed2bbc24c0859e8758d5e7809d42b7 Mon Sep 17 00:00:00 2001 From: Daniel Krzeminski Date: Thu, 25 Jan 2024 18:41:37 -0600 Subject: [PATCH 250/279] Show warning on invalid toml configuration (#4165) Co-authored-by: Jelle Zijlstra --- CHANGES.md | 1 + src/black/__init__.py | 17 +++++++++++++++++ tests/data/incorrect_spelling.toml | 5 +++++ tests/test_black.py | 17 +++++++++++++++++ 4 files changed, 40 insertions(+) create mode 100644 tests/data/incorrect_spelling.toml diff --git a/CHANGES.md b/CHANGES.md index 95ef0095102..45cb967e74a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -62,6 +62,7 @@ release: +- Print warning when toml config contains an invalid key (#4165) - Fix symlink handling, properly catch and ignore symlinks that point outside of root (#4161) - Fix cache mtime logic that resulted in false positive cache hits (#4128) diff --git a/src/black/__init__.py b/src/black/__init__.py index e3cbaab5f1d..961ed9479a8 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -142,6 +142,7 @@ def read_pyproject_toml( if not config: return None else: + spellcheck_pyproject_toml_keys(ctx, list(config), value) # Sanitize the values to be Click friendly. For more information please see: # https://github.com/psf/black/issues/1458 # https://github.com/pallets/click/issues/1567 @@ -181,6 +182,22 @@ def read_pyproject_toml( return value +def spellcheck_pyproject_toml_keys( + ctx: click.Context, config_keys: List[str], config_file_path: str +) -> None: + invalid_keys: List[str] = [] + available_config_options = {param.name for param in ctx.command.params} + for key in config_keys: + if key not in available_config_options: + invalid_keys.append(key) + if invalid_keys: + keys_str = ", ".join(map(repr, invalid_keys)) + out( + f"Invalid config keys detected: {keys_str} (in {config_file_path})", + fg="red", + ) + + def target_version_option_callback( c: click.Context, p: Union[click.Option, click.Parameter], v: Tuple[str, ...] ) -> List[TargetVersion]: diff --git a/tests/data/incorrect_spelling.toml b/tests/data/incorrect_spelling.toml new file mode 100644 index 00000000000..560c9e27be2 --- /dev/null +++ b/tests/data/incorrect_spelling.toml @@ -0,0 +1,5 @@ +[tool.black] +ine_length = 50 +target-ersion = ['py37'] +exclude='\.pyi?$' +include='\.py?$' \ No newline at end of file diff --git a/tests/test_black.py b/tests/test_black.py index 2b5fab5d28d..a979a95b674 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -106,6 +106,7 @@ class FakeContext(click.Context): def __init__(self) -> None: self.default_map: Dict[str, Any] = {} self.params: Dict[str, Any] = {} + self.command: click.Command = black.main # Dummy root, since most of the tests don't care about it self.obj: Dict[str, Any] = {"root": PROJECT_ROOT} @@ -1538,6 +1539,22 @@ def test_parse_pyproject_toml(self) -> None: self.assertEqual(config["exclude"], r"\.pyi?$") self.assertEqual(config["include"], r"\.py?$") + def test_spellcheck_pyproject_toml(self) -> None: + test_toml_file = THIS_DIR / "data" / "incorrect_spelling.toml" + result = BlackRunner().invoke( + black.main, + [ + "--code=print('hello world')", + "--verbose", + f"--config={str(test_toml_file)}", + ], + ) + + assert ( + r"Invalid config keys detected: 'ine_length', 'target_ersion' (in" + rf" {test_toml_file})" in result.stderr + ) + def test_parse_pyproject_toml_project_metadata(self) -> None: for test_toml, expected in [ ("only_black_pyproject.toml", ["py310"]), From 4f47cac1925a2232892ceae438e2c62f81517714 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 25 Jan 2024 17:00:47 -0800 Subject: [PATCH 251/279] Add --unstable flag (#4096) --- CHANGES.md | 17 ++ docs/faq.md | 7 +- docs/the_black_code_style/current_style.md | 6 + docs/the_black_code_style/future_style.md | 217 +++++++++--------- docs/the_black_code_style/index.md | 8 +- .../black_as_a_server.md | 6 + docs/usage_and_configuration/the_basics.md | 31 ++- pyproject.toml | 9 +- src/black/__init__.py | 49 +++- src/black/mode.py | 39 ++-- src/blackd/__init__.py | 104 +++++---- tests/data/cases/preview_cantfit.py | 14 -- tests/data/cases/preview_cantfit_string.py | 18 ++ tests/data/cases/preview_comments7.py | 2 +- tests/data/cases/preview_long_dict_values.py | 2 +- tests/data/cases/preview_long_strings.py | 2 +- .../preview_long_strings__east_asian_width.py | 2 +- .../cases/preview_long_strings__edge_case.py | 2 +- .../cases/preview_long_strings__regression.py | 2 +- tests/data/cases/preview_multiline_strings.py | 2 +- ...eview_return_annotation_brackets_string.py | 2 +- tests/test_black.py | 63 ++--- tests/util.py | 16 +- 23 files changed, 368 insertions(+), 252 deletions(-) create mode 100644 tests/data/cases/preview_cantfit_string.py diff --git a/CHANGES.md b/CHANGES.md index 45cb967e74a..0496603e2c0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -34,6 +34,14 @@ changes: - Fix incorrect formatting of certain async statements (#3609) - Allow combining `# fmt: skip` with other comments (#3959) +There are already a few improvements in the `--preview` style, which are slated for the +2025 stable style. Try them out and +[share your feedback](https://github.com/psf/black/issues). In the past, the preview +style has included some features that we were not able to stabilize. This year, we're +adding a separate `--unstable` style for features with known problems. Now, the +`--preview` style only includes features that we actually expect to make it into next +year's stable style. + ### Stable style @@ -53,6 +61,12 @@ release: +- Add `--unstable` style, covering preview features that have known problems that would + block them from going into the stable style. Also add the `--enable-unstable-feature` + flag; for example, use + `--enable-unstable-feature hug_parens_with_braces_and_square_brackets` to apply this + preview style throughout 2024, even if a later Black release downgrades the feature to + unstable (#4096) - Format module docstrings the same as class and function docstrings (#4095) - Fix crash when using a walrus in a dictionary (#4155) - Fix unnecessary parentheses when wrapping long dicts (#4135) @@ -66,6 +80,9 @@ release: - Fix symlink handling, properly catch and ignore symlinks that point outside of root (#4161) - Fix cache mtime logic that resulted in false positive cache hits (#4128) +- Remove the long-deprecated `--experimental-string-processing` flag. This feature can + currently be enabled with `--preview --enable-unstable-feature string_processing`. + (#4096) ### Packaging diff --git a/docs/faq.md b/docs/faq.md index c62e1b504b5..124a096efac 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -41,9 +41,10 @@ other tools, such as `# noqa`, may be moved by _Black_. See below for more detai Stable. _Black_ aims to enforce one style and one style only, with some room for pragmatism. See [The Black Code Style](the_black_code_style/index.md) for more details. -Starting in 2022, the formatting output will be stable for the releases made in the same -year (other than unintentional bugs). It is possible to opt-in to the latest formatting -styles, using the `--preview` flag. +Starting in 2022, the formatting output is stable for the releases made in the same year +(other than unintentional bugs). At the beginning of every year, the first release will +make changes to the stable style. It is possible to opt in to the latest formatting +styles using the `--preview` flag. ## Why is my file not formatted? diff --git a/docs/the_black_code_style/current_style.md b/docs/the_black_code_style/current_style.md index 00bd81416dc..ca5d1d4a701 100644 --- a/docs/the_black_code_style/current_style.md +++ b/docs/the_black_code_style/current_style.md @@ -449,6 +449,12 @@ file that are not enforced yet but might be in a future version of the formatter _Black_ will normalize line endings (`\n` or `\r\n`) based on the first line ending of the file. +### Form feed characters + +_Black_ will retain form feed characters on otherwise empty lines at the module level. +Only one form feed is retained for a group of consecutive empty lines. Where there are +two empty lines in a row, the form feed is placed on the second line. + ## Pragmatism Early versions of _Black_ used to be absolutist in some respects. They took after its diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index f55ea5f60a9..1cdd25fdb7c 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -1,54 +1,5 @@ # The (future of the) Black code style -```{warning} -Changes to this document often aren't tied and don't relate to releases of -_Black_. It's recommended that you read the latest version available. -``` - -## Using backslashes for with statements - -[Backslashes are bad and should be never be used](labels/why-no-backslashes) however -there is one exception: `with` statements using multiple context managers. Before Python -3.9 Python's grammar does not allow organizing parentheses around the series of context -managers. - -We don't want formatting like: - -```py3 -with make_context_manager1() as cm1, make_context_manager2() as cm2, make_context_manager3() as cm3, make_context_manager4() as cm4: - ... # nothing to split on - line too long -``` - -So _Black_ will, when we implement this, format it like this: - -```py3 -with \ - make_context_manager1() as cm1, \ - make_context_manager2() as cm2, \ - make_context_manager3() as cm3, \ - make_context_manager4() as cm4 \ -: - ... # backslashes and an ugly stranded colon -``` - -Although when the target version is Python 3.9 or higher, _Black_ uses parentheses -instead in `--preview` mode (see below) since they're allowed in Python 3.9 and higher. - -An alternative to consider if the backslashes in the above formatting are undesirable is -to use {external:py:obj}`contextlib.ExitStack` to combine context managers in the -following way: - -```python -with contextlib.ExitStack() as exit_stack: - cm1 = exit_stack.enter_context(make_context_manager1()) - cm2 = exit_stack.enter_context(make_context_manager2()) - cm3 = exit_stack.enter_context(make_context_manager3()) - cm4 = exit_stack.enter_context(make_context_manager4()) - ... -``` - -(labels/preview-style)= - ## Preview style Experimental, potentially disruptive style changes are gathered under the `--preview` @@ -56,62 +7,38 @@ CLI flag. At the end of each year, these changes may be adopted into the default as described in [The Black Code Style](index.md). Because the functionality is experimental, feedback and issue reports are highly encouraged! -### Improved string processing - -_Black_ will split long string literals and merge short ones. Parentheses are used where -appropriate. When split, parts of f-strings that don't need formatting are converted to -plain strings. User-made splits are respected when they do not exceed the line length -limit. Line continuation backslashes are converted into parenthesized strings. -Unnecessary parentheses are stripped. The stability and status of this feature is -tracked in [this issue](https://github.com/psf/black/issues/2188). - -### Improved line breaks - -For assignment expressions, _Black_ now prefers to split and wrap the right side of the -assignment instead of left side. For example: - -```python -some_dict[ - "with_a_long_key" -] = some_looooooooong_module.some_looooooooooooooong_function_name( - first_argument, second_argument, third_argument -) -``` - -will be changed to: +In the past, the preview style included some features with known bugs, so that we were +unable to move these features to the stable style. Therefore, such features are now +moved to the `--unstable` style. All features in the `--preview` style are expected to +make it to next year's stable style; features in the `--unstable` style will be +stabilized only if issues with them are fixed. If bugs are discovered in a `--preview` +feature, it is demoted to the `--unstable` style. To avoid thrash when a feature is +demoted from the `--preview` to the `--unstable` style, users can use the +`--enable-unstable-feature` flag to enable specific unstable features. -```python -some_dict["with_a_long_key"] = ( - some_looooooooong_module.some_looooooooooooooong_function_name( - first_argument, second_argument, third_argument - ) -) -``` +Currently, the following features are included in the preview style: -### Improved parentheses management +- `hex_codes_in_unicode_sequences`: normalize casing of Unicode escape characters in + strings +- `unify_docstring_detection`: fix inconsistencies in whether certain strings are + detected as docstrings +- `hug_parens_with_braces_and_square_brackets`: more compact formatting of nested + brackets ([see below](labels/hug-parens)) +- `no_normalize_fmt_skip_whitespace`: whitespace before `# fmt: skip` comments is no + longer normalized -For dict literals with long values, they are now wrapped in parentheses. Unnecessary -parentheses are now removed. For example: +(labels/unstable-features)= -```python -my_dict = { - "a key in my dict": a_very_long_variable - * and_a_very_long_function_call() - / 100000.0, - "another key": (short_value), -} -``` +The unstable style additionally includes the following features: -will be changed to: +- `string_processing`: split long string literals and related changes + ([see below](labels/string-processing)) +- `wrap_long_dict_values_in_parens`: add parentheses to long values in dictionaries + ([see below](labels/wrap-long-dict-values)) +- `multiline_string_handling`: more compact formatting of expressions involving + multiline strings ([see below](labels/multiline-string-handling)) -```python -my_dict = { - "a key in my dict": ( - a_very_long_variable * and_a_very_long_function_call() / 100000.0 - ), - "another key": short_value, -} -``` +(labels/hug-parens)= ### Improved multiline dictionary and list indentation for sole function parameter @@ -185,6 +112,46 @@ foo( ) ``` +(labels/string-processing)= + +### Improved string processing + +_Black_ will split long string literals and merge short ones. Parentheses are used where +appropriate. When split, parts of f-strings that don't need formatting are converted to +plain strings. User-made splits are respected when they do not exceed the line length +limit. Line continuation backslashes are converted into parenthesized strings. +Unnecessary parentheses are stripped. The stability and status of this feature is +tracked in [this issue](https://github.com/psf/black/issues/2188). + +(labels/wrap-long-dict-values)= + +### Improved parentheses management in dicts + +For dict literals with long values, they are now wrapped in parentheses. Unnecessary +parentheses are now removed. For example: + +```python +my_dict = { + "a key in my dict": a_very_long_variable + * and_a_very_long_function_call() + / 100000.0, + "another key": (short_value), +} +``` + +will be changed to: + +```python +my_dict = { + "a key in my dict": ( + a_very_long_variable * and_a_very_long_function_call() / 100000.0 + ), + "another key": short_value, +} +``` + +(labels/multiline-string-handling)= + ### Improved multiline string handling _Black_ is smarter when formatting multiline strings, especially in function arguments, @@ -297,13 +264,51 @@ s = ( # Top comment ) ``` -======= +## Potential future changes + +This section lists changes that we may want to make in the future, but that aren't +implemented yet. + +### Using backslashes for with statements + +[Backslashes are bad and should be never be used](labels/why-no-backslashes) however +there is one exception: `with` statements using multiple context managers. Before Python +3.9 Python's grammar does not allow organizing parentheses around the series of context +managers. + +We don't want formatting like: + +```py3 +with make_context_manager1() as cm1, make_context_manager2() as cm2, make_context_manager3() as cm3, make_context_manager4() as cm4: + ... # nothing to split on - line too long +``` + +So _Black_ will, when we implement this, format it like this: + +```py3 +with \ + make_context_manager1() as cm1, \ + make_context_manager2() as cm2, \ + make_context_manager3() as cm3, \ + make_context_manager4() as cm4 \ +: + ... # backslashes and an ugly stranded colon +``` + +Although when the target version is Python 3.9 or higher, _Black_ uses parentheses +instead in `--preview` mode (see below) since they're allowed in Python 3.9 and higher. -### Form feed characters +An alternative to consider if the backslashes in the above formatting are undesirable is +to use {external:py:obj}`contextlib.ExitStack` to combine context managers in the +following way: -_Black_ will now retain form feed characters on otherwise empty lines at the module -level. Only one form feed is retained for a group of consecutive empty lines. Where -there are two empty lines in a row, the form feed will be placed on the second line. +```python +with contextlib.ExitStack() as exit_stack: + cm1 = exit_stack.enter_context(make_context_manager1()) + cm2 = exit_stack.enter_context(make_context_manager2()) + cm3 = exit_stack.enter_context(make_context_manager3()) + cm4 = exit_stack.enter_context(make_context_manager4()) + ... +``` -_Black_ already retained form feed literals inside a comment or inside a string. This -remains the case. +(labels/preview-style)= diff --git a/docs/the_black_code_style/index.md b/docs/the_black_code_style/index.md index 1719347eec8..58f28673022 100644 --- a/docs/the_black_code_style/index.md +++ b/docs/the_black_code_style/index.md @@ -42,9 +42,11 @@ _Black_: enabled by newer Python language syntax as well as due to improvements in the formatting logic. -- The `--preview` flag is exempt from this policy. There are no guarantees around the - stability of the output with that flag passed into _Black_. This flag is intended for - allowing experimentation with the proposed changes to the _Black_ code style. +- The `--preview` and `--unstable` flags are exempt from this policy. There are no + guarantees around the stability of the output with these flags passed into _Black_. + They are intended for allowing experimentation with proposed changes to the _Black_ + code style. The `--preview` style at the end of a year should closely match the stable + style for the next year, but we may always make changes. Documentation for both the current and future styles can be found: diff --git a/docs/usage_and_configuration/black_as_a_server.md b/docs/usage_and_configuration/black_as_a_server.md index f24fb34d915..0a7edb57fd7 100644 --- a/docs/usage_and_configuration/black_as_a_server.md +++ b/docs/usage_and_configuration/black_as_a_server.md @@ -62,6 +62,12 @@ The headers controlling how source code is formatted are: - `X-Preview`: corresponds to the `--preview` command line flag. If present and its value is not an empty string, experimental and potentially disruptive style changes will be used. +- `X-Unstable`: corresponds to the `--unstable` command line flag. If present and its + value is not an empty string, experimental style changes that are known to be buggy + will be used. +- `X-Enable-Unstable-Feature`: corresponds to the `--enable-unstable-feature` flag. The + contents of the flag must be a comma-separated list of unstable features to be + enabled. Example: `X-Enable-Unstable-Feature: feature1, feature2`. - `X-Fast-Or-Safe`: if set to `fast`, `blackd` will act as _Black_ does when passed the `--fast` command line flag. - `X-Python-Variant`: if set to `pyi`, `blackd` will act as _Black_ does when passed the diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index b541f07907c..a42e093155b 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -144,9 +144,34 @@ magic trailing comma is ignored. #### `--preview` -Enable potentially disruptive style changes that may be added to Black's main -functionality in the next major release. Read more about -[our preview style](labels/preview-style). +Enable potentially disruptive style changes that we expect to add to Black's main +functionality in the next major release. Use this if you want a taste of what next +year's style will look like. + +Read more about [our preview style](labels/preview-style). + +There is no guarantee on the code style produced by this flag across releases. + +#### `--unstable` + +Enable all style changes in `--preview`, plus additional changes that we would like to +make eventually, but that have known issues that need to be fixed before they can move +back to the `--preview` style. Use this if you want to experiment with these changes and +help fix issues with them. + +There is no guarantee on the code style produced by this flag across releases. + +#### `--enable-unstable-feature` + +Enable specific features from the `--unstable` style. See +[the preview style documentation](labels/unstable-features) for the list of supported +features. This flag can only be used when `--preview` is enabled. Users are encouraged +to use this flag if they use `--preview` style and a feature that affects their code is +moved from the `--preview` to the `--unstable` style, but they want to avoid the thrash +from undoing this change. + +There are no guarantees on the behavior of these features, or even their existence, +across releases. (labels/exit-code)= diff --git a/pyproject.toml b/pyproject.toml index 24b9c07674d..fa3654b8d67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,10 +16,11 @@ extend-exclude = ''' | profiling )/ ''' -# We use preview style for formatting Black itself. If you -# want stable formatting across releases, you should keep -# this off. -preview = true +# We use the unstable style for formatting Black itself. If you +# want bug-free formatting, you should keep this off. If you want +# stable formatting across releases, you should also keep `preview = true` +# (which is implied by this flag) off. +unstable = true # Build system information and other project-specific configuration below. # NOTE: You don't need this in your own Black configuration. diff --git a/src/black/__init__.py b/src/black/__init__.py index 961ed9479a8..ebc7ac8eda5 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -68,7 +68,7 @@ from black.lines import EmptyLineTracker, LinesBlock from black.mode import FUTURE_FLAG_TO_FEATURE, VERSION_TO_FEATURES, Feature from black.mode import Mode as Mode # re-exported -from black.mode import TargetVersion, supports_feature +from black.mode import Preview, TargetVersion, supports_feature from black.nodes import ( STARS, is_number_token, @@ -209,6 +209,13 @@ def target_version_option_callback( return [TargetVersion[val.upper()] for val in v] +def enable_unstable_feature_callback( + c: click.Context, p: Union[click.Option, click.Parameter], v: Tuple[str, ...] +) -> List[Preview]: + """Compute the features from an --enable-unstable-feature flag.""" + return [Preview[val] for val in v] + + def re_compile_maybe_verbose(regex: str) -> Pattern[str]: """Compile a regular expression string in `regex`. @@ -303,12 +310,6 @@ def validate_regex( is_flag=True, help="Don't use trailing commas as a reason to split lines.", ) -@click.option( - "--experimental-string-processing", - is_flag=True, - hidden=True, - help="(DEPRECATED and now included in --preview) Normalize string literals.", -) @click.option( "--preview", is_flag=True, @@ -317,6 +318,26 @@ def validate_regex( " functionality in the next major release." ), ) +@click.option( + "--unstable", + is_flag=True, + help=( + "Enable potentially disruptive style changes that have known bugs or are not" + " currently expected to make it into the stable style Black's next major" + " release. Implies --preview." + ), +) +@click.option( + "--enable-unstable-feature", + type=click.Choice([v.name for v in Preview]), + callback=enable_unstable_feature_callback, + multiple=True, + help=( + "Enable specific features included in the `--unstable` style. Requires" + " `--preview`. No compatibility guarantees are provided on the behavior" + " or existence of any unstable features." + ), +) @click.option( "--check", is_flag=True, @@ -507,8 +528,9 @@ def main( # noqa: C901 skip_source_first_line: bool, skip_string_normalization: bool, skip_magic_trailing_comma: bool, - experimental_string_processing: bool, preview: bool, + unstable: bool, + enable_unstable_feature: List[Preview], quiet: bool, verbose: bool, required_version: Optional[str], @@ -534,6 +556,14 @@ def main( # noqa: C901 out(main.get_usage(ctx) + "\n\nOne of 'SRC' or 'code' is required.") ctx.exit(1) + # It doesn't do anything if --unstable is also passed, so just allow it. + if enable_unstable_feature and not (preview or unstable): + out( + main.get_usage(ctx) + + "\n\n'--enable-unstable-feature' requires '--preview'." + ) + ctx.exit(1) + root, method = ( find_project_root(src, stdin_filename) if code is None else (None, None) ) @@ -595,9 +625,10 @@ def main( # noqa: C901 skip_source_first_line=skip_source_first_line, string_normalization=not skip_string_normalization, magic_trailing_comma=not skip_magic_trailing_comma, - experimental_string_processing=experimental_string_processing, preview=preview, + unstable=unstable, python_cell_magics=set(python_cell_magics), + enabled_features=set(enable_unstable_feature), ) lines: List[Tuple[int, int]] = [] diff --git a/src/black/mode.py b/src/black/mode.py index a1519f17bcc..68919fb4901 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -9,7 +9,6 @@ from hashlib import sha256 from operator import attrgetter from typing import Dict, Final, Set -from warnings import warn from black.const import DEFAULT_LINE_LENGTH @@ -179,6 +178,16 @@ class Preview(Enum): multiline_string_handling = auto() +UNSTABLE_FEATURES: Set[Preview] = { + # Many issues, see summary in https://github.com/psf/black/issues/4042 + Preview.string_processing, + # See issues #3452 and #4158 + Preview.wrap_long_dict_values_in_parens, + # See issue #4159 + Preview.multiline_string_handling, +} + + class Deprecated(UserWarning): """Visible deprecation warning.""" @@ -192,28 +201,24 @@ class Mode: is_ipynb: bool = False skip_source_first_line: bool = False magic_trailing_comma: bool = True - experimental_string_processing: bool = False python_cell_magics: Set[str] = field(default_factory=set) preview: bool = False - - def __post_init__(self) -> None: - if self.experimental_string_processing: - warn( - "`experimental string processing` has been included in `preview`" - " and deprecated. Use `preview` instead.", - Deprecated, - ) + unstable: bool = False + enabled_features: Set[Preview] = field(default_factory=set) def __contains__(self, feature: Preview) -> bool: """ Provide `Preview.FEATURE in Mode` syntax that mirrors the ``preview`` flag. - The argument is not checked and features are not differentiated. - They only exist to make development easier by clarifying intent. + In unstable mode, all features are enabled. In preview mode, all features + except those in UNSTABLE_FEATURES are enabled. Any features in + `self.enabled_features` are also enabled. """ - if feature is Preview.string_processing: - return self.preview or self.experimental_string_processing - return self.preview + if self.unstable: + return True + if feature in self.enabled_features: + return True + return self.preview and feature not in UNSTABLE_FEATURES def get_cache_key(self) -> str: if self.target_versions: @@ -231,7 +236,9 @@ def get_cache_key(self) -> str: str(int(self.is_ipynb)), str(int(self.skip_source_first_line)), str(int(self.magic_trailing_comma)), - str(int(self.experimental_string_processing)), + sha256( + (",".join(sorted(f.name for f in self.enabled_features))).encode() + ).hexdigest(), str(int(self.preview)), sha256((",".join(sorted(self.python_cell_magics))).encode()).hexdigest(), ] diff --git a/src/blackd/__init__.py b/src/blackd/__init__.py index 6b0f3d33295..7041671f596 100644 --- a/src/blackd/__init__.py +++ b/src/blackd/__init__.py @@ -8,6 +8,7 @@ try: from aiohttp import web + from multidict import MultiMapping from .middlewares import cors except ImportError as ie: @@ -34,6 +35,8 @@ SKIP_STRING_NORMALIZATION_HEADER = "X-Skip-String-Normalization" SKIP_MAGIC_TRAILING_COMMA = "X-Skip-Magic-Trailing-Comma" PREVIEW = "X-Preview" +UNSTABLE = "X-Unstable" +ENABLE_UNSTABLE_FEATURE = "X-Enable-Unstable-Feature" FAST_OR_SAFE_HEADER = "X-Fast-Or-Safe" DIFF_HEADER = "X-Diff" @@ -45,6 +48,8 @@ SKIP_STRING_NORMALIZATION_HEADER, SKIP_MAGIC_TRAILING_COMMA, PREVIEW, + UNSTABLE, + ENABLE_UNSTABLE_FEATURE, FAST_OR_SAFE_HEADER, DIFF_HEADER, ] @@ -53,6 +58,10 @@ BLACK_VERSION_HEADER = "X-Black-Version" +class HeaderError(Exception): + pass + + class InvalidVariantHeader(Exception): pass @@ -93,55 +102,21 @@ async def handle(request: web.Request, executor: Executor) -> web.Response: return web.Response( status=501, text="This server only supports protocol version 1" ) - try: - line_length = int( - request.headers.get(LINE_LENGTH_HEADER, black.DEFAULT_LINE_LENGTH) - ) - except ValueError: - return web.Response(status=400, text="Invalid line length header value") - if PYTHON_VARIANT_HEADER in request.headers: - value = request.headers[PYTHON_VARIANT_HEADER] - try: - pyi, versions = parse_python_variant_header(value) - except InvalidVariantHeader as e: - return web.Response( - status=400, - text=f"Invalid value for {PYTHON_VARIANT_HEADER}: {e.args[0]}", - ) - else: - pyi = False - versions = set() - - skip_string_normalization = bool( - request.headers.get(SKIP_STRING_NORMALIZATION_HEADER, False) - ) - skip_magic_trailing_comma = bool( - request.headers.get(SKIP_MAGIC_TRAILING_COMMA, False) - ) - skip_source_first_line = bool( - request.headers.get(SKIP_SOURCE_FIRST_LINE, False) - ) - preview = bool(request.headers.get(PREVIEW, False)) fast = False if request.headers.get(FAST_OR_SAFE_HEADER, "safe") == "fast": fast = True - mode = black.FileMode( - target_versions=versions, - is_pyi=pyi, - line_length=line_length, - skip_source_first_line=skip_source_first_line, - string_normalization=not skip_string_normalization, - magic_trailing_comma=not skip_magic_trailing_comma, - preview=preview, - ) + try: + mode = parse_mode(request.headers) + except HeaderError as e: + return web.Response(status=400, text=e.args[0]) req_bytes = await request.content.read() charset = request.charset if request.charset is not None else "utf8" req_str = req_bytes.decode(charset) then = datetime.now(timezone.utc) header = "" - if skip_source_first_line: + if mode.skip_source_first_line: first_newline_position: int = req_str.find("\n") + 1 header = req_str[:first_newline_position] req_str = req_str[first_newline_position:] @@ -190,6 +165,57 @@ async def handle(request: web.Request, executor: Executor) -> web.Response: return web.Response(status=500, headers=headers, text=str(e)) +def parse_mode(headers: MultiMapping[str]) -> black.Mode: + try: + line_length = int(headers.get(LINE_LENGTH_HEADER, black.DEFAULT_LINE_LENGTH)) + except ValueError: + raise HeaderError("Invalid line length header value") from None + + if PYTHON_VARIANT_HEADER in headers: + value = headers[PYTHON_VARIANT_HEADER] + try: + pyi, versions = parse_python_variant_header(value) + except InvalidVariantHeader as e: + raise HeaderError( + f"Invalid value for {PYTHON_VARIANT_HEADER}: {e.args[0]}", + ) from None + else: + pyi = False + versions = set() + + skip_string_normalization = bool( + headers.get(SKIP_STRING_NORMALIZATION_HEADER, False) + ) + skip_magic_trailing_comma = bool(headers.get(SKIP_MAGIC_TRAILING_COMMA, False)) + skip_source_first_line = bool(headers.get(SKIP_SOURCE_FIRST_LINE, False)) + + preview = bool(headers.get(PREVIEW, False)) + unstable = bool(headers.get(UNSTABLE, False)) + enable_features: Set[black.Preview] = set() + enable_unstable_features = headers.get(ENABLE_UNSTABLE_FEATURE, "").split(",") + for piece in enable_unstable_features: + piece = piece.strip() + if piece: + try: + enable_features.add(black.Preview[piece]) + except KeyError: + raise HeaderError( + f"Invalid value for {ENABLE_UNSTABLE_FEATURE}: {piece}", + ) from None + + return black.FileMode( + target_versions=versions, + is_pyi=pyi, + line_length=line_length, + skip_source_first_line=skip_source_first_line, + string_normalization=not skip_string_normalization, + magic_trailing_comma=not skip_magic_trailing_comma, + preview=preview, + unstable=unstable, + enabled_features=enable_features, + ) + + def parse_python_variant_header(value: str) -> Tuple[bool, Set[black.TargetVersion]]: if value == "pyi": return True, set() diff --git a/tests/data/cases/preview_cantfit.py b/tests/data/cases/preview_cantfit.py index d5da6654f0c..29789c7e653 100644 --- a/tests/data/cases/preview_cantfit.py +++ b/tests/data/cases/preview_cantfit.py @@ -20,12 +20,6 @@ normal_name = but_the_function_name_is_now_ridiculously_long_and_it_is_still_super_annoying( [1, 2, 3], arg1, [1, 2, 3], arg2, [1, 2, 3], arg3 ) -# long arguments -normal_name = normal_function_name( - "but with super long string arguments that on their own exceed the line limit so there's no way it can ever fit", - "eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs", - this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it=0, -) string_variable_name = ( "a string that is waaaaaaaayyyyyyyy too long, even in parens, there's nothing you can do" # noqa ) @@ -78,14 +72,6 @@ [1, 2, 3], arg1, [1, 2, 3], arg2, [1, 2, 3], arg3 ) ) -# long arguments -normal_name = normal_function_name( - "but with super long string arguments that on their own exceed the line limit so" - " there's no way it can ever fit", - "eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs" - " with spam and eggs and spam with eggs", - this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it=0, -) string_variable_name = "a string that is waaaaaaaayyyyyyyy too long, even in parens, there's nothing you can do" # noqa for key in """ hostname diff --git a/tests/data/cases/preview_cantfit_string.py b/tests/data/cases/preview_cantfit_string.py new file mode 100644 index 00000000000..3b48e318ade --- /dev/null +++ b/tests/data/cases/preview_cantfit_string.py @@ -0,0 +1,18 @@ +# flags: --unstable +# long arguments +normal_name = normal_function_name( + "but with super long string arguments that on their own exceed the line limit so there's no way it can ever fit", + "eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs", + this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it=0, +) + +# output + +# long arguments +normal_name = normal_function_name( + "but with super long string arguments that on their own exceed the line limit so" + " there's no way it can ever fit", + "eggs with spam and eggs and spam with eggs with spam and eggs and spam with eggs" + " with spam and eggs and spam with eggs", + this_is_a_ridiculously_long_name_and_nobody_in_their_right_mind_would_use_one_like_it=0, +) diff --git a/tests/data/cases/preview_comments7.py b/tests/data/cases/preview_comments7.py index 006d4f7266f..e4d547138db 100644 --- a/tests/data/cases/preview_comments7.py +++ b/tests/data/cases/preview_comments7.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable from .config import ( Any, Bool, diff --git a/tests/data/cases/preview_long_dict_values.py b/tests/data/cases/preview_long_dict_values.py index 54da76038dc..a19210605f6 100644 --- a/tests/data/cases/preview_long_dict_values.py +++ b/tests/data/cases/preview_long_dict_values.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable my_dict = { "something_something": r"Lorem ipsum dolor sit amet, an sed convenire eloquentiam \t" diff --git a/tests/data/cases/preview_long_strings.py b/tests/data/cases/preview_long_strings.py index 19ac47a7032..86fa1b0c7e1 100644 --- a/tests/data/cases/preview_long_strings.py +++ b/tests/data/cases/preview_long_strings.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable x = "This is a really long string that can't possibly be expected to fit all together on one line. In fact it may even take up three or more lines... like four or five... but probably just three." x += "This is a really long string that can't possibly be expected to fit all together on one line. In fact it may even take up three or more lines... like four or five... but probably just three." diff --git a/tests/data/cases/preview_long_strings__east_asian_width.py b/tests/data/cases/preview_long_strings__east_asian_width.py index d190f422a60..022b0452522 100644 --- a/tests/data/cases/preview_long_strings__east_asian_width.py +++ b/tests/data/cases/preview_long_strings__east_asian_width.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable # The following strings do not have not-so-many chars, but are long enough # when these are rendered in a monospace font (if the renderer respects # Unicode East Asian Width properties). diff --git a/tests/data/cases/preview_long_strings__edge_case.py b/tests/data/cases/preview_long_strings__edge_case.py index a8e8971968c..28497e731bc 100644 --- a/tests/data/cases/preview_long_strings__edge_case.py +++ b/tests/data/cases/preview_long_strings__edge_case.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable some_variable = "This string is long but not so long that it needs to be split just yet" some_variable = 'This string is long but not so long that it needs to be split just yet' some_variable = "This string is long, just long enough that it needs to be split, u get?" diff --git a/tests/data/cases/preview_long_strings__regression.py b/tests/data/cases/preview_long_strings__regression.py index 5e76a8cf61c..afe2b311cf4 100644 --- a/tests/data/cases/preview_long_strings__regression.py +++ b/tests/data/cases/preview_long_strings__regression.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable class A: def foo(): result = type(message)("") diff --git a/tests/data/cases/preview_multiline_strings.py b/tests/data/cases/preview_multiline_strings.py index 3ff643610b7..9288f6991bd 100644 --- a/tests/data/cases/preview_multiline_strings.py +++ b/tests/data/cases/preview_multiline_strings.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable """cow say""", call(3, "dogsay", textwrap.dedent("""dove diff --git a/tests/data/cases/preview_return_annotation_brackets_string.py b/tests/data/cases/preview_return_annotation_brackets_string.py index fea0ea6839a..2f937cf54ef 100644 --- a/tests/data/cases/preview_return_annotation_brackets_string.py +++ b/tests/data/cases/preview_return_annotation_brackets_string.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable # Long string example def frobnicate() -> "ThisIsTrulyUnreasonablyExtremelyLongClassName | list[ThisIsTrulyUnreasonablyExtremelyLongClassName]": pass diff --git a/tests/test_black.py b/tests/test_black.py index a979a95b674..6dbe25a90b6 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -158,13 +158,11 @@ def test_empty_ff(self) -> None: @patch("black.dump_to_file", dump_to_stderr) def test_one_empty_line(self) -> None: - mode = black.Mode(preview=True) for nl in ["\n", "\r\n"]: source = expected = nl - assert_format(source, expected, mode=mode) + assert_format(source, expected) def test_one_empty_line_ff(self) -> None: - mode = black.Mode(preview=True) for nl in ["\n", "\r\n"]: expected = nl tmp_file = Path(black.dump_to_file(nl)) @@ -175,20 +173,13 @@ def test_one_empty_line_ff(self) -> None: with open(tmp_file, "wb") as f: f.write(nl.encode("utf-8")) try: - self.assertFalse( - ff(tmp_file, mode=mode, write_back=black.WriteBack.YES) - ) + self.assertFalse(ff(tmp_file, write_back=black.WriteBack.YES)) with open(tmp_file, "rb") as f: actual = f.read().decode("utf-8") finally: os.unlink(tmp_file) self.assertFormatEqual(expected, actual) - def test_experimental_string_processing_warns(self) -> None: - self.assertWarns( - black.mode.Deprecated, black.Mode, experimental_string_processing=True - ) - def test_piping(self) -> None: _, source, expected = read_data_from_file( PROJECT_ROOT / "src/black/__init__.py" @@ -252,21 +243,6 @@ def test_piping_diff_with_color(self) -> None: self.assertIn("\033[31m", actual) self.assertIn("\033[0m", actual) - @patch("black.dump_to_file", dump_to_stderr) - def _test_wip(self) -> None: - source, expected = read_data("miscellaneous", "wip") - sys.settrace(tracefunc) - mode = replace( - DEFAULT_MODE, - experimental_string_processing=False, - target_versions={black.TargetVersion.PY38}, - ) - actual = fs(source, mode=mode) - sys.settrace(None) - self.assertFormatEqual(expected, actual) - black.assert_equivalent(source, actual) - black.assert_stable(source, actual, black.FileMode()) - def test_pep_572_version_detection(self) -> None: source, _ = read_data("cases", "pep_572") root = black.lib2to3_parse(source) @@ -374,7 +350,7 @@ def test_detect_debug_f_strings(self) -> None: @patch("black.dump_to_file", dump_to_stderr) def test_string_quotes(self) -> None: source, expected = read_data("miscellaneous", "string_quotes") - mode = black.Mode(preview=True) + mode = black.Mode(unstable=True) assert_format(source, expected, mode) mode = replace(mode, string_normalization=False) not_normalized = fs(source, mode=mode) @@ -1052,7 +1028,6 @@ def test_format_file_contents(self) -> None: black.format_file_contents(invalid, mode=mode, fast=False) self.assertEqual(str(e.exception), "Cannot parse: 1:7: return if you can") - mode = black.Mode(preview=True) just_crlf = "\r\n" with self.assertRaises(black.NothingChanged): black.format_file_contents(just_crlf, mode=mode, fast=False) @@ -1396,7 +1371,6 @@ def get_output(*args: Any, **kwargs: Any) -> io.TextIOWrapper: return get_output - mode = black.Mode(preview=True) for content, expected in cases: output = io.StringIO() io_TextIOWrapper = io.TextIOWrapper @@ -1407,26 +1381,27 @@ def get_output(*args: Any, **kwargs: Any) -> io.TextIOWrapper: fast=True, content=content, write_back=black.WriteBack.YES, - mode=mode, + mode=DEFAULT_MODE, ) except io.UnsupportedOperation: pass # StringIO does not support detach assert output.getvalue() == expected - # An empty string is the only test case for `preview=False` - output = io.StringIO() - io_TextIOWrapper = io.TextIOWrapper - with patch("io.TextIOWrapper", _new_wrapper(output, io_TextIOWrapper)): - try: - black.format_stdin_to_stdout( - fast=True, - content="", - write_back=black.WriteBack.YES, - mode=DEFAULT_MODE, - ) - except io.UnsupportedOperation: - pass # StringIO does not support detach - assert output.getvalue() == "" + def test_cli_unstable(self) -> None: + self.invokeBlack(["--unstable", "-c", "0"], exit_code=0) + self.invokeBlack(["--preview", "-c", "0"], exit_code=0) + # Must also pass --preview + self.invokeBlack( + ["--enable-unstable-feature", "string_processing", "-c", "0"], exit_code=1 + ) + self.invokeBlack( + ["--preview", "--enable-unstable-feature", "string_processing", "-c", "0"], + exit_code=0, + ) + self.invokeBlack( + ["--unstable", "--enable-unstable-feature", "string_processing", "-c", "0"], + exit_code=0, + ) def test_invalid_cli_regex(self) -> None: for option in ["--include", "--exclude", "--extend-exclude", "--force-exclude"]: diff --git a/tests/util.py b/tests/util.py index 9ea30e62fe3..d5425f1f743 100644 --- a/tests/util.py +++ b/tests/util.py @@ -112,16 +112,24 @@ def assert_format( # For both preview and non-preview tests, ensure that Black doesn't crash on # this code, but don't pass "expected" because the precise output may differ. try: + if mode.unstable: + new_mode = replace(mode, unstable=False, preview=False) + else: + new_mode = replace(mode, preview=not mode.preview) _assert_format_inner( source, None, - replace(mode, preview=not mode.preview), + new_mode, fast=fast, minimum_version=minimum_version, lines=lines, ) except Exception as e: - text = "non-preview" if mode.preview else "preview" + text = ( + "unstable" + if mode.unstable + else "non-preview" if mode.preview else "preview" + ) raise FormatFailure( f"Black crashed formatting this case in {text} mode." ) from e @@ -138,7 +146,7 @@ def assert_format( _assert_format_inner( source, None, - replace(mode, preview=preview_mode, line_length=1), + replace(mode, preview=preview_mode, line_length=1, unstable=False), fast=fast, minimum_version=minimum_version, lines=lines, @@ -241,6 +249,7 @@ def get_flags_parser() -> argparse.ArgumentParser: "--skip-magic-trailing-comma", default=False, action="store_true" ) parser.add_argument("--preview", default=False, action="store_true") + parser.add_argument("--unstable", default=False, action="store_true") parser.add_argument("--fast", default=False, action="store_true") parser.add_argument( "--minimum-version", @@ -278,6 +287,7 @@ def parse_mode(flags_line: str) -> TestCaseArgs: is_ipynb=args.ipynb, magic_trailing_comma=not args.skip_magic_trailing_comma, preview=args.preview, + unstable=args.unstable, ) if args.line_ranges: lines = parse_line_ranges(args.line_ranges) From 0e6e46b9eb45f5a22062fe84c2c2ff46bd0d738e Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 25 Jan 2024 20:35:21 -0800 Subject: [PATCH 252/279] Prepare release 24.1.0 (#4170) --- CHANGES.md | 46 +++------------------ docs/integrations/source_version_control.md | 4 +- docs/usage_and_configuration/the_basics.md | 6 +-- scripts/release.py | 4 +- 4 files changed, 13 insertions(+), 47 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0496603e2c0..ff921de69eb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,11 +1,9 @@ # Change Log -## Unreleased +## 24.1.0 ### Highlights - - This release introduces the new 2024 stable style (#4106), stabilizing the following changes: @@ -44,8 +42,6 @@ year's stable style. ### Stable style - - Several bug fixes were made in features that are moved to the stable style in this release: @@ -59,14 +55,12 @@ release: ### Preview style - - - Add `--unstable` style, covering preview features that have known problems that would block them from going into the stable style. Also add the `--enable-unstable-feature` flag; for example, use `--enable-unstable-feature hug_parens_with_braces_and_square_brackets` to apply this - preview style throughout 2024, even if a later Black release downgrades the feature to - unstable (#4096) + preview feature throughout 2024, even if a later Black release downgrades the feature + to unstable (#4096) - Format module docstrings the same as class and function docstrings (#4095) - Fix crash when using a walrus in a dictionary (#4155) - Fix unnecessary parentheses when wrapping long dicts (#4135) @@ -74,48 +68,18 @@ release: ### Configuration - - -- Print warning when toml config contains an invalid key (#4165) -- Fix symlink handling, properly catch and ignore symlinks that point outside of root - (#4161) +- Print warning when configuration in `pyproject.toml` contains an invalid key (#4165) +- Fix symlink handling, properly ignoring symlinks that point outside of root (#4161) - Fix cache mtime logic that resulted in false positive cache hits (#4128) - Remove the long-deprecated `--experimental-string-processing` flag. This feature can currently be enabled with `--preview --enable-unstable-feature string_processing`. (#4096) -### Packaging - - - -### Parser - - - -### Performance - - - -### Output - - - -### _Blackd_ - - - ### Integrations - - - Revert the change to run Black's pre-commit integration only on specific git hooks (#3940) for better compatibility with older versions of pre-commit (#4137) -### Documentation - - - ## 23.12.1 ### Packaging diff --git a/docs/integrations/source_version_control.md b/docs/integrations/source_version_control.md index 3b895193941..259c1c1eaf3 100644 --- a/docs/integrations/source_version_control.md +++ b/docs/integrations/source_version_control.md @@ -8,7 +8,7 @@ Use [pre-commit](https://pre-commit.com/). Once you repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.12.1 + rev: 24.1.0 hooks: - id: black # It is recommended to specify the latest version of Python @@ -35,7 +35,7 @@ include Jupyter Notebooks. To use this hook, simply replace the hook's `id: blac repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.12.1 + rev: 24.1.0 hooks: - id: black-jupyter # It is recommended to specify the latest version of Python diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index a42e093155b..562fd7d5905 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -266,8 +266,8 @@ configuration file for consistent results across environments. ```console $ black --version -black, 23.12.1 (compiled: yes) -$ black --required-version 23.12.1 -c "format = 'this'" +black, 24.1.0 (compiled: yes) +$ black --required-version 24.1.0 -c "format = 'this'" format = "this" $ black --required-version 31.5b2 -c "still = 'beta?!'" Oh no! 💥 💔 💥 The required version does not match the running version! @@ -363,7 +363,7 @@ You can check the version of _Black_ you have installed using the `--version` fl ```console $ black --version -black, 23.12.1 +black, 24.1.0 ``` #### `--config` diff --git a/scripts/release.py b/scripts/release.py index d588429c2d3..c5336506396 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -169,7 +169,9 @@ def get_next_version(self) -> str: calver_parts = base_calver.split(".") base_calver = f"{calver_parts[0]}.{int(calver_parts[1])}" # Remove leading 0 git_tags = get_git_tags() - same_month_releases = [t for t in git_tags if t.startswith(base_calver)] + same_month_releases = [ + t for t in git_tags if t.startswith(base_calver) and "a" not in t + ] if len(same_month_releases) < 1: return f"{base_calver}.0" same_month_version = same_month_releases[-1].split(".", 2)[-1] From 659c29a41c7c686687aef21f57b95bcfa236b03b Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 25 Jan 2024 21:12:38 -0800 Subject: [PATCH 253/279] New changelog --- CHANGES.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index ff921de69eb..9a9be4bbeae 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,52 @@ # Change Log +## Unreleased + +### Highlights + + + +### Stable style + + + +### Preview style + + + +### Configuration + + + +### Packaging + + + +### Parser + + + +### Performance + + + +### Output + + + +### _Blackd_ + + + +### Integrations + + + +### Documentation + + + ## 24.1.0 ### Highlights From ed770ba4dd50c419148a0fca2b43937a7447e1f9 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 26 Jan 2024 11:54:49 -0800 Subject: [PATCH 254/279] Fix cache file length (#4176) - Ensure total file length stays under 96 - Hash the path only if it's too long - Proceed normally (with a warning) if the cache can't be read Fixes #4172 --- CHANGES.md | 3 +++ src/black/cache.py | 9 ++++++++- src/black/mode.py | 21 +++++++++++++++++---- tests/test_black.py | 25 +++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9a9be4bbeae..e4240eacfca 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -18,6 +18,9 @@ +- Shorten the length of the name of the cache file to fix crashes on file systems that + do not support long paths (#4176) + ### Packaging diff --git a/src/black/cache.py b/src/black/cache.py index cfdbc21e92a..35bddb573d2 100644 --- a/src/black/cache.py +++ b/src/black/cache.py @@ -13,6 +13,7 @@ from _black_version import version as __version__ from black.mode import Mode +from black.output import err if sys.version_info >= (3, 11): from typing import Self @@ -64,7 +65,13 @@ def read(cls, mode: Mode) -> Self: resolve the issue. """ cache_file = get_cache_file(mode) - if not cache_file.exists(): + try: + exists = cache_file.exists() + except OSError as e: + # Likely file too long; see #4172 and #4174 + err(f"Unable to read cache file {cache_file} due to {e}") + return cls(mode, cache_file) + if not exists: return cls(mode, cache_file) with cache_file.open("rb") as fobj: diff --git a/src/black/mode.py b/src/black/mode.py index 68919fb4901..128d2b9f108 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -192,6 +192,9 @@ class Deprecated(UserWarning): """Visible deprecation warning.""" +_MAX_CACHE_KEY_PART_LENGTH: Final = 32 + + @dataclass class Mode: target_versions: Set[TargetVersion] = field(default_factory=set) @@ -228,6 +231,19 @@ def get_cache_key(self) -> str: ) else: version_str = "-" + if len(version_str) > _MAX_CACHE_KEY_PART_LENGTH: + version_str = sha256(version_str.encode()).hexdigest()[ + :_MAX_CACHE_KEY_PART_LENGTH + ] + features_and_magics = ( + ",".join(sorted(f.name for f in self.enabled_features)) + + "@" + + ",".join(sorted(self.python_cell_magics)) + ) + if len(features_and_magics) > _MAX_CACHE_KEY_PART_LENGTH: + features_and_magics = sha256(features_and_magics.encode()).hexdigest()[ + :_MAX_CACHE_KEY_PART_LENGTH + ] parts = [ version_str, str(self.line_length), @@ -236,10 +252,7 @@ def get_cache_key(self) -> str: str(int(self.is_ipynb)), str(int(self.skip_source_first_line)), str(int(self.magic_trailing_comma)), - sha256( - (",".join(sorted(f.name for f in self.enabled_features))).encode() - ).hexdigest(), str(int(self.preview)), - sha256((",".join(sorted(self.python_cell_magics))).encode()).hexdigest(), + features_and_magics, ] return ".".join(parts) diff --git a/tests/test_black.py b/tests/test_black.py index 6dbe25a90b6..123ea0bb88a 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -44,6 +44,7 @@ from black import re_compile_maybe_verbose as compile_pattern from black.cache import FileData, get_cache_dir, get_cache_file from black.debug import DebugVisitor +from black.mode import Mode, Preview from black.output import color_diff, diff from black.report import Report @@ -2065,6 +2066,30 @@ def test_get_cache_dir( monkeypatch.setenv("BLACK_CACHE_DIR", str(workspace2)) assert get_cache_dir().parent == workspace2 + def test_cache_file_length(self) -> None: + cases = [ + DEFAULT_MODE, + # all of the target versions + Mode(target_versions=set(TargetVersion)), + # all of the features + Mode(enabled_features=set(Preview)), + # all of the magics + Mode(python_cell_magics={f"magic{i}" for i in range(500)}), + # all of the things + Mode( + target_versions=set(TargetVersion), + enabled_features=set(Preview), + python_cell_magics={f"magic{i}" for i in range(500)}, + ), + ] + for case in cases: + cache_file = get_cache_file(case) + # Some common file systems enforce a maximum path length + # of 143 (issue #4174). We can't do anything if the directory + # path is too long, but ensure the name of the cache file itself + # doesn't get too crazy. + assert len(cache_file.name) <= 96 + def test_cache_broken_file(self) -> None: mode = DEFAULT_MODE with cache_dir() as workspace: From 1607e9ab20ad550cf940482d0d361ca31fc03189 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sat, 27 Jan 2024 12:34:02 -0800 Subject: [PATCH 255/279] Fix missing space in option description (#4182) --- src/black/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index ebc7ac8eda5..8ab5b47f974 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -279,7 +279,7 @@ def validate_regex( is_flag=True, help=( "Format all input files like Jupyter Notebooks regardless of file extension." - "This is useful when piping source on standard input." + " This is useful when piping source on standard input." ), ) @click.option( From 8bf04549ffd276a1bad6eb110e66e6557ee630d9 Mon Sep 17 00:00:00 2001 From: cobalt <61329810+RedGuy12@users.noreply.github.com> Date: Sat, 27 Jan 2024 15:55:22 -0600 Subject: [PATCH 256/279] Consistently add trailing comma on typed parameters (#4164) Signed-off-by: RedGuy12 <61329810+RedGuy12@users.noreply.github.com> Co-authored-by: Jelle Zijlstra --- CHANGES.md | 2 ++ docs/the_black_code_style/future_style.md | 2 ++ src/black/files.py | 2 +- src/black/linegen.py | 10 +++++++- src/black/mode.py | 1 + src/blib2to3/pgen2/parse.py | 2 +- .../data/cases/typed_params_trailing_comma.py | 24 +++++++++++++++++++ 7 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 tests/data/cases/typed_params_trailing_comma.py diff --git a/CHANGES.md b/CHANGES.md index e4240eacfca..6278aed77d8 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,8 @@ +- Consistently add trailing comma on typed parameters (#4164) + ### Configuration diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index 1cdd25fdb7c..d5faae36911 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -26,6 +26,8 @@ Currently, the following features are included in the preview style: brackets ([see below](labels/hug-parens)) - `no_normalize_fmt_skip_whitespace`: whitespace before `# fmt: skip` comments is no longer normalized +- `typed_params_trailing_comma`: consistently add trailing commas to typed function + parameters (labels/unstable-features)= diff --git a/src/black/files.py b/src/black/files.py index 65951efdbe8..1eb8745572b 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -131,7 +131,7 @@ def parse_pyproject_toml(path_config: str) -> Dict[str, Any]: def infer_target_version( - pyproject_toml: Dict[str, Any] + pyproject_toml: Dict[str, Any], ) -> Optional[List[TargetVersion]]: """Infer Black's target version from the project metadata in pyproject.toml. diff --git a/src/black/linegen.py b/src/black/linegen.py index a276805f2fe..c74ff9c0b4b 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -48,6 +48,7 @@ is_one_sequence_between, is_one_tuple, is_parent_function_or_class, + is_part_of_annotation, is_rpar_token, is_stub_body, is_stub_suite, @@ -1041,7 +1042,14 @@ def bracket_split_build_line( no_commas = ( original.is_def and opening_bracket.value == "(" - and not any(leaf.type == token.COMMA for leaf in leaves) + and not any( + leaf.type == token.COMMA + and ( + Preview.typed_params_trailing_comma not in original.mode + or not is_part_of_annotation(leaf) + ) + for leaf in leaves + ) # In particular, don't add one within a parenthesized return annotation. # Unfortunately the indicator we're in a return annotation (RARROW) may # be defined directly in the parent node, the parent of the parent ... diff --git a/src/black/mode.py b/src/black/mode.py index 128d2b9f108..22352e7c6a8 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -176,6 +176,7 @@ class Preview(Enum): no_normalize_fmt_skip_whitespace = auto() wrap_long_dict_values_in_parens = auto() multiline_string_handling = auto() + typed_params_trailing_comma = auto() UNSTABLE_FEATURES: Set[Preview] = { diff --git a/src/blib2to3/pgen2/parse.py b/src/blib2to3/pgen2/parse.py index ad51a3dad08..ad1d795b51a 100644 --- a/src/blib2to3/pgen2/parse.py +++ b/src/blib2to3/pgen2/parse.py @@ -50,7 +50,7 @@ def lam_sub(grammar: Grammar, node: RawNode) -> NL: def stack_copy( - stack: List[Tuple[DFAS, int, RawNode]] + stack: List[Tuple[DFAS, int, RawNode]], ) -> List[Tuple[DFAS, int, RawNode]]: """Nodeless stack copy.""" return [(dfa, label, DUMMY_NODE) for dfa, label, _ in stack] diff --git a/tests/data/cases/typed_params_trailing_comma.py b/tests/data/cases/typed_params_trailing_comma.py new file mode 100644 index 00000000000..a53b908b18b --- /dev/null +++ b/tests/data/cases/typed_params_trailing_comma.py @@ -0,0 +1,24 @@ +# flags: --preview +def long_function_name_goes_here( + x: Callable[List[int]] +) -> Union[List[int], float, str, bytes, Tuple[int]]: + pass + + +def long_function_name_goes_here( + x: Callable[[str, Any], int] +) -> Union[List[int], float, str, bytes, Tuple[int]]: + pass + + +# output +def long_function_name_goes_here( + x: Callable[List[int]], +) -> Union[List[int], float, str, bytes, Tuple[int]]: + pass + + +def long_function_name_goes_here( + x: Callable[[str, Any], int], +) -> Union[List[int], float, str, bytes, Tuple[int]]: + pass From 79fc1158a98281dac798feb14b8fddb4051e4a42 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 27 Jan 2024 23:24:36 -0500 Subject: [PATCH 257/279] chore: ignore node_modules (produced by a pre-commit check) (#4184) Signed-off-by: Henry Schreiner --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4a4f1b738ad..6e23797d110 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ src/_black_version.py .hypothesis/ venv/ .ipynb_checkpoints/ +node_modules/ From e026c93888f91a47a9c9f4e029f3eb07d96375e6 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sat, 27 Jan 2024 20:51:32 -0800 Subject: [PATCH 258/279] Prepare release 24.1.1 (#4186) --- CHANGES.md | 44 ++------------------- docs/integrations/source_version_control.md | 4 +- docs/usage_and_configuration/the_basics.md | 6 +-- 3 files changed, 8 insertions(+), 46 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6278aed77d8..a794f421694 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,57 +1,19 @@ # Change Log -## Unreleased +## 24.1.1 -### Highlights - - - -### Stable style - - +Bugfix release to fix a bug that made Black unusable on certain file systems with strict +limits on path length. ### Preview style - - - Consistently add trailing comma on typed parameters (#4164) ### Configuration - - - Shorten the length of the name of the cache file to fix crashes on file systems that do not support long paths (#4176) -### Packaging - - - -### Parser - - - -### Performance - - - -### Output - - - -### _Blackd_ - - - -### Integrations - - - -### Documentation - - - ## 24.1.0 ### Highlights diff --git a/docs/integrations/source_version_control.md b/docs/integrations/source_version_control.md index 259c1c1eaf3..92279707d84 100644 --- a/docs/integrations/source_version_control.md +++ b/docs/integrations/source_version_control.md @@ -8,7 +8,7 @@ Use [pre-commit](https://pre-commit.com/). Once you repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.1.0 + rev: 24.1.1 hooks: - id: black # It is recommended to specify the latest version of Python @@ -35,7 +35,7 @@ include Jupyter Notebooks. To use this hook, simply replace the hook's `id: blac repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.1.0 + rev: 24.1.1 hooks: - id: black-jupyter # It is recommended to specify the latest version of Python diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 562fd7d5905..dc9d9a64c68 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -266,8 +266,8 @@ configuration file for consistent results across environments. ```console $ black --version -black, 24.1.0 (compiled: yes) -$ black --required-version 24.1.0 -c "format = 'this'" +black, 24.1.1 (compiled: yes) +$ black --required-version 24.1.1 -c "format = 'this'" format = "this" $ black --required-version 31.5b2 -c "still = 'beta?!'" Oh no! 💥 💔 💥 The required version does not match the running version! @@ -363,7 +363,7 @@ You can check the version of _Black_ you have installed using the `--version` fl ```console $ black --version -black, 24.1.0 +black, 24.1.1 ``` #### `--config` From 0b4364b7e38356428565fceb706a76ebf9a373e5 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 28 Jan 2024 05:37:12 -0800 Subject: [PATCH 259/279] Add new release template --- CHANGES.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index a794f421694..5ee1394282e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,52 @@ # Change Log +## Unreleased + +### Highlights + + + +### Stable style + + + +### Preview style + + + +### Configuration + + + +### Packaging + + + +### Parser + + + +### Performance + + + +### Output + + + +### _Blackd_ + + + +### Integrations + + + +### Documentation + + + ## 24.1.1 Bugfix release to fix a bug that made Black unusable on certain file systems with strict From d919746fae2ba47442fc33699a5569fa04db305d Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sun, 28 Jan 2024 07:05:56 -0800 Subject: [PATCH 260/279] Swallow warnings when performing AST checks (#4189) Fixes #4188 --- CHANGES.md | 3 +++ src/black/parsing.py | 16 ++++++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5ee1394282e..6da30afc707 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -34,6 +34,9 @@ +- Black will swallow any `SyntaxWarning`s or `DeprecationWarning`s produced by the `ast` + module when performing equivalence checks (#4189) + ### _Blackd_ diff --git a/src/black/parsing.py b/src/black/parsing.py index 178a7ef10e2..63c5e71a0fe 100644 --- a/src/black/parsing.py +++ b/src/black/parsing.py @@ -4,6 +4,7 @@ import ast import sys +import warnings from typing import Iterable, Iterator, List, Set, Tuple from black.mode import VERSION_TO_FEATURES, Feature, TargetVersion, supports_feature @@ -109,13 +110,16 @@ def lib2to3_unparse(node: Node) -> str: return code -def parse_single_version( +def _parse_single_version( src: str, version: Tuple[int, int], *, type_comments: bool ) -> ast.AST: filename = "" - return ast.parse( - src, filename, feature_version=version, type_comments=type_comments - ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", SyntaxWarning) + warnings.simplefilter("ignore", DeprecationWarning) + return ast.parse( + src, filename, feature_version=version, type_comments=type_comments + ) def parse_ast(src: str) -> ast.AST: @@ -125,7 +129,7 @@ def parse_ast(src: str) -> ast.AST: first_error = "" for version in sorted(versions, reverse=True): try: - return parse_single_version(src, version, type_comments=True) + return _parse_single_version(src, version, type_comments=True) except SyntaxError as e: if not first_error: first_error = str(e) @@ -133,7 +137,7 @@ def parse_ast(src: str) -> ast.AST: # Try to parse without type comments for version in sorted(versions, reverse=True): try: - return parse_single_version(src, version, type_comments=False) + return _parse_single_version(src, version, type_comments=False) except SyntaxError: pass From 177e306363b9da5fcef621d15c205453ad1cfc74 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 28 Jan 2024 23:07:34 -0800 Subject: [PATCH 261/279] Bump pypa/cibuildwheel from 2.16.2 to 2.16.4 (#4191) Bumps [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) from 2.16.2 to 2.16.4. - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v2.16.2...v2.16.4) --- .github/workflows/pypi_upload.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index 8e3eb67a10d..52525419f0a 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -89,7 +89,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: pypa/cibuildwheel@v2.16.2 + - uses: pypa/cibuildwheel@v2.16.4 with: only: ${{ matrix.only }} From 2bc5ce8ae1f1ad52909b4656444c1e9988e23cab Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 29 Jan 2024 10:56:48 -0500 Subject: [PATCH 262/279] feat: add schema and validate-pyproject support (#4181) Signed-off-by: Henry Schreiner Co-authored-by: Jelle Zijlstra --- .github/workflows/lint.yml | 5 + .github/workflows/test.yml | 2 +- .pre-commit-config.yaml | 11 +- CHANGES.md | 2 + pyproject.toml | 3 + scripts/generate_schema.py | 74 +++++++++++++ src/black/resources/__init__.py | 0 src/black/resources/black.schema.json | 149 ++++++++++++++++++++++++++ src/black/schema.py | 20 ++++ tests/test_schema.py | 17 +++ tox.ini | 10 +- 11 files changed, 288 insertions(+), 5 deletions(-) create mode 100644 scripts/generate_schema.py create mode 100644 src/black/resources/__init__.py create mode 100644 src/black/resources/black.schema.json create mode 100644 src/black/schema.py create mode 100644 tests/test_schema.py diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2d016cef7a6..8cb335bffb5 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -40,3 +40,8 @@ jobs: - name: Format ourselves run: | tox -e run_self + + - name: Regenerate schema + run: | + tox -e generate_schema + git diff --exit-code diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 55359a23303..fc274208258 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -106,4 +106,4 @@ jobs: python -m pip install -e ".[uvloop]" - name: Format ourselves - run: python -m black --check . + run: python -m black --check src/ tests/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 13479565527..05569eed8db 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,9 +42,9 @@ repos: rev: v1.8.0 hooks: - id: mypy - exclude: ^docs/conf.py - args: ["--config-file", "pyproject.toml"] - additional_dependencies: + exclude: ^(docs/conf.py|scripts/generate_schema.py)$ + args: [] + additional_dependencies: &mypy_deps - types-PyYAML - tomli >= 0.2.6, < 2.0.0 - click >= 8.1.0, != 8.1.4, != 8.1.5 @@ -56,6 +56,11 @@ repos: - types-commonmark - urllib3 - hypothesmith + - id: mypy + name: mypy (Python 3.10) + files: scripts/generate_schema.py + args: ["--python-version=3.10"] + additional_dependencies: *mypy_deps - repo: https://github.com/pre-commit/mirrors-prettier rev: v4.0.0-alpha.8 diff --git a/CHANGES.md b/CHANGES.md index 6da30afc707..4fd030df6df 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -45,6 +45,8 @@ +- Add a JSONSchema and provide a validate-pyproject entry-point (#4181) + ### Documentation +- _Black_ now ignores `pyproject.toml` that is missing a `tool.black` section when + discovering project root and configuration. Since _Black_ continues to use version + control as an indicator of project root, this is expected to primarily change behavior + for users in a monorepo setup (desirably). If you wish to preserve previous behavior, + simply add an empty `[tool.black]` to the previously discovered `pyproject.toml` + (#4204) ### Packaging diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index dc9d9a64c68..61c52450165 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -456,10 +456,11 @@ of tools like [Poetry](https://python-poetry.org/), ### Where _Black_ looks for the file -By default _Black_ looks for `pyproject.toml` starting from the common base directory of -all files and directories passed on the command line. If it's not there, it looks in -parent directories. It stops looking when it finds the file, or a `.git` directory, or a -`.hg` directory, or the root of the file system, whichever comes first. +By default _Black_ looks for `pyproject.toml` containing a `[tool.black]` section +starting from the common base directory of all files and directories passed on the +command line. If it's not there, it looks in parent directories. It stops looking when +it finds the file, or a `.git` directory, or a `.hg` directory, or the root of the file +system, whichever comes first. If you're formatting standard input, _Black_ will look for configuration starting from the current working directory. diff --git a/src/black/files.py b/src/black/files.py index 1eb8745572b..960f13ee270 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -42,6 +42,12 @@ import colorama # noqa: F401 +@lru_cache +def _load_toml(path: Union[Path, str]) -> Dict[str, Any]: + with open(path, "rb") as f: + return tomllib.load(f) + + @lru_cache def find_project_root( srcs: Sequence[str], stdin_filename: Optional[str] = None @@ -84,7 +90,9 @@ def find_project_root( return directory, ".hg directory" if (directory / "pyproject.toml").is_file(): - return directory, "pyproject.toml" + pyproject_toml = _load_toml(directory / "pyproject.toml") + if "black" in pyproject_toml.get("tool", {}): + return directory, "pyproject.toml" return directory, "file system root" @@ -117,8 +125,7 @@ def parse_pyproject_toml(path_config: str) -> Dict[str, Any]: If parsing fails, will raise a tomllib.TOMLDecodeError. """ - with open(path_config, "rb") as f: - pyproject_toml = tomllib.load(f) + pyproject_toml = _load_toml(path_config) config: Dict[str, Any] = pyproject_toml.get("tool", {}).get("black", {}) config = {k.replace("--", "").replace("-", "_"): v for k, v in config.items()} diff --git a/tests/test_black.py b/tests/test_black.py index 123ea0bb88a..f876d365b12 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1668,9 +1668,9 @@ def test_find_project_root(self) -> None: src_dir.mkdir() root_pyproject = root / "pyproject.toml" - root_pyproject.touch() + root_pyproject.write_text("[tool.black]", encoding="utf-8") src_pyproject = src_dir / "pyproject.toml" - src_pyproject.touch() + src_pyproject.write_text("[tool.black]", encoding="utf-8") src_python = src_dir / "foo.py" src_python.touch() @@ -1693,6 +1693,20 @@ def test_find_project_root(self) -> None: (src_dir.resolve(), "pyproject.toml"), ) + src_sub = src_dir / "sub" + src_sub.mkdir() + + src_sub_pyproject = src_sub / "pyproject.toml" + src_sub_pyproject.touch() # empty + + src_sub_python = src_sub / "bar.py" + + # we skip src_sub_pyproject since it is missing the [tool.black] section + self.assertEqual( + black.find_project_root((src_sub_python,)), + (src_dir.resolve(), "pyproject.toml"), + ) + @patch( "black.files.find_user_pyproject_toml", ) From 9728b8e9b8720b5e401249c077e32de46a479161 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 1 Feb 2024 21:58:51 -0800 Subject: [PATCH 268/279] Move hug_parens_with_braces_and_square_brackets into the unstable style (#4198) Primarily because of #4036 (a crash) but also because of the feedback in #4098 and #4099. --- CHANGES.md | 3 +++ docs/the_black_code_style/future_style.md | 4 ++-- src/black/mode.py | 2 ++ .../preview_hug_parens_with_braces_and_square_brackets.py | 2 +- ...eview_hug_parens_with_braces_and_square_brackets_no_ll1.py | 2 +- 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a3ffba610d9..4d646a2779e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,9 @@ +- Move the `hug_parens_with_braces_and_square_brackets` feature to the unstable style + due to an outstanding crash and proposed formatting tweaks (#4198) + ### Configuration - _Black_ now ignores `pyproject.toml` that is missing a `tool.black` section when diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index 1c11186f811..86e5aa806b2 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -24,8 +24,6 @@ Currently, the following features are included in the preview style: strings - `unify_docstring_detection`: fix inconsistencies in whether certain strings are detected as docstrings -- `hug_parens_with_braces_and_square_brackets`: more compact formatting of nested - brackets ([see below](labels/hug-parens)) - `no_normalize_fmt_skip_whitespace`: whitespace before `# fmt: skip` comments is no longer normalized - `typed_params_trailing_comma`: consistently add trailing commas to typed function @@ -41,6 +39,8 @@ The unstable style additionally includes the following features: ([see below](labels/wrap-long-dict-values)) - `multiline_string_handling`: more compact formatting of expressions involving multiline strings ([see below](labels/multiline-string-handling)) +- `hug_parens_with_braces_and_square_brackets`: more compact formatting of nested + brackets ([see below](labels/hug-parens)) (labels/hug-parens)= diff --git a/src/black/mode.py b/src/black/mode.py index 22352e7c6a8..5738bd6b793 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -186,6 +186,8 @@ class Preview(Enum): Preview.wrap_long_dict_values_in_parens, # See issue #4159 Preview.multiline_string_handling, + # See issue #4036 (crash), #4098, #4099 (proposed tweaks) + Preview.hug_parens_with_braces_and_square_brackets, } diff --git a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py index 47a6a0bcae6..cbbcf16d3bd 100644 --- a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py +++ b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets.py @@ -1,4 +1,4 @@ -# flags: --preview +# flags: --unstable def foo_brackets(request): return JsonResponse( { diff --git a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets_no_ll1.py b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets_no_ll1.py index fdebdf69c20..16ebea379bc 100644 --- a/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets_no_ll1.py +++ b/tests/data/cases/preview_hug_parens_with_braces_and_square_brackets_no_ll1.py @@ -1,4 +1,4 @@ -# flags: --preview --no-preview-line-length-1 +# flags: --unstable --no-preview-line-length-1 # split out from preview_hug_parens_with_brackes_and_square_brackets, as it produces # different code on the second pass with line-length 1 in many cases. # Seems to be about whether the last string in a sequence gets wrapped in parens or not. From 632f44bd68b818cbc9dfd57e7485f0e5c3863b76 Mon Sep 17 00:00:00 2001 From: cobalt <61329810+RedGuy12@users.noreply.github.com> Date: Fri, 2 Feb 2024 00:00:41 -0600 Subject: [PATCH 269/279] docs: Refactor pycodestyle/Flake8 compatibility docs (#4194) Signed-off-by: RedGuy12 <61329810+RedGuy12@users.noreply.github.com> Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Co-authored-by: Jelle Zijlstra --- .flake8 | 4 +- docs/compatible_configs/flake8/.flake8 | 2 +- docs/compatible_configs/flake8/setup.cfg | 2 +- docs/compatible_configs/flake8/tox.ini | 2 +- docs/compatible_configs/pycodestyle/.flake8 | 3 + docs/compatible_configs/pycodestyle/setup.cfg | 3 + docs/compatible_configs/pycodestyle/tox.ini | 3 + docs/faq.md | 9 +- docs/guides/using_black_with_other_tools.md | 130 ++++++++++-------- docs/the_black_code_style/current_style.md | 33 +---- 10 files changed, 94 insertions(+), 97 deletions(-) create mode 100644 docs/compatible_configs/pycodestyle/.flake8 create mode 100644 docs/compatible_configs/pycodestyle/setup.cfg create mode 100644 docs/compatible_configs/pycodestyle/tox.ini diff --git a/.flake8 b/.flake8 index 85f51cf9f05..f8dca18e7cf 100644 --- a/.flake8 +++ b/.flake8 @@ -1,8 +1,8 @@ [flake8] # B905 should be enabled when we drop support for 3.9 -ignore = E203, E266, E501, E704, W503, B905, B907 +ignore = E203, E266, E501, E701, E704, W503, B905, B907 # line length is intentionally set to 80 here because black uses Bugbear -# See https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#line-length for more details +# See https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#bugbear for more details max-line-length = 80 max-complexity = 18 select = B,C,E,F,W,T4,B9 diff --git a/docs/compatible_configs/flake8/.flake8 b/docs/compatible_configs/flake8/.flake8 index 8dd399ab55b..0d4ade348d6 100644 --- a/docs/compatible_configs/flake8/.flake8 +++ b/docs/compatible_configs/flake8/.flake8 @@ -1,3 +1,3 @@ [flake8] max-line-length = 88 -extend-ignore = E203 +extend-ignore = E203,E701 diff --git a/docs/compatible_configs/flake8/setup.cfg b/docs/compatible_configs/flake8/setup.cfg index 8dd399ab55b..0d4ade348d6 100644 --- a/docs/compatible_configs/flake8/setup.cfg +++ b/docs/compatible_configs/flake8/setup.cfg @@ -1,3 +1,3 @@ [flake8] max-line-length = 88 -extend-ignore = E203 +extend-ignore = E203,E701 diff --git a/docs/compatible_configs/flake8/tox.ini b/docs/compatible_configs/flake8/tox.ini index 8dd399ab55b..0d4ade348d6 100644 --- a/docs/compatible_configs/flake8/tox.ini +++ b/docs/compatible_configs/flake8/tox.ini @@ -1,3 +1,3 @@ [flake8] max-line-length = 88 -extend-ignore = E203 +extend-ignore = E203,E701 diff --git a/docs/compatible_configs/pycodestyle/.flake8 b/docs/compatible_configs/pycodestyle/.flake8 new file mode 100644 index 00000000000..34225907524 --- /dev/null +++ b/docs/compatible_configs/pycodestyle/.flake8 @@ -0,0 +1,3 @@ +[pycodestyle] +max-line-length = 88 +ignore = E203,E701 diff --git a/docs/compatible_configs/pycodestyle/setup.cfg b/docs/compatible_configs/pycodestyle/setup.cfg new file mode 100644 index 00000000000..34225907524 --- /dev/null +++ b/docs/compatible_configs/pycodestyle/setup.cfg @@ -0,0 +1,3 @@ +[pycodestyle] +max-line-length = 88 +ignore = E203,E701 diff --git a/docs/compatible_configs/pycodestyle/tox.ini b/docs/compatible_configs/pycodestyle/tox.ini new file mode 100644 index 00000000000..34225907524 --- /dev/null +++ b/docs/compatible_configs/pycodestyle/tox.ini @@ -0,0 +1,3 @@ +[pycodestyle] +max-line-length = 88 +ignore = E203,E701 diff --git a/docs/faq.md b/docs/faq.md index 124a096efac..d19ff8e7ace 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -77,13 +77,10 @@ following will not be formatted: - invalid syntax, as it can't be safely distinguished from automagics in the absence of a running `IPython` kernel. -## Why are Flake8's E203 and W503 violated? +## Why does Flake8 report warnings? -Because they go against PEP 8. E203 falsely triggers on list -[slices](the_black_code_style/current_style.md#slices), and adhering to W503 hinders -readability because operators are misaligned. Disable W503 and enable the -disabled-by-default counterpart W504. E203 should be disabled while changes are still -[discussed](https://github.com/PyCQA/pycodestyle/issues/373). +Some of Flake8's rules conflict with Black's style. We recommend disabling these rules. +See [Using _Black_ with other tools](labels/why-pycodestyle-warnings). ## Which Python versions does Black support? diff --git a/docs/guides/using_black_with_other_tools.md b/docs/guides/using_black_with_other_tools.md index e642a1aef33..187e3a3e6f5 100644 --- a/docs/guides/using_black_with_other_tools.md +++ b/docs/guides/using_black_with_other_tools.md @@ -134,10 +134,10 @@ profile = black -### Flake8 +### pycodestyle -[Flake8](https://pypi.org/p/flake8/) is a code linter. It warns you of syntax errors, -possible bugs, stylistic errors, etc. For the most part, Flake8 follows +[pycodestyle](https://pycodestyle.pycqa.org/) is a code linter. It warns you of syntax +errors, possible bugs, stylistic errors, etc. For the most part, pycodestyle follows [PEP 8](https://www.python.org/dev/peps/pep-0008/) when warning about stylistic errors. There are a few deviations that cause incompatibilities with _Black_. @@ -145,67 +145,115 @@ There are a few deviations that cause incompatibilities with _Black_. ``` max-line-length = 88 -extend-ignore = E203, E704 +ignore = E203,E701 ``` +(labels/why-pycodestyle-warnings)= + #### Why those options above? +##### `max-line-length` + +As with isort, pycodestyle should be configured to allow lines up to the length limit of +`88`, _Black_'s default. + +##### `E203` + In some cases, as determined by PEP 8, _Black_ will enforce an equal amount of -whitespace around slice operators. Due to this, Flake8 will raise -`E203 whitespace before ':'` warnings. Since this warning is not PEP 8 compliant, Flake8 -should be configured to ignore it via `extend-ignore = E203`. +whitespace around slice operators. Due to this, pycodestyle will raise +`E203 whitespace before ':'` warnings. Since this warning is not PEP 8 compliant, it +should be disabled. + +##### `E701` / `E704` + +_Black_ will collapse implementations of classes and functions consisting solely of `..` +to a single line. This matches how such examples are formatted in PEP 8. It remains true +that in all other cases Black will prevent multiple statements on the same line, in +accordance with PEP 8 generally discouraging this. + +However, `pycodestyle` does not mirror this logic and may raise +`E701 multiple statements on one line (colon)` in this situation. Its +disabled-by-default `E704 multiple statements on one line (def)` rule may also raise +warnings and should not be enabled. + +##### `W503` When breaking a line, _Black_ will break it before a binary operator. This is compliant with PEP 8 as of [April 2016](https://github.com/python/peps/commit/c59c4376ad233a62ca4b3a6060c81368bd21e85b#diff-64ec08cc46db7540f18f2af46037f599). There's a disabled-by-default warning in Flake8 which goes against this PEP 8 recommendation called `W503 line break before binary operator`. It should not be enabled -in your configuration. - -Also, as like with isort, flake8 should be configured to allow lines up to the length -limit of `88`, _Black_'s default. This explains `max-line-length = 88`. +in your configuration. You can use its counterpart +`W504 line break after binary operator` instead. #### Formats
-.flake8 +setup.cfg, .pycodestyle, tox.ini ```ini -[flake8] +[pycodestyle] max-line-length = 88 -extend-ignore = E203, E704 +ignore = E203,E701 ```
-
-setup.cfg +### Flake8 -```ini +[Flake8](https://pypi.org/p/flake8/) is a wrapper around multiple linters, including +pycodestyle. As such, it has many of the same issues. + +#### Bugbear + +It's recommended to use [the Bugbear plugin](https://github.com/PyCQA/flake8-bugbear) +and enable +[its B950 check](https://github.com/PyCQA/flake8-bugbear#opinionated-warnings#:~:text=you%20expect%20it.-,B950,-%3A%20Line%20too%20long) +instead of using Flake8's E501, because it aligns with +[Black's 10% rule](labels/line-length). + +Install Bugbear and use the following config: + +``` +[flake8] +max-line-length = 80 +extend-select = B950 +extend-ignore = E203,E501,E701 +``` + +#### Minimal Configuration + +In cases where you can't or don't want to install Bugbear, you can use this minimally +compatible config: + +``` [flake8] max-line-length = 88 -extend-ignore = E203, E704 +extend-ignore = E203,E701 ``` -
+#### Why those options above? + +See [the pycodestyle section](labels/why-pycodestyle-warnings) above. + +#### Formats
-tox.ini +.flake8, setup.cfg, tox.ini ```ini [flake8] max-line-length = 88 -extend-ignore = E203, E704 +extend-ignore = E203,E701 ```
### Pylint -[Pylint](https://pypi.org/p/pylint/) is also a code linter like Flake8. It has the same -checks as flake8 and more. In particular, it has more formatting checks regarding style -conventions like variable naming. With so many checks, Pylint is bound to have some -mixed feelings about _Black_'s formatting style. +[Pylint](https://pypi.org/p/pylint/) is also a code linter like Flake8. It has many of +the same checks as Flake8 and more. It particularly has more formatting checks regarding +style conventions like variable naming. #### Configuration @@ -252,35 +300,3 @@ max-line-length = "88" ``` - -### pycodestyle - -[pycodestyle](https://pycodestyle.pycqa.org/) is also a code linter like Flake8. - -#### Configuration - -``` -max-line-length = 88 -ignore = E203 -``` - -#### Why those options above? - -pycodestyle should be configured to only complain about lines that surpass `88` -characters via `max_line_length = 88`. - -See -[Why are Flake8’s E203 and W503 violated?](https://black.readthedocs.io/en/stable/faq.html#why-are-flake8-s-e203-and-w503-violated) - -#### Formats - -
-setup.cfg - -```cfg -[pycodestyle] -ignore = E203 -max_line_length = 88 -``` - -
diff --git a/docs/the_black_code_style/current_style.md b/docs/the_black_code_style/current_style.md index ca5d1d4a701..586c79074af 100644 --- a/docs/the_black_code_style/current_style.md +++ b/docs/the_black_code_style/current_style.md @@ -143,7 +143,7 @@ significantly shorter files than sticking with 80 (the most popular), or even 79 by the standard library). In general, [90-ish seems like the wise choice](https://youtu.be/wf-BqAjZb8M?t=260). -If you're paid by the line of code you write, you can pass `--line-length` with a lower +If you're paid by the lines of code you write, you can pass `--line-length` with a lower number. _Black_ will try to respect that. However, sometimes it won't be able to without breaking other rules. In those rare cases, auto-formatted code will exceed your allotted limit. @@ -153,35 +153,10 @@ harder to work with line lengths exceeding 100 characters. It also adversely aff side-by-side diff review on typical screen resolutions. Long lines also make it harder to present code neatly in documentation or talk slides. -#### Flake8 +#### Flake8 and other linters -If you use Flake8, you have a few options: - -1. Recommended is using [Bugbear](https://github.com/PyCQA/flake8-bugbear) and enabling - its B950 check instead of using Flake8's E501, because it aligns with Black's 10% - rule. Install Bugbear and use the following config: - - ```ini - [flake8] - max-line-length = 80 - ... - select = C,E,F,W,B,B950 - extend-ignore = E203, E501, E704 - ``` - - The rationale for B950 is explained in - [Bugbear's documentation](https://github.com/PyCQA/flake8-bugbear#opinionated-warnings). - -2. For a minimally compatible config: - - ```ini - [flake8] - max-line-length = 88 - extend-ignore = E203, E704 - ``` - -An explanation of why E203 is disabled can be found in the [Slices section](#slices) of -this page. +See [Using _Black_ with other tools](../guides/using_black_with_other_tools.md) about +linter compatibility. ### Empty lines From a08b480a2f39fb4fc7b210d8b450e33d3879f77d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 01:09:21 -0800 Subject: [PATCH 270/279] Bump pypa/cibuildwheel from 2.16.4 to 2.16.5 (#4212) --- .github/workflows/pypi_upload.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml index 52525419f0a..3c99f66cc7b 100644 --- a/.github/workflows/pypi_upload.yml +++ b/.github/workflows/pypi_upload.yml @@ -89,7 +89,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: pypa/cibuildwheel@v2.16.4 + - uses: pypa/cibuildwheel@v2.16.5 with: only: ${{ matrix.only }} From 3e80de3447dee272e9977ab58b1560a669b7b848 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 04:47:35 -0800 Subject: [PATCH 271/279] Bump furo from 2023.9.10 to 2024.1.29 in /docs (#4211) Bumps [furo](https://github.com/pradyunsg/furo) from 2023.9.10 to 2024.1.29. - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2023.09.10...2024.01.29) --- updated-dependencies: - dependency-name: furo dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index b5b9e22fc84..3bc058a0721 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,4 +6,4 @@ Sphinx==7.2.6 docutils==0.20.1 sphinxcontrib-programoutput==0.17 sphinx_copybutton==0.5.2 -furo==2023.9.10 +furo==2024.1.29 From 7edb50f5a0afc56bb648dc14640ced144366b43a Mon Sep 17 00:00:00 2001 From: Brandon J <153339574+veryslowcode@users.noreply.github.com> Date: Mon, 5 Feb 2024 05:56:07 -0700 Subject: [PATCH 272/279] fix: additional newline added to docstring when the previous line length is less than the line length limit minus 1 (#4185) Co-authored-by: Jelle Zijlstra --- CHANGES.md | 2 ++ docs/the_black_code_style/future_style.md | 2 ++ src/black/linegen.py | 13 ++++++++++--- src/black/mode.py | 1 + src/black/resources/black.schema.json | 3 ++- tests/data/cases/docstring_newline_preview.py | 4 ++++ 6 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 tests/data/cases/docstring_newline_preview.py diff --git a/CHANGES.md b/CHANGES.md index 4d646a2779e..a726a91457a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,6 +16,8 @@ - Move the `hug_parens_with_braces_and_square_brackets` feature to the unstable style due to an outstanding crash and proposed formatting tweaks (#4198) +- Checking for newline before adding one on docstring that is almost at the line limit + (#4185) ### Configuration diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index 86e5aa806b2..d7640765b30 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -28,6 +28,8 @@ Currently, the following features are included in the preview style: longer normalized - `typed_params_trailing_comma`: consistently add trailing commas to typed function parameters +- `docstring_check_for_newline`: checks if there is a newline before the terminating + quotes of a docstring (labels/unstable-features)= diff --git a/src/black/linegen.py b/src/black/linegen.py index c74ff9c0b4b..c45a1308013 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -477,15 +477,22 @@ def visit_STRING(self, leaf: Leaf) -> Iterator[Line]: last_line_length = len(lines[-1]) if docstring else 0 # If adding closing quotes would cause the last line to exceed - # the maximum line length then put a line break before the - # closing quotes + # the maximum line length, and the closing quote is not + # prefixed by a newline then put a line break before + # the closing quotes if ( len(lines) > 1 and last_line_length + quote_len > self.mode.line_length and len(indent) + quote_len <= self.mode.line_length and not has_trailing_backslash ): - leaf.value = prefix + quote + docstring + "\n" + indent + quote + if ( + Preview.docstring_check_for_newline in self.mode + and leaf.value[-1 - quote_len] == "\n" + ): + leaf.value = prefix + quote + docstring + quote + else: + leaf.value = prefix + quote + docstring + "\n" + indent + quote else: leaf.value = prefix + quote + docstring + quote else: diff --git a/src/black/mode.py b/src/black/mode.py index 5738bd6b793..9593a90d170 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -177,6 +177,7 @@ class Preview(Enum): wrap_long_dict_values_in_parens = auto() multiline_string_handling = auto() typed_params_trailing_comma = auto() + docstring_check_for_newline = auto() UNSTABLE_FEATURES: Set[Preview] = { diff --git a/src/black/resources/black.schema.json b/src/black/resources/black.schema.json index 40bf39137f7..5b1b1320eb4 100644 --- a/src/black/resources/black.schema.json +++ b/src/black/resources/black.schema.json @@ -85,7 +85,8 @@ "no_normalize_fmt_skip_whitespace", "wrap_long_dict_values_in_parens", "multiline_string_handling", - "typed_params_trailing_comma" + "typed_params_trailing_comma", + "docstring_check_for_newline" ] }, "description": "Enable specific features included in the `--unstable` style. Requires `--preview`. No compatibility guarantees are provided on the behavior or existence of any unstable features." diff --git a/tests/data/cases/docstring_newline_preview.py b/tests/data/cases/docstring_newline_preview.py new file mode 100644 index 00000000000..5c129ca5f80 --- /dev/null +++ b/tests/data/cases/docstring_newline_preview.py @@ -0,0 +1,4 @@ +# flags: --preview +""" +87 characters ............................................................................ +""" From 32230e6f5c4bc6bb5c469451e15f3f54d9884b51 Mon Sep 17 00:00:00 2001 From: Seung Wan Yoo <74849806+wannieman98@users.noreply.github.com> Date: Mon, 5 Feb 2024 22:33:11 +0900 Subject: [PATCH 273/279] fix: bug where the doublestar operation had inconsistent formatting. (#4154) Co-authored-by: Jelle Zijlstra --- CHANGES.md | 2 + docs/the_black_code_style/future_style.md | 2 + src/black/mode.py | 1 + src/black/resources/black.schema.json | 1 + src/black/trans.py | 138 ++++++++++++++---- ...simple_lookup_for_doublestar_expression.py | 14 ++ 6 files changed, 132 insertions(+), 26 deletions(-) create mode 100644 tests/data/cases/is_simple_lookup_for_doublestar_expression.py diff --git a/CHANGES.md b/CHANGES.md index a726a91457a..5d4858da811 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,6 +16,8 @@ - Move the `hug_parens_with_braces_and_square_brackets` feature to the unstable style due to an outstanding crash and proposed formatting tweaks (#4198) +- Fixed a bug where base expressions caused inconsistent formatting of \*\* in tenary + expression (#4154) - Checking for newline before adding one on docstring that is almost at the line limit (#4185) diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index d7640765b30..f4534680645 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -28,6 +28,8 @@ Currently, the following features are included in the preview style: longer normalized - `typed_params_trailing_comma`: consistently add trailing commas to typed function parameters +- `is_simple_lookup_for_doublestar_expression`: fix line length computation for certain + expressions that involve the power operator - `docstring_check_for_newline`: checks if there is a newline before the terminating quotes of a docstring diff --git a/src/black/mode.py b/src/black/mode.py index 9593a90d170..f9fad082e03 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -177,6 +177,7 @@ class Preview(Enum): wrap_long_dict_values_in_parens = auto() multiline_string_handling = auto() typed_params_trailing_comma = auto() + is_simple_lookup_for_doublestar_expression = auto() docstring_check_for_newline = auto() diff --git a/src/black/resources/black.schema.json b/src/black/resources/black.schema.json index 5b1b1320eb4..04ddb32e087 100644 --- a/src/black/resources/black.schema.json +++ b/src/black/resources/black.schema.json @@ -86,6 +86,7 @@ "wrap_long_dict_values_in_parens", "multiline_string_handling", "typed_params_trailing_comma", + "is_simple_lookup_for_doublestar_expression", "docstring_check_for_newline" ] }, diff --git a/src/black/trans.py b/src/black/trans.py index 7c7335a005b..29a978c6b71 100644 --- a/src/black/trans.py +++ b/src/black/trans.py @@ -29,7 +29,7 @@ from black.comments import contains_pragma_comment from black.lines import Line, append_leaves -from black.mode import Feature, Mode +from black.mode import Feature, Mode, Preview from black.nodes import ( CLOSING_BRACKETS, OPENING_BRACKETS, @@ -94,43 +94,36 @@ def hug_power_op( else: raise CannotTransform("No doublestar token was found in the line.") - def is_simple_lookup(index: int, step: Literal[1, -1]) -> bool: + def is_simple_lookup(index: int, kind: Literal[1, -1]) -> bool: # Brackets and parentheses indicate calls, subscripts, etc. ... # basically stuff that doesn't count as "simple". Only a NAME lookup # or dotted lookup (eg. NAME.NAME) is OK. - if step == -1: - disallowed = {token.RPAR, token.RSQB} - else: - disallowed = {token.LPAR, token.LSQB} - - while 0 <= index < len(line.leaves): - current = line.leaves[index] - if current.type in disallowed: - return False - if current.type not in {token.NAME, token.DOT} or current.value == "for": - # If the current token isn't disallowed, we'll assume this is simple as - # only the disallowed tokens are semantically attached to this lookup - # expression we're checking. Also, stop early if we hit the 'for' bit - # of a comprehension. - return True + if Preview.is_simple_lookup_for_doublestar_expression not in mode: + return original_is_simple_lookup_func(line, index, kind) - index += step - - return True + else: + if kind == -1: + return handle_is_simple_look_up_prev( + line, index, {token.RPAR, token.RSQB} + ) + else: + return handle_is_simple_lookup_forward( + line, index, {token.LPAR, token.LSQB} + ) - def is_simple_operand(index: int, kind: Literal["base", "exponent"]) -> bool: + def is_simple_operand(index: int, kind: Literal[1, -1]) -> bool: # An operand is considered "simple" if's a NAME, a numeric CONSTANT, a simple # lookup (see above), with or without a preceding unary operator. start = line.leaves[index] if start.type in {token.NAME, token.NUMBER}: - return is_simple_lookup(index, step=(1 if kind == "exponent" else -1)) + return is_simple_lookup(index, kind) if start.type in {token.PLUS, token.MINUS, token.TILDE}: if line.leaves[index + 1].type in {token.NAME, token.NUMBER}: - # step is always one as bases with a preceding unary op will be checked + # kind is always one as bases with a preceding unary op will be checked # for simplicity starting from the next token (so it'll hit the check # above). - return is_simple_lookup(index + 1, step=1) + return is_simple_lookup(index + 1, kind=1) return False @@ -145,9 +138,9 @@ def is_simple_operand(index: int, kind: Literal["base", "exponent"]) -> bool: should_hug = ( (0 < idx < len(line.leaves) - 1) and leaf.type == token.DOUBLESTAR - and is_simple_operand(idx - 1, kind="base") + and is_simple_operand(idx - 1, kind=-1) and line.leaves[idx - 1].value != "lambda" - and is_simple_operand(idx + 1, kind="exponent") + and is_simple_operand(idx + 1, kind=1) ) if should_hug: new_leaf.prefix = "" @@ -162,6 +155,99 @@ def is_simple_operand(index: int, kind: Literal["base", "exponent"]) -> bool: yield new_line +def original_is_simple_lookup_func( + line: Line, index: int, step: Literal[1, -1] +) -> bool: + if step == -1: + disallowed = {token.RPAR, token.RSQB} + else: + disallowed = {token.LPAR, token.LSQB} + + while 0 <= index < len(line.leaves): + current = line.leaves[index] + if current.type in disallowed: + return False + if current.type not in {token.NAME, token.DOT} or current.value == "for": + # If the current token isn't disallowed, we'll assume this is + # simple as only the disallowed tokens are semantically + # attached to this lookup expression we're checking. Also, + # stop early if we hit the 'for' bit of a comprehension. + return True + + index += step + + return True + + +def handle_is_simple_look_up_prev(line: Line, index: int, disallowed: Set[int]) -> bool: + """ + Handling the determination of is_simple_lookup for the lines prior to the doublestar + token. This is required because of the need to isolate the chained expression + to determine the bracket or parenthesis belong to the single expression. + """ + contains_disallowed = False + chain = [] + + while 0 <= index < len(line.leaves): + current = line.leaves[index] + chain.append(current) + if not contains_disallowed and current.type in disallowed: + contains_disallowed = True + if not is_expression_chained(chain): + return not contains_disallowed + + index -= 1 + + return True + + +def handle_is_simple_lookup_forward( + line: Line, index: int, disallowed: Set[int] +) -> bool: + """ + Handling decision is_simple_lookup for the lines behind the doublestar token. + This function is simplified to keep consistent with the prior logic and the forward + case are more straightforward and do not need to care about chained expressions. + """ + while 0 <= index < len(line.leaves): + current = line.leaves[index] + if current.type in disallowed: + return False + if current.type not in {token.NAME, token.DOT} or ( + current.type == token.NAME and current.value == "for" + ): + # If the current token isn't disallowed, we'll assume this is simple as + # only the disallowed tokens are semantically attached to this lookup + # expression we're checking. Also, stop early if we hit the 'for' bit + # of a comprehension. + return True + + index += 1 + + return True + + +def is_expression_chained(chained_leaves: List[Leaf]) -> bool: + """ + Function to determine if the variable is a chained call. + (e.g., foo.lookup, foo().lookup, (foo.lookup())) will be recognized as chained call) + """ + if len(chained_leaves) < 2: + return True + + current_leaf = chained_leaves[-1] + past_leaf = chained_leaves[-2] + + if past_leaf.type == token.NAME: + return current_leaf.type in {token.DOT} + elif past_leaf.type in {token.RPAR, token.RSQB}: + return current_leaf.type in {token.RSQB, token.RPAR} + elif past_leaf.type in {token.LPAR, token.LSQB}: + return current_leaf.type in {token.NAME, token.LPAR, token.LSQB} + else: + return False + + class StringTransformer(ABC): """ An implementation of the Transformer protocol that relies on its diff --git a/tests/data/cases/is_simple_lookup_for_doublestar_expression.py b/tests/data/cases/is_simple_lookup_for_doublestar_expression.py new file mode 100644 index 00000000000..a0d2e2ba842 --- /dev/null +++ b/tests/data/cases/is_simple_lookup_for_doublestar_expression.py @@ -0,0 +1,14 @@ +# flags: --preview +m2 = None if not isinstance(dist, Normal) else m** 2 + s * 2 +m3 = None if not isinstance(dist, Normal) else m ** 2 + s * 2 +m4 = None if not isinstance(dist, Normal) else m**2 + s * 2 +m5 = obj.method(another_obj.method()).attribute **2 +m6 = None if ... else m**2 + s**2 + + +# output +m2 = None if not isinstance(dist, Normal) else m**2 + s * 2 +m3 = None if not isinstance(dist, Normal) else m**2 + s * 2 +m4 = None if not isinstance(dist, Normal) else m**2 + s * 2 +m5 = obj.method(another_obj.method()).attribute ** 2 +m6 = None if ... else m**2 + s**2 \ No newline at end of file From dab37a6a1117d690d683121edc4d7c8fb8dd75a7 Mon Sep 17 00:00:00 2001 From: Logan Hunt <39638017+dosisod@users.noreply.github.com> Date: Wed, 7 Feb 2024 06:55:02 -0800 Subject: [PATCH 274/279] Remove redundant parentheses in `case` statement `if` guards (#4214) A follow up to #4024 but for `if` guards in `case` statements. I noticed this when #4024 was made stable, and noticed I had some code that had extra parens around the `if` guard. --- CHANGES.md | 1 + docs/the_black_code_style/future_style.md | 2 + src/black/linegen.py | 2 + src/black/mode.py | 1 + src/black/resources/black.schema.json | 3 +- .../remove_redundant_parens_in_case_guard.py | 114 ++++++++++++++++++ 6 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 tests/data/cases/remove_redundant_parens_in_case_guard.py diff --git a/CHANGES.md b/CHANGES.md index 5d4858da811..3583348f377 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -20,6 +20,7 @@ expression (#4154) - Checking for newline before adding one on docstring that is almost at the line limit (#4185) +- Remove redundant parentheses in `case` statement `if` guards (#4214). ### Configuration diff --git a/docs/the_black_code_style/future_style.md b/docs/the_black_code_style/future_style.md index f4534680645..4ae46cecded 100644 --- a/docs/the_black_code_style/future_style.md +++ b/docs/the_black_code_style/future_style.md @@ -32,6 +32,8 @@ Currently, the following features are included in the preview style: expressions that involve the power operator - `docstring_check_for_newline`: checks if there is a newline before the terminating quotes of a docstring +- `remove_redundant_guard_parens`: Removes redundant parentheses in `if` guards for + `case` blocks. (labels/unstable-features)= diff --git a/src/black/linegen.py b/src/black/linegen.py index c45a1308013..2b7fbc492cf 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -529,6 +529,8 @@ def __post_init__(self) -> None: # PEP 634 self.visit_match_stmt = self.visit_match_case self.visit_case_block = self.visit_match_case + if Preview.remove_redundant_guard_parens in self.mode: + self.visit_guard = partial(v, keywords=Ø, parens={"if"}) def _hugging_power_ops_line_to_string( diff --git a/src/black/mode.py b/src/black/mode.py index f9fad082e03..90c10c324a5 100644 --- a/src/black/mode.py +++ b/src/black/mode.py @@ -179,6 +179,7 @@ class Preview(Enum): typed_params_trailing_comma = auto() is_simple_lookup_for_doublestar_expression = auto() docstring_check_for_newline = auto() + remove_redundant_guard_parens = auto() UNSTABLE_FEATURES: Set[Preview] = { diff --git a/src/black/resources/black.schema.json b/src/black/resources/black.schema.json index 04ddb32e087..8252a6c4bd8 100644 --- a/src/black/resources/black.schema.json +++ b/src/black/resources/black.schema.json @@ -87,7 +87,8 @@ "multiline_string_handling", "typed_params_trailing_comma", "is_simple_lookup_for_doublestar_expression", - "docstring_check_for_newline" + "docstring_check_for_newline", + "remove_redundant_guard_parens" ] }, "description": "Enable specific features included in the `--unstable` style. Requires `--preview`. No compatibility guarantees are provided on the behavior or existence of any unstable features." diff --git a/tests/data/cases/remove_redundant_parens_in_case_guard.py b/tests/data/cases/remove_redundant_parens_in_case_guard.py new file mode 100644 index 00000000000..bec4a3c3fcd --- /dev/null +++ b/tests/data/cases/remove_redundant_parens_in_case_guard.py @@ -0,0 +1,114 @@ +# flags: --minimum-version=3.10 --preview --line-length=79 + +match 1: + case _ if (True): + pass + + +match 1: + case _ if ( + True + ): + pass + + +match 1: + case _ if ( + # this is a comment + True + ): + pass + + +match 1: + case _ if ( + True + # this is a comment + ): + pass + + +match 1: + case _ if ( + True # this is a comment + ): + pass + + +match 1: + case _ if ( # this is a comment + True + ): + pass + + +match 1: + case _ if ( + True + ): # this is a comment + pass + + +match 1: + case _ if (True): # comment over the line limit unless parens are removed x + pass + + +match 1: + case _ if (True): # comment over the line limit and parens should go to next line + pass + + +# output + +match 1: + case _ if True: + pass + + +match 1: + case _ if True: + pass + + +match 1: + case _ if ( + # this is a comment + True + ): + pass + + +match 1: + case _ if ( + True + # this is a comment + ): + pass + + +match 1: + case _ if True: # this is a comment + pass + + +match 1: + case _ if True: # this is a comment + pass + + +match 1: + case _ if True: # this is a comment + pass + + +match 1: + case _ if True: # comment over the line limit unless parens are removed x + pass + + +match 1: + case ( + _ + ) if True: # comment over the line limit and parens should go to next line + pass From a20100395cf6179a81289452efad1d8e72b19682 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Sat, 10 Feb 2024 23:55:01 -0800 Subject: [PATCH 275/279] Simplify check for symlinks that resolve outside root (#4221) This PR does not change any behaviour. There have been 1-2 issues about symlinks recently. Both over and under resolving can cause problems. This makes a case where we resolve more explicit and prevent a resolved path from leaking out via the return. --- src/black/__init__.py | 11 ++++------- src/black/files.py | 25 +++++++++++-------------- tests/test_black.py | 11 +++++++---- 3 files changed, 22 insertions(+), 25 deletions(-) diff --git a/src/black/__init__.py b/src/black/__init__.py index 8ab5b47f974..2d4c7f655ad 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -50,9 +50,9 @@ gen_python_files, get_gitignore, get_root_relative_path, - normalize_path_maybe_ignore, parse_pyproject_toml, path_is_excluded, + resolves_outside_root_or_cannot_stat, wrap_stream_for_windows, ) from black.handle_ipynb_magics import ( @@ -763,12 +763,9 @@ def get_sources( ) continue - normalized_path: Optional[str] = normalize_path_maybe_ignore( - path, root, report - ) - if normalized_path is None: + if resolves_outside_root_or_cannot_stat(path, root, report): if verbose: - out(f'Skipping invalid source: "{normalized_path}"', fg="red") + out(f'Skipping invalid source: "{path}"', fg="red") continue if is_stdin: @@ -780,7 +777,7 @@ def get_sources( continue if verbose: - out(f'Found input source: "{normalized_path}"', fg="blue") + out(f'Found input source: "{path}"', fg="blue") sources.add(path) elif path.is_dir(): path = root / (path.resolve().relative_to(root)) diff --git a/src/black/files.py b/src/black/files.py index 960f13ee270..6c05105450c 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -254,26 +254,24 @@ def get_gitignore(root: Path) -> PathSpec: raise -def normalize_path_maybe_ignore( +def resolves_outside_root_or_cannot_stat( path: Path, root: Path, report: Optional[Report] = None, -) -> Optional[str]: - """Normalize `path`. May return `None` if `path` was ignored. - - `report` is where "path ignored" output goes. +) -> bool: + """ + Returns whether the path is a symbolic link that points outside the + root directory. Also returns True if we failed to resolve the path. """ try: - abspath = path if path.is_absolute() else Path.cwd() / path - normalized_path = abspath.resolve() - root_relative_path = get_root_relative_path(normalized_path, root, report) - + if sys.version_info < (3, 8, 6): + path = path.absolute() # https://bugs.python.org/issue33660 + resolved_path = path.resolve() + return get_root_relative_path(resolved_path, root, report) is None except OSError as e: if report: report.path_ignored(path, f"cannot be read because {e}") - return None - - return root_relative_path + return True def get_root_relative_path( @@ -369,8 +367,7 @@ def gen_python_files( report.path_ignored(child, "matches the --force-exclude regular expression") continue - normalized_path = normalize_path_maybe_ignore(child, root, report) - if normalized_path is None: + if resolves_outside_root_or_cannot_stat(child, root, report): continue if child.is_dir(): diff --git a/tests/test_black.py b/tests/test_black.py index f876d365b12..5c6920c2b77 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1760,12 +1760,15 @@ def test_bpo_33660_workaround(self) -> None: return # https://bugs.python.org/issue33660 + # Can be removed when we drop support for Python 3.8.5 root = Path("/") with change_directory(root): path = Path("workspace") / "project" report = black.Report(verbose=True) - normalized_path = black.normalize_path_maybe_ignore(path, root, report) - self.assertEqual(normalized_path, "workspace/project") + resolves_outside = black.resolves_outside_root_or_cannot_stat( + path, root, report + ) + self.assertIs(resolves_outside, False) def test_normalize_path_ignore_windows_junctions_outside_of_root(self) -> None: if system() != "Windows": @@ -1778,13 +1781,13 @@ def test_normalize_path_ignore_windows_junctions_outside_of_root(self) -> None: os.system(f"mklink /J {junction_dir} {junction_target_outside_of_root}") report = black.Report(verbose=True) - normalized_path = black.normalize_path_maybe_ignore( + resolves_outside = black.resolves_outside_root_or_cannot_stat( junction_dir, root, report ) # Manually delete for Python < 3.8 os.system(f"rmdir {junction_dir}") - self.assertEqual(normalized_path, None) + self.assertIs(resolves_outside, True) def test_newline_comment_interaction(self) -> None: source = "class A:\\\r\n# type: ignore\n pass\n" From 23dfc5b2c3b0694a8c27e58e28439591976aaf94 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 12 Feb 2024 00:04:09 -0800 Subject: [PATCH 276/279] Fix ignoring input files for symlink reasons (#4222) This relates to #4015, #4161 and the behaviour of os.getcwd() Black is a big user of pathlib and as such loves doing `.resolve()`, since for a long time it was the only good way of getting an absolute path in pathlib. However, this has two problems: The first minor problem is performance, e.g. in #3751 I (safely) got rid of a bunch of `.resolve()` which made Black 40% faster on cached runs. The second more important problem is that always resolving symlinks results in unintuitive exclusion behaviour. For instance, a gitignored symlink should never alter formatting of your actual code. This kind of thing was reported by users a few times. In #3846, I improved the exclusion rule logic for symlinks in `gen_python_files` and everything was good. But `gen_python_files` isn't enough, there's also `get_sources`, which handles user specified paths directly (instead of files Black discovers). So in #4015, I made a very similar change to #3846 for `get_sources`, and this is where some problems began. The core issue was the line: ``` root_relative_path = path.absolute().relative_to(root).as_posix() ``` The first issue is that despite root being computed from user inputs, we call `.resolve()` while computing it (likely unecessarily). Which means that `path` may not actually be relative to `root`. So I started off this PR trying to fix that, when I ran into the second issue. Which is that `os.getcwd()` (as called by `os.path.abspath` or `Path.absolute` or `Path.cwd`) also often resolves symlinks! ``` >>> import os >>> os.environ.get("PWD") '/Users/shantanu/dev/black/symlink/bug' >>> os.getcwd() '/Users/shantanu/dev/black/actual/bug' ``` This also meant that the breakage often would not show up when input relative paths. This doesn't affect `gen_python_files` / #3846 because things are always absolute and known to be relative to `root`. Anyway, it looks like #4161 fixed the crash by just swallowing the error and ignoring the file. Instead, we should just try to compute the actual relative path. I think this PR should be quite safe, but we could also consider reverting some of the previous changes; the associated issues weren't too popular. At the same time, I think there's still behaviour that can be improved and I kind of want to make larger changes, but maybe I'll save that for if we do something like #3952 Hopefully fixes #4205, fixes #4209, actual fix for #4077 --- CHANGES.md | 1 + src/black/__init__.py | 15 +++--- src/black/files.py | 44 ++++++++++------ tests/test_black.py | 120 ++++++++++++++++++++++++++++++++---------- 4 files changed, 127 insertions(+), 53 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3583348f377..4fd63ac41eb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -24,6 +24,7 @@ ### Configuration +- Fix issue where _Black_ would ignore input files in the presence of symlinks (#4222) - _Black_ now ignores `pyproject.toml` that is missing a `tool.black` section when discovering project root and configuration. Since _Black_ continues to use version control as an indicator of project root, this is expected to primarily change behavior diff --git a/src/black/__init__.py b/src/black/__init__.py index 2d4c7f655ad..f82b9fec5b7 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -44,12 +44,12 @@ STDIN_PLACEHOLDER, ) from black.files import ( + best_effort_relative_path, find_project_root, find_pyproject_toml, find_user_pyproject_toml, gen_python_files, get_gitignore, - get_root_relative_path, parse_pyproject_toml, path_is_excluded, resolves_outside_root_or_cannot_stat, @@ -734,6 +734,7 @@ def get_sources( """Compute the set of files to be formatted.""" sources: Set[Path] = set() + assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}" using_default_exclude = exclude is None exclude = re_compile_maybe_verbose(DEFAULT_EXCLUDES) if exclude is None else exclude gitignore: Optional[Dict[Path, PathSpec]] = None @@ -749,11 +750,12 @@ def get_sources( # Compare the logic here to the logic in `gen_python_files`. if is_stdin or path.is_file(): - root_relative_path = get_root_relative_path(path, root, report) - - if root_relative_path is None: + if resolves_outside_root_or_cannot_stat(path, root, report): + if verbose: + out(f'Skipping invalid source: "{path}"', fg="red") continue + root_relative_path = best_effort_relative_path(path, root).as_posix() root_relative_path = "/" + root_relative_path # Hard-exclude any files that matches the `--force-exclude` regex. @@ -763,11 +765,6 @@ def get_sources( ) continue - if resolves_outside_root_or_cannot_stat(path, root, report): - if verbose: - out(f'Skipping invalid source: "{path}"', fg="red") - continue - if is_stdin: path = Path(f"{STDIN_PLACEHOLDER}{str(path)}") diff --git a/src/black/files.py b/src/black/files.py index 6c05105450c..c0cadbfd890 100644 --- a/src/black/files.py +++ b/src/black/files.py @@ -48,6 +48,11 @@ def _load_toml(path: Union[Path, str]) -> Dict[str, Any]: return tomllib.load(f) +@lru_cache +def _cached_resolve(path: Path) -> Path: + return path.resolve() + + @lru_cache def find_project_root( srcs: Sequence[str], stdin_filename: Optional[str] = None @@ -67,9 +72,9 @@ def find_project_root( if stdin_filename is not None: srcs = tuple(stdin_filename if s == "-" else s for s in srcs) if not srcs: - srcs = [str(Path.cwd().resolve())] + srcs = [str(_cached_resolve(Path.cwd()))] - path_srcs = [Path(Path.cwd(), src).resolve() for src in srcs] + path_srcs = [_cached_resolve(Path(Path.cwd(), src)) for src in srcs] # A list of lists of parents for each 'src'. 'src' is included as a # "parent" of itself if it is a directory @@ -236,7 +241,7 @@ def find_user_pyproject_toml() -> Path: else: config_root = os.environ.get("XDG_CONFIG_HOME", "~/.config") user_config_path = Path(config_root).expanduser() / "black" - return user_config_path.resolve() + return _cached_resolve(user_config_path) @lru_cache @@ -266,27 +271,31 @@ def resolves_outside_root_or_cannot_stat( try: if sys.version_info < (3, 8, 6): path = path.absolute() # https://bugs.python.org/issue33660 - resolved_path = path.resolve() - return get_root_relative_path(resolved_path, root, report) is None + resolved_path = _cached_resolve(path) except OSError as e: if report: report.path_ignored(path, f"cannot be read because {e}") return True - - -def get_root_relative_path( - path: Path, - root: Path, - report: Optional[Report] = None, -) -> Optional[str]: - """Returns the file path relative to the 'root' directory""" try: - root_relative_path = path.absolute().relative_to(root).as_posix() + resolved_path.relative_to(root) except ValueError: if report: report.path_ignored(path, f"is a symbolic link that points outside {root}") - return None - return root_relative_path + return True + return False + + +def best_effort_relative_path(path: Path, root: Path) -> Path: + # Precondition: resolves_outside_root_or_cannot_stat(path, root) is False + try: + return path.absolute().relative_to(root) + except ValueError: + pass + root_parent = next((p for p in path.parents if _cached_resolve(p) == root), None) + if root_parent is not None: + return path.relative_to(root_parent) + # something adversarial, fallback to path guaranteed by precondition + return _cached_resolve(path).relative_to(root) def _path_is_ignored( @@ -339,7 +348,8 @@ def gen_python_files( assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}" for child in paths: - root_relative_path = child.absolute().relative_to(root).as_posix() + assert child.is_absolute() + root_relative_path = child.relative_to(root).as_posix() # First ignore files matching .gitignore, if passed if gitignore_dict and _path_is_ignored( diff --git a/tests/test_black.py b/tests/test_black.py index 5c6920c2b77..41f87cd16f8 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -2567,32 +2567,32 @@ def test_symlinks(self) -> None: gitignore = PathSpec.from_lines("gitwildmatch", []) regular = MagicMock() - regular.absolute.return_value = root / "regular.py" + regular.relative_to.return_value = Path("regular.py") regular.resolve.return_value = root / "regular.py" regular.is_dir.return_value = False regular.is_file.return_value = True outside_root_symlink = MagicMock() - outside_root_symlink.absolute.return_value = root / "symlink.py" + outside_root_symlink.relative_to.return_value = Path("symlink.py") outside_root_symlink.resolve.return_value = Path("/nowhere") outside_root_symlink.is_dir.return_value = False outside_root_symlink.is_file.return_value = True ignored_symlink = MagicMock() - ignored_symlink.absolute.return_value = root / ".mypy_cache" / "symlink.py" + ignored_symlink.relative_to.return_value = Path(".mypy_cache") / "symlink.py" ignored_symlink.is_dir.return_value = False ignored_symlink.is_file.return_value = True # A symlink that has an excluded name, but points to an included name symlink_excluded_name = MagicMock() - symlink_excluded_name.absolute.return_value = root / "excluded_name" + symlink_excluded_name.relative_to.return_value = Path("excluded_name") symlink_excluded_name.resolve.return_value = root / "included_name.py" symlink_excluded_name.is_dir.return_value = False symlink_excluded_name.is_file.return_value = True # A symlink that has an included name, but points to an excluded name symlink_included_name = MagicMock() - symlink_included_name.absolute.return_value = root / "included_name.py" + symlink_included_name.relative_to.return_value = Path("included_name.py") symlink_included_name.resolve.return_value = root / "excluded_name" symlink_included_name.is_dir.return_value = False symlink_included_name.is_file.return_value = True @@ -2626,39 +2626,100 @@ def test_symlinks(self) -> None: outside_root_symlink.resolve.assert_called_once() ignored_symlink.resolve.assert_not_called() + def test_get_sources_symlink_and_force_exclude(self) -> None: + with TemporaryDirectory() as tempdir: + tmp = Path(tempdir).resolve() + actual = tmp / "actual" + actual.mkdir() + symlink = tmp / "symlink" + symlink.symlink_to(actual) + + actual_proj = actual / "project" + actual_proj.mkdir() + (actual_proj / "module.py").write_text("print('hello')", encoding="utf-8") + + symlink_proj = symlink / "project" + + with change_directory(symlink_proj): + assert_collected_sources( + src=["module.py"], + root=symlink_proj.resolve(), + expected=["module.py"], + ) + + absolute_module = symlink_proj / "module.py" + assert_collected_sources( + src=[absolute_module], + root=symlink_proj.resolve(), + expected=[absolute_module], + ) + + # a few tricky tests for force_exclude + flat_symlink = symlink_proj / "symlink_module.py" + flat_symlink.symlink_to(actual_proj / "module.py") + assert_collected_sources( + src=[flat_symlink], + root=symlink_proj.resolve(), + force_exclude=r"/symlink_module.py", + expected=[], + ) + + target = actual_proj / "target" + target.mkdir() + (target / "another.py").write_text("print('hello')", encoding="utf-8") + (symlink_proj / "nested").symlink_to(target) + + assert_collected_sources( + src=[symlink_proj / "nested" / "another.py"], + root=symlink_proj.resolve(), + force_exclude=r"nested", + expected=[], + ) + assert_collected_sources( + src=[symlink_proj / "nested" / "another.py"], + root=symlink_proj.resolve(), + force_exclude=r"target", + expected=[symlink_proj / "nested" / "another.py"], + ) + def test_get_sources_with_stdin_symlink_outside_root( self, ) -> None: path = THIS_DIR / "data" / "include_exclude_tests" stdin_filename = str(path / "b/exclude/a.py") outside_root_symlink = Path("/target_directory/a.py") + root = Path("target_dir/").resolve().absolute() with patch("pathlib.Path.resolve", return_value=outside_root_symlink): assert_collected_sources( - root=Path("target_directory/"), + root=root, src=["-"], expected=[], stdin_filename=stdin_filename, ) - @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None)) def test_get_sources_with_stdin(self) -> None: src = ["-"] expected = ["-"] - assert_collected_sources(src, expected, include="", exclude=r"/exclude/|a\.py") + assert_collected_sources( + src, + root=THIS_DIR.resolve(), + expected=expected, + include="", + exclude=r"/exclude/|a\.py", + ) - @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None)) def test_get_sources_with_stdin_filename(self) -> None: src = ["-"] stdin_filename = str(THIS_DIR / "data/collections.py") expected = [f"__BLACK_STDIN_FILENAME__{stdin_filename}"] assert_collected_sources( src, - expected, + root=THIS_DIR.resolve(), + expected=expected, exclude=r"/exclude/a\.py", stdin_filename=stdin_filename, ) - @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None)) def test_get_sources_with_stdin_filename_and_exclude(self) -> None: # Exclude shouldn't exclude stdin_filename since it is mimicking the # file being passed directly. This is the same as @@ -2669,12 +2730,12 @@ def test_get_sources_with_stdin_filename_and_exclude(self) -> None: expected = [f"__BLACK_STDIN_FILENAME__{stdin_filename}"] assert_collected_sources( src, - expected, + root=THIS_DIR.resolve(), + expected=expected, exclude=r"/exclude/|a\.py", stdin_filename=stdin_filename, ) - @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None)) def test_get_sources_with_stdin_filename_and_extend_exclude(self) -> None: # Extend exclude shouldn't exclude stdin_filename since it is mimicking the # file being passed directly. This is the same as @@ -2685,12 +2746,12 @@ def test_get_sources_with_stdin_filename_and_extend_exclude(self) -> None: expected = [f"__BLACK_STDIN_FILENAME__{stdin_filename}"] assert_collected_sources( src, - expected, + root=THIS_DIR.resolve(), + expected=expected, extend_exclude=r"/exclude/|a\.py", stdin_filename=stdin_filename, ) - @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None)) def test_get_sources_with_stdin_filename_and_force_exclude(self) -> None: # Force exclude should exclude the file when passing it through # stdin_filename @@ -2698,27 +2759,32 @@ def test_get_sources_with_stdin_filename_and_force_exclude(self) -> None: stdin_filename = str(path / "b/exclude/a.py") assert_collected_sources( src=["-"], + root=THIS_DIR.resolve(), expected=[], force_exclude=r"/exclude/|a\.py", stdin_filename=stdin_filename, ) - @patch("black.find_project_root", lambda *args: (THIS_DIR.resolve(), None)) def test_get_sources_with_stdin_filename_and_force_exclude_and_symlink( self, ) -> None: # Force exclude should exclude a symlink based on the symlink, not its target - path = THIS_DIR / "data" / "include_exclude_tests" - stdin_filename = str(path / "symlink.py") - expected = [f"__BLACK_STDIN_FILENAME__{stdin_filename}"] - target = path / "b/exclude/a.py" - with patch("pathlib.Path.resolve", return_value=target): - assert_collected_sources( - src=["-"], - expected=expected, - force_exclude=r"exclude/a\.py", - stdin_filename=stdin_filename, - ) + with TemporaryDirectory() as tempdir: + tmp = Path(tempdir).resolve() + (tmp / "exclude").mkdir() + (tmp / "exclude" / "a.py").write_text("print('hello')", encoding="utf-8") + (tmp / "symlink.py").symlink_to(tmp / "exclude" / "a.py") + + stdin_filename = str(tmp / "symlink.py") + expected = [f"__BLACK_STDIN_FILENAME__{stdin_filename}"] + with change_directory(tmp): + assert_collected_sources( + src=["-"], + root=tmp, + expected=expected, + force_exclude=r"exclude/a\.py", + stdin_filename=stdin_filename, + ) class TestDeFactoAPI: From 35e97769190d8c8fe94d9ea2d70d7d88b22a2642 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Feb 2024 06:19:25 -0800 Subject: [PATCH 277/279] Bump pre-commit/action from 3.0.0 to 3.0.1 (#4225) Bumps [pre-commit/action](https://github.com/pre-commit/action) from 3.0.0 to 3.0.1. - [Release notes](https://github.com/pre-commit/action/releases) - [Commits](https://github.com/pre-commit/action/compare/v3.0.0...v3.0.1) --- updated-dependencies: - dependency-name: pre-commit/action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8cb335bffb5..f75734400ce 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -35,7 +35,7 @@ jobs: python -m pip install tox - name: Run pre-commit hooks - uses: pre-commit/action@v3.0.0 + uses: pre-commit/action@v3.0.1 - name: Format ourselves run: | From 8af439407c051d55f3221cc93795d20bd9af49c9 Mon Sep 17 00:00:00 2001 From: cobalt <61329810+RedGuy12@users.noreply.github.com> Date: Mon, 12 Feb 2024 08:27:50 -0600 Subject: [PATCH 278/279] fix: Don't remove comments along with parens (#4218) Signed-off-by: RedGuy12 <61329810+RedGuy12@users.noreply.github.com> --- CHANGES.md | 3 + src/black/linegen.py | 6 + tests/data/cases/comments_in_double_parens.py | 113 ++++++++++++++++++ 3 files changed, 122 insertions(+) create mode 100644 tests/data/cases/comments_in_double_parens.py diff --git a/CHANGES.md b/CHANGES.md index 4fd63ac41eb..b1a6ae3bc1c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -10,6 +10,9 @@ +- Fixed a bug where comments where mistakenly removed along with redundant parentheses + (#4218) + ### Preview style diff --git a/src/black/linegen.py b/src/black/linegen.py index 2b7fbc492cf..cc8e41dfb20 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -1553,6 +1553,9 @@ def maybe_make_parens_invisible_in_atom( not is_type_ignore_comment_string(middle.prefix.strip()) ): first.value = "" + if first.prefix.strip(): + # Preserve comments before first paren + middle.prefix = first.prefix + middle.prefix last.value = "" maybe_make_parens_invisible_in_atom( middle, @@ -1564,6 +1567,9 @@ def maybe_make_parens_invisible_in_atom( # Strip the invisible parens from `middle` by replacing # it with the child in-between the invisible parens middle.replace(middle.children[1]) + if middle.children[-1].prefix.strip(): + # Preserve comments before last paren + last.prefix = middle.children[-1].prefix + last.prefix return False diff --git a/tests/data/cases/comments_in_double_parens.py b/tests/data/cases/comments_in_double_parens.py new file mode 100644 index 00000000000..80e7a5e5bf5 --- /dev/null +++ b/tests/data/cases/comments_in_double_parens.py @@ -0,0 +1,113 @@ +if ( + True + # sdf +): + print("hw") + +if (( + True + # sdf +)): + print("hw") + +if (( + # type: ignore + True +)): + print("hw") + +if (( + True + # type: ignore +)): + print("hw") + +if ( + # a long comment about + # the condition below + (a or b) +): + pass + +def return_true(): + return ( + ( + True # this comment gets removed accidentally + ) + ) + +def return_true(): + return (True) # this comment gets removed accidentally + + +if ( + # huh comment + (True) +): + ... + +if ( + # huh + ( + # comment + True + ) +): + ... + + +# output + +if ( + True + # sdf +): + print("hw") + +if ( + True + # sdf +): + print("hw") + +if ( + # type: ignore + True +): + print("hw") + +if ( + True + # type: ignore +): + print("hw") + +if ( + # a long comment about + # the condition below + a + or b +): + pass + + +def return_true(): + return True # this comment gets removed accidentally + + +def return_true(): + return True # this comment gets removed accidentally + + +if ( + # huh comment + True +): + ... + +if ( + # huh + # comment + True +): + ... From 6fdf8a4af28071ed1d079c01122b34c5d587207a Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Mon, 12 Feb 2024 12:11:15 -0800 Subject: [PATCH 279/279] Prepare release 24.2.0 (#4226) --- CHANGES.md | 35 +-------------------- docs/integrations/source_version_control.md | 4 +-- docs/usage_and_configuration/the_basics.md | 6 ++-- 3 files changed, 6 insertions(+), 39 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b1a6ae3bc1c..39191c1cd82 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,22 +1,14 @@ # Change Log -## Unreleased - -### Highlights - - +## 24.2.0 ### Stable style - - - Fixed a bug where comments where mistakenly removed along with redundant parentheses (#4218) ### Preview style - - - Move the `hug_parens_with_braces_and_square_brackets` feature to the unstable style due to an outstanding crash and proposed formatting tweaks (#4198) - Fixed a bug where base expressions caused inconsistent formatting of \*\* in tenary @@ -35,40 +27,15 @@ simply add an empty `[tool.black]` to the previously discovered `pyproject.toml` (#4204) -### Packaging - - - -### Parser - - - -### Performance - - - ### Output - - - Black will swallow any `SyntaxWarning`s or `DeprecationWarning`s produced by the `ast` module when performing equivalence checks (#4189) -### _Blackd_ - - - ### Integrations - - - Add a JSONSchema and provide a validate-pyproject entry-point (#4181) -### Documentation - - - ## 24.1.1 Bugfix release to fix a bug that made Black unusable on certain file systems with strict diff --git a/docs/integrations/source_version_control.md b/docs/integrations/source_version_control.md index 92279707d84..f24043026fa 100644 --- a/docs/integrations/source_version_control.md +++ b/docs/integrations/source_version_control.md @@ -8,7 +8,7 @@ Use [pre-commit](https://pre-commit.com/). Once you repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.1.1 + rev: 24.2.0 hooks: - id: black # It is recommended to specify the latest version of Python @@ -35,7 +35,7 @@ include Jupyter Notebooks. To use this hook, simply replace the hook's `id: blac repos: # Using this mirror lets us use mypyc-compiled black, which is about 2x faster - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.1.1 + rev: 24.2.0 hooks: - id: black-jupyter # It is recommended to specify the latest version of Python diff --git a/docs/usage_and_configuration/the_basics.md b/docs/usage_and_configuration/the_basics.md index 61c52450165..ea7a2dae5ce 100644 --- a/docs/usage_and_configuration/the_basics.md +++ b/docs/usage_and_configuration/the_basics.md @@ -266,8 +266,8 @@ configuration file for consistent results across environments. ```console $ black --version -black, 24.1.1 (compiled: yes) -$ black --required-version 24.1.1 -c "format = 'this'" +black, 24.2.0 (compiled: yes) +$ black --required-version 24.2.0 -c "format = 'this'" format = "this" $ black --required-version 31.5b2 -c "still = 'beta?!'" Oh no! 💥 💔 💥 The required version does not match the running version! @@ -363,7 +363,7 @@ You can check the version of _Black_ you have installed using the `--version` fl ```console $ black --version -black, 24.1.1 +black, 24.2.0 ``` #### `--config`