Regression test for 3181, add documentation for toml configuration

doc/user_guide/run.rst

@@ -100,6 +100,9 @@ configuration file in the following order and uses the first one it finds:
 #. ``.pylintrc`` in the current working directory
 #. ``pyproject.toml`` in the current working directory,
    providing it has at least one ``tool.pylint.`` section.
+   The ``pyproject.toml`` must prepend section names with ``tool.pylint.``,
+   for example ``[tool.pylint.'MESSAGES CONTROL']``. They can also be passed
+   in on the command line.
 #. ``setup.cfg`` in the current working directory,
    providing it has at least one ``pylint.`` section
 #. If the current working directory is in a Python package, Pylint searches \

pylint/config/option_manager_mixin.py

@@ -274,7 +274,10 @@ def read_config_file(self, config_file=None, verbose=None):
             self.set_current_module(config_file)
             parser = self.cfgfile_parser
             if config_file.endswith(".toml"):
-                self._parse_toml(config_file, parser)
+                try:
+                    self._parse_toml(config_file, parser)
+                except toml.TomlDecodeError as e:
+                    self.add_message("config-parse-error", line=0, args=str(e))
             else:
                 # Use this encoding in order to strip the BOM marker, if any.
                 with open(config_file, encoding="utf_8_sig") as fp:

pylint/lint/pylinter.py

@@ -98,6 +98,11 @@ def _load_reporter_by_class(reporter_class: str) -> type:
         "Used when an exception occurred while building the Astroid "
         "representation which could be handled by astroid.",
     ),
+    "F0011": (
+        "error while parsing the configuration: %s",
+        "config-parse-error",
+        "Used when an exception occurred while parsing a pylint configuration file.",
+    ),
     "I0001": (
         "Unable to run raw checkers on built-in module %s",
         "raw-checker-failed",

pylint/testutils/configuration_test.py

@@ -7,7 +7,7 @@
 import logging
 import unittest
 from pathlib import Path
-from typing import Any, Dict, Tuple, Union
+from typing import Any, Dict, List, Tuple, Union
 from unittest.mock import Mock
 
 from pylint.lint import Run
@@ -19,16 +19,14 @@
 
 
 def get_expected_or_default(
-    tested_configuration_file: str, suffix: str, default: ConfigurationValue
+    tested_configuration_file: Union[str, Path],
+    suffix: str,
+    default: ConfigurationValue,
 ) -> str:
     """Return the expected value from the file if it exists, or the given default."""
-
-    def get_path_according_to_suffix() -> Path:
-        path = Path(tested_configuration_file)
-        return path.parent / f"{path.stem}.{suffix}"
-
     expected = default
-    expected_result_path = get_path_according_to_suffix()
+    path = Path(tested_configuration_file)
+    expected_result_path = path.parent / f"{path.stem}.{suffix}"
     if expected_result_path.exists():
         with open(expected_result_path, encoding="utf8") as f:
             expected = f.read()
@@ -70,22 +68,61 @@ def get_expected_configuration(
     return result
 
 
+def get_related_files(
+    tested_configuration_file: Union[str, Path], suffix_filter: str
+) -> List[Path]:
+    """Return all the file related to a test conf file endind with a suffix."""
+    conf_path = Path(tested_configuration_file)
+    return [
+        p
+        for p in conf_path.parent.iterdir()
+        if str(p.stem).startswith(conf_path.stem) and str(p).endswith(suffix_filter)
+    ]
+
+
 def get_expected_output(
-    configuration_path: str, user_specific_path: Path
+    configuration_path: Union[str, Path], user_specific_path: Path
 ) -> Tuple[int, str]:
     """Get the expected output of a functional test."""
-    output = get_expected_or_default(configuration_path, suffix="out", default="")
-    if output:
+    exit_code = 0
+    msg = (
+        "we expect a single file of the form 'filename.32.out' where 'filename' represents "
+        "the name of the configuration file, and '32' the expected error code."
+    )
+    possible_out_files = get_related_files(configuration_path, suffix_filter="out")
+    if len(possible_out_files) > 1:
+        logging.error(
+            "Too much .out files for %s %s.",
+            configuration_path,
+            msg,
+        )
+        return -1, "out file is broken"
+    if not possible_out_files:
         # logging is helpful to see what the expected exit code is and why.
         # The output of the program is checked during the test so printing
         # messes with the result.
-        logging.info(
-            "Output exists for %s so the expected exit code is 2", configuration_path
-        )
-        exit_code = 2
-    else:
         logging.info(".out file does not exists, so the expected exit code is 0")
-        exit_code = 0
+        return 0, ""
+    path = possible_out_files[0]
+    try:
+        exit_code = int(str(path.stem).rsplit(".", maxsplit=1)[-1])
+    except Exception as e:  # pylint: disable=broad-except
+        logging.error(
+            "Wrong format for .out file name for %s %s: %s",
+            configuration_path,
+            msg,
+            e,
+        )
+        return -1, "out file is broken"
+
+    output = get_expected_or_default(
+        configuration_path, suffix=f"{exit_code}.out", default=""
+    )
+    logging.info(
+        "Output exists for %s so the expected exit code is %s",
+        configuration_path,
+        exit_code,
+    )
     return exit_code, output.format(
         abspath=configuration_path,
         relpath=Path(configuration_path).relative_to(user_specific_path),

tests/config/functional/toml/issue_3181/toml_decode_error.1.out

@@ -0,0 +1,2 @@
+************* Module {abspath}
+{relpath}:1:0: F0011: error while parsing the configuration: Found invalid character in key name: '*'. Try quoting the key name. (line 3 column 2 char 48) (config-parse-error)

tests/config/functional/toml/issue_3181/toml_decode_error.toml

@@ -0,0 +1,3 @@
+# TOML decode error crash pylint
+[tool.pylint]
+***

tests/config/functional/toml/issue_3181/top_level_list_of_disable.2.out

@@ -0,0 +1,3 @@
+************* Module {abspath}
+{relpath}:1:0: E0014: Out-of-place setting encountered in top level configuration-section 'max-line-length' : '120' (bad-configuration-section)
+{relpath}:1:0: E0014: Out-of-place setting encountered in top level configuration-section 'disable' : '['C0330']' (bad-configuration-section)

tests/config/functional/toml/issue_3181/top_level_list_of_disable.toml

@@ -0,0 +1,4 @@
+# This crashed previously see https://github.com/PyCQA/pylint/issues/3181
+[tool.pylint]
+max-line-length = 120
+disable = ["C0330"]

tests/testutils/test_configuration_test.py

@@ -0,0 +1,23 @@
+import logging
+from pathlib import Path
+
+from pytest import LogCaptureFixture
+
+from pylint.testutils.configuration_test import get_expected_output
+
+HERE = Path(__file__).parent
+USER_SPECIFIC_PATH = HERE.parent.parent
+DATA_DIRECTORY = HERE / "data"
+
+
+def test_get_expected_output(caplog: LogCaptureFixture) -> None:
+    caplog.set_level(logging.INFO)
+    exit_code, _ = get_expected_output(DATA_DIRECTORY / "t.toml", USER_SPECIFIC_PATH)
+    assert "Too much .out files" in str(caplog.text)
+    assert exit_code == -1
+    exit_code, _ = get_expected_output(DATA_DIRECTORY / "u.toml", USER_SPECIFIC_PATH)
+    assert exit_code == -1
+    assert "Wrong format for .out file name" in str(caplog.text)
+    exit_code, _ = get_expected_output(DATA_DIRECTORY / "v.toml", USER_SPECIFIC_PATH)
+    assert exit_code == 0
+    assert ".out file does not exists" in str(caplog.text)