Preserve file order of messages during successive daemon runs (#13780)

This fixes an annoyance where the messages got reshuffled between daemon
runs.

Also if there are messages from files that didn't generate messages
during the previous run, move them towards the end to make them more
visible.

The implementation is a bit messy since we only have a list of formatted
lines where it's most natural to sort the messages, but individual
messages can be split across multiple lines.

Fix #13141.

mypy/server/update.py

@@ -115,6 +115,7 @@
 from __future__ import annotations
 
 import os
+import re
 import sys
 import time
 from typing import Callable, NamedTuple, Sequence, Union
@@ -182,7 +183,7 @@ def __init__(self, result: BuildResult) -> None:
         # Merge in any root dependencies that may not have been loaded
         merge_dependencies(manager.load_fine_grained_deps(FAKE_ROOT_MODULE), self.deps)
         self.previous_targets_with_errors = manager.errors.targets()
-        self.previous_messages = result.errors[:]
+        self.previous_messages: list[str] = result.errors[:]
         # Module, if any, that had blocking errors in the last run as (id, path) tuple.
         self.blocking_error: tuple[str, str] | None = None
         # Module that we haven't processed yet but that are known to be stale.
@@ -290,6 +291,7 @@ def update(
                     messages = self.manager.errors.new_messages()
                     break
 
+        messages = sort_messages_preserving_file_order(messages, self.previous_messages)
         self.previous_messages = messages[:]
         return messages
 
@@ -1260,3 +1262,61 @@ def refresh_suppressed_submodules(
                                 state.suppressed.append(submodule)
                                 state.suppressed_set.add(submodule)
     return messages
+
+
+def extract_fnam_from_message(message: str) -> str | None:
+    m = re.match(r"([^:]+):[0-9]+: (error|note): ", message)
+    if m:
+        return m.group(1)
+    return None
+
+
+def extract_possible_fnam_from_message(message: str) -> str:
+    # This may return non-path things if there is some random colon on the line
+    return message.split(":", 1)[0]
+
+
+def sort_messages_preserving_file_order(
+    messages: list[str], prev_messages: list[str]
+) -> list[str]:
+    """Sort messages so that the order of files is preserved.
+
+    An update generates messages so that the files can be in a fairly
+    arbitrary order.  Preserve the order of files to avoid messages
+    getting reshuffled continuously.  If there are messages in
+    additional files, sort them towards the end.
+    """
+    # Calculate file order from the previous messages
+    n = 0
+    order = {}
+    for msg in prev_messages:
+        fnam = extract_fnam_from_message(msg)
+        if fnam and fnam not in order:
+            order[fnam] = n
+            n += 1
+
+    # Related messages must be sorted as a group of successive lines
+    groups = []
+    i = 0
+    while i < len(messages):
+        msg = messages[i]
+        maybe_fnam = extract_possible_fnam_from_message(msg)
+        group = [msg]
+        if maybe_fnam in order:
+            # This looks like a file name. Collect all lines related to this message.
+            while (
+                i + 1 < len(messages)
+                and extract_possible_fnam_from_message(messages[i + 1]) not in order
+                and extract_fnam_from_message(messages[i + 1]) is None
+                and not messages[i + 1].startswith("mypy: ")
+            ):
+                i += 1
+                group.append(messages[i])
+        groups.append((order.get(maybe_fnam, n), group))
+        i += 1
+
+    groups = sorted(groups, key=lambda g: g[0])
+    result = []
+    for key, group in groups:
+        result.extend(group)
+    return result

mypy/test/testfinegrained.py

@@ -17,6 +17,7 @@
 import os
 import re
 import sys
+import unittest
 from typing import Any, cast
 
 import pytest
@@ -30,6 +31,7 @@
 from mypy.modulefinder import BuildSource
 from mypy.options import Options
 from mypy.server.mergecheck import check_consistency
+from mypy.server.update import sort_messages_preserving_file_order
 from mypy.test.config import test_temp_dir
 from mypy.test.data import DataDrivenTestCase, DataSuite, DeleteFile, UpdateFile
 from mypy.test.helpers import (
@@ -369,3 +371,70 @@ def get_inspect(self, program_text: str, incremental_step: int) -> list[tuple[st
 
 def normalize_messages(messages: list[str]) -> list[str]:
     return [re.sub("^tmp" + re.escape(os.sep), "", message) for message in messages]
+
+
+class TestMessageSorting(unittest.TestCase):
+    def test_simple_sorting(self) -> None:
+        msgs = ['x.py:1: error: "int" not callable', 'foo/y.py:123: note: "X" not defined']
+        old_msgs = ['foo/y.py:12: note: "Y" not defined', 'x.py:8: error: "str" not callable']
+        assert sort_messages_preserving_file_order(msgs, old_msgs) == list(reversed(msgs))
+        assert sort_messages_preserving_file_order(list(reversed(msgs)), old_msgs) == list(
+            reversed(msgs)
+        )
+
+    def test_long_form_sorting(self) -> None:
+        # Multi-line errors should be sorted together and not split.
+        msg1 = [
+            'x.py:1: error: "int" not callable',
+            "and message continues (x: y)",
+            "    1()",
+            "    ^~~",
+        ]
+        msg2 = [
+            'foo/y.py: In function "f":',
+            'foo/y.py:123: note: "X" not defined',
+            "and again message continues",
+        ]
+        old_msgs = ['foo/y.py:12: note: "Y" not defined', 'x.py:8: error: "str" not callable']
+        assert sort_messages_preserving_file_order(msg1 + msg2, old_msgs) == msg2 + msg1
+        assert sort_messages_preserving_file_order(msg2 + msg1, old_msgs) == msg2 + msg1
+
+    def test_mypy_error_prefix(self) -> None:
+        # Some errors don't have a file and start with "mypy: ". These
+        # shouldn't be sorted together with file-specific errors.
+        msg1 = 'x.py:1: error: "int" not callable'
+        msg2 = 'foo/y:123: note: "X" not defined'
+        msg3 = "mypy: Error not associated with a file"
+        old_msgs = [
+            "mypy: Something wrong",
+            'foo/y:12: note: "Y" not defined',
+            'x.py:8: error: "str" not callable',
+        ]
+        assert sort_messages_preserving_file_order([msg1, msg2, msg3], old_msgs) == [
+            msg2,
+            msg1,
+            msg3,
+        ]
+        assert sort_messages_preserving_file_order([msg3, msg2, msg1], old_msgs) == [
+            msg2,
+            msg1,
+            msg3,
+        ]
+
+    def test_new_file_at_the_end(self) -> None:
+        msg1 = 'x.py:1: error: "int" not callable'
+        msg2 = 'foo/y.py:123: note: "X" not defined'
+        new1 = "ab.py:3: error: Problem: error"
+        new2 = "aaa:3: error: Bad"
+        old_msgs = ['foo/y.py:12: note: "Y" not defined', 'x.py:8: error: "str" not callable']
+        assert sort_messages_preserving_file_order([msg1, msg2, new1], old_msgs) == [
+            msg2,
+            msg1,
+            new1,
+        ]
+        assert sort_messages_preserving_file_order([new1, msg1, msg2, new2], old_msgs) == [
+            msg2,
+            msg1,
+            new1,
+            new2,
+        ]

test-data/unit/fine-grained-blockers.test

@@ -317,8 +317,8 @@ a.py:1: error: invalid syntax
 ==
 a.py:1: error: invalid syntax
 ==
-b.py:3: error: Too many arguments for "f"
 a.py:3: error: Too many arguments for "g"
+b.py:3: error: Too many arguments for "f"
 
 [case testDeleteFileWithBlockingError-only_when_nocache]
 -- Different cache/no-cache tests because:

test-data/unit/fine-grained-follow-imports.test

@@ -587,8 +587,8 @@ def f() -> None:
 main.py:2: error: Cannot find implementation or library stub for module named "p"
 main.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
 ==
-p/m.py:1: error: "str" not callable
 p/__init__.py:1: error: "int" not callable
+p/m.py:1: error: "str" not callable
 
 [case testFollowImportsNormalPackageInitFileStub]
 # flags: --follow-imports=normal
@@ -610,11 +610,11 @@ x x x
 main.py:1: error: Cannot find implementation or library stub for module named "p"
 main.py:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
 ==
-p/m.pyi:1: error: "str" not callable
 p/__init__.pyi:1: error: "int" not callable
-==
 p/m.pyi:1: error: "str" not callable
+==
 p/__init__.pyi:1: error: "int" not callable
+p/m.pyi:1: error: "str" not callable
 
 [case testFollowImportsNormalNamespacePackages]
 # flags: --follow-imports=normal --namespace-packages
@@ -638,12 +638,12 @@ main.py:2: error: Cannot find implementation or library stub for module named "p
 main.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
 main.py:2: error: Cannot find implementation or library stub for module named "p2"
 ==
-p2/m2.py:1: error: "str" not callable
 p1/m1.py:1: error: "int" not callable
+p2/m2.py:1: error: "str" not callable
 ==
+p1/m1.py:1: error: "int" not callable
 main.py:2: error: Cannot find implementation or library stub for module named "p2.m2"
 main.py:2: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
-p1/m1.py:1: error: "int" not callable
 
 [case testFollowImportsNormalNewFileOnCommandLine]
 # flags: --follow-imports=normal
@@ -659,8 +659,8 @@ p1/m1.py:1: error: "int" not callable
 [out]
 main.py:1: error: "int" not callable
 ==
-x.py:1: error: "str" not callable
 main.py:1: error: "int" not callable
+x.py:1: error: "str" not callable
 
 [case testFollowImportsNormalSearchPathUpdate-only_when_nocache]
 # flags: --follow-imports=normal
@@ -678,8 +678,8 @@ import bar
 
 [out]
 ==
-src/bar.py:1: error: "int" not callable
 src/foo.py:2: error: "str" not callable
+src/bar.py:1: error: "int" not callable
 
 [case testFollowImportsNormalSearchPathUpdate2-only_when_cache]
 # flags: --follow-imports=normal

test-data/unit/fine-grained-modules.test

@@ -38,8 +38,8 @@ def f(x: int) -> None: pass
 ==
 a.py:2: error: Incompatible return value type (got "int", expected "str")
 ==
-b.py:2: error: Too many arguments for "f"
 a.py:2: error: Incompatible return value type (got "int", expected "str")
+b.py:2: error: Too many arguments for "f"
 ==
 
 [case testAddFileFixesError]

test-data/unit/fine-grained.test

@@ -1814,9 +1814,9 @@ def f() -> Iterator[None]:
 [out]
 main:2: note: Revealed type is "contextlib.GeneratorContextManager[None]"
 ==
+main:2: note: Revealed type is "contextlib.GeneratorContextManager[None]"
 a.py:3: error: Cannot find implementation or library stub for module named "b"
 a.py:3: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
-main:2: note: Revealed type is "contextlib.GeneratorContextManager[None]"
 ==
 main:2: note: Revealed type is "contextlib.GeneratorContextManager[None]"
 
@@ -8689,8 +8689,8 @@ main:2: note: Revealed type is "builtins.int"
 ==
 main:2: note: Revealed type is "Literal[1]"
 ==
-mod.py:2: error: Incompatible types in assignment (expression has type "Literal[2]", variable has type "Literal[1]")
 main:2: note: Revealed type is "Literal[1]"
+mod.py:2: error: Incompatible types in assignment (expression has type "Literal[2]", variable has type "Literal[1]")
 
 [case testLiteralFineGrainedFunctionConversion]
 from mod import foo
@@ -9178,10 +9178,10 @@ a.py:1: error: Type signature has too few arguments
 a.py:5: error: Type signature has too few arguments
 a.py:11: error: Type signature has too few arguments
 ==
+c.py:1: error: Type signature has too few arguments
 a.py:1: error: Type signature has too few arguments
 a.py:5: error: Type signature has too few arguments
 a.py:11: error: Type signature has too few arguments
-c.py:1: error: Type signature has too few arguments
 
 [case testErrorReportingNewAnalyzer]
 # flags: --disallow-any-generics
@@ -10072,3 +10072,53 @@ class Base(Protocol):
 [out]
 ==
 main:6: error: Call to abstract method "meth" of "Base" with trivial body via super() is unsafe
+
+[case testPrettyMessageSorting]
+# flags: --pretty
+import a
+
+[file a.py]
+1 + ''
+import b
+
+[file b.py]
+object + 1
+
+[file b.py.2]
+object + 1
+1()
+
+[out]
+b.py:1: error: Unsupported left operand type for + ("Type[object]")
+    object + 1
+    ^
+a.py:1: error: Unsupported operand types for + ("int" and "str")
+    1 + ''
+        ^
+==
+b.py:1: error: Unsupported left operand type for + ("Type[object]")
+    object + 1
+    ^
+b.py:2: error: "int" not callable
+    1()
+    ^
+a.py:1: error: Unsupported operand types for + ("int" and "str")
+    1 + ''
+        ^
+[out version>=3.8]
+b.py:1: error: Unsupported left operand type for + ("Type[object]")
+    object + 1
+    ^~~~~~~~~~
+a.py:1: error: Unsupported operand types for + ("int" and "str")
+    1 + ''
+        ^~
+==
+b.py:1: error: Unsupported left operand type for + ("Type[object]")
+    object + 1
+    ^~~~~~~~~~
+b.py:2: error: "int" not callable
+    1()
+    ^~~
+a.py:1: error: Unsupported operand types for + ("int" and "str")
+    1 + ''
+        ^~