Merge pull request #37 from PyNumLab/codex/review-fortran-parser-behavior

Treat executable statements at file scope as syntax errors

Author: saidctb

README.md

@@ -112,9 +112,18 @@ path when `--language` is omitted. C source/header files require explicit
 `--language fortran` or `--language c`. C parsing, semantic IR, `.pyi`
 generation, and wrap-readiness are available in explicit C mode. Selecting a
 frontend that conflicts with a recognized C or Fortran source suffix is an
-error. Once selected, a frontend also rejects unmistakable declarations or
-program-unit syntax that are not from the selected language outside ignored execution/function
-bodies, rather than silently dropping them.
+error. Once selected, a frontend validates the grammar regions it models and
+rejects unparsed syntax outside intentionally ignored execution/function
+bodies, rather than guessing another language from keyword spellings or
+silently dropping malformed input.
+
+Parse failures print a compiler-style diagnostic without a Python traceback.
+Use `--debug` to re-raise the parser error and print the traceback;
+`--debug-traceback` remains accepted as a compatibility alias. Diagnostic codes
+such as `PARSE_UNSUPPORTED_DECLARATION`, `CPARSE_INVALID_SPECIFIER_SEQUENCE`,
+and `CPARSE_INVALID_SYNTAX` are stable, explicit error-category identifiers for
+tests, tools, and documentation. The current categories are listed in
+[`docs/diagnostic_codes.md`](docs/diagnostic_codes.md).
 
 For parse output, `--show-vars` expands scope-level variables that are normally
 summarized as `vars=N`. Use `--print-limit N` to keep large repeated sections
@@ -718,8 +727,12 @@ visitor then parses only its own substring, splits it into header,
 specification, optional execution, and optional `contains` regions, and recurses
 into direct child units where that grammar allows children. Shared declaration
 helpers parse variables, procedure arguments/results, and type fields, then
-push them into the active scope. Procedure execution bodies and internal
-subprograms are ignored for wrapper metadata; procedure-local interfaces are
+push them into the active scope. Nested unit boundaries and placement outside
+execution regions are checked even when they do not produce wrapper metadata.
+Internal procedures inside a host procedure's `contains` block are
+structurally sliced, then their declarations and bodies are skipped. After an
+execution boundary is detected, procedure bodies and standalone included
+execution fragments are intentionally skipped. Procedure-local interfaces are
 retained for callback typing.
 Parameter variables keep both `value` and serialized `symbolic_value` when the
 parser has that information. `value` is literal/evaluated only; if an

c_parser/cli.py

@@ -3,15 +3,26 @@
 
 import argparse
 import json
+import os
+import sys
 from collections.abc import Callable, Sequence
 from pathlib import Path
 from typing import Any
 
-from .models import CFile, c_model_to_dict
+from .models import CFile, CParseError, c_model_to_dict
 from .parser import CParser
 
 
 _C_SOURCE_SUFFIXES = {".c", ".h", ".i"}
+_TRUE_VALUES = {"1", "true", "yes", "on"}
+
+
+def _env_flag(name: str) -> bool:
+    return os.getenv(name, "").strip().lower() in _TRUE_VALUES
+
+
+def _diagnostic_color_enabled(*, disabled: bool) -> bool:
+    return not disabled and "NO_COLOR" not in os.environ
 
 
 def _collect_c_extensions(path: Path) -> list[Path]:
@@ -88,9 +99,23 @@ def main(argv: list[str] | None = None) -> int:
     parser.add_argument("paths", nargs="+", help="C source/header file(s) or directory path(s)")
     parser.add_argument("--json", action="store_true", help="Print JSON to stdout")
     parser.add_argument("--out", type=str, help="Write parser JSON to a file")
+    parser.add_argument("--no-color", action="store_true", help="Disable ANSI color in parse diagnostics")
+    parser.add_argument(
+        "--debug",
+        "--debug-traceback",
+        dest="debug",
+        action="store_true",
+        help="Re-raise parser errors so Python prints a traceback for parser debugging.",
+    )
     args = parser.parse_args(argv)
 
-    payload = parse_c_report(args.paths)
+    try:
+        payload = parse_c_report(args.paths)
+    except CParseError as exc:
+        if args.debug or _env_flag("C_PARSER_DEBUG"):
+            raise
+        print(exc.format_diagnostic(color=_diagnostic_color_enabled(disabled=args.no_color), debug=False), file=sys.stderr)
+        return 1
     if args.out:
         Path(args.out).write_text(json.dumps(payload, indent=2), encoding="utf-8")
         return 0

c_parser/models.py

@@ -106,7 +106,7 @@ def c_model_to_dict(obj: Any, _seen: set[int] | None = None) -> Any:
 class CParseError(ValueError):
     """C parser error with compiler-style diagnostic rendering support."""
 
-    default_code = "CPARSE001"
+    default_code = "CPARSE_ERROR"
 
     def __init__(
         self,

c_parser/parser.py

@@ -116,8 +116,6 @@
     "_Alignas",
     "alignas",
 )
-_CXX_DECLARATION_KEYWORDS = {"using", "namespace", "template", "class"}
-_CXX_ACCESS_SPECIFIERS = {"public", "private", "protected"}
 _RAW_CONDITIONAL_DIRECTIVE_RE = re.compile(
     r"^\s*#\s*(?P<directive>if|ifdef|ifndef|elif|else|endif)\b"
 )
@@ -230,6 +228,12 @@ class _InvalidSpecifierSequence(ValueError):
     pass
 
 
+class _InvalidCGrammarSyntax(ValueError):
+    """Raised internally when a nested C grammar region is malformed."""
+
+    pass
+
+
 def _looks_like_existing_source_path(value: object) -> bool:
     """Return whether `value` can safely be treated as an existing source path."""
     if isinstance(value, Path):
@@ -274,21 +278,6 @@ def _is_source_key(key: str) -> bool:
     return PurePosixPath(key).suffix.lower() == ".c"
 
 
-def _looks_like_cxx_declaration(text: str) -> bool:
-    """Detect obvious C++ declarations so they become explicit diagnostics."""
-    stripped = text.lstrip()
-    identifier = _IDENTIFIER_RE.match(stripped)
-    if identifier is None:
-        return False
-
-    word = identifier.group(0)
-    if word in _CXX_DECLARATION_KEYWORDS:
-        return True
-    if word in _CXX_ACCESS_SPECIFIERS:
-        return stripped[identifier.end() :].lstrip().startswith(":")
-    return False
-
-
 class CParser:
     """Parser orchestration object for the partial typed C model.
 
@@ -584,7 +573,7 @@ def _could_start_c_external_declaration(text: str) -> bool:
     @staticmethod
     def _raise_for_invalid_top_level_syntax(segment: CTopLevelSegment) -> None:
         text = segment.text.strip()
-        if not text or _looks_like_cxx_declaration(text):
+        if not text:
             return
         tokens = lex_c_source(text)
         has_scope_operator = any(
@@ -606,6 +595,25 @@ def _raise_for_invalid_top_level_syntax(segment: CTopLevelSegment) -> None:
             code="CPARSE_INVALID_SYNTAX",
         )
 
+    def _invalid_syntax_error(
+        self,
+        segment: CTopLevelSegment,
+        text: str,
+        *,
+        context: str,
+        offset: int = 0,
+    ) -> CParseError:
+        """Build the fatal diagnostic used when a C grammar region is invalid."""
+        location = self._source_location_at(segment, offset)
+        return CParseError(
+            f"Invalid C syntax in {context}: {text.strip()}",
+            filename=location.filename,
+            line_number=location.line,
+            column=location.column,
+            source_line=location.source_line,
+            code="CPARSE_INVALID_SYNTAX",
+        )
+
     def _macro_dependencies(
         self,
         source: str,
@@ -1183,7 +1191,7 @@ def _invalid_specifier_error(
             line_number=location.line,
             column=location.column,
             source_line=location.source_line,
-            code="CPARSE003",
+            code="CPARSE_INVALID_SPECIFIER_SEQUENCE",
         )
 
     def _atomic_type_specifier_parts(self, spec_text: str) -> tuple[str, str] | None:
@@ -1669,7 +1677,7 @@ def _parse_parameter(self, text: str) -> CParameter | None:
             return None
         spec_text, declarator = self._split_declaration_specifiers(stripped)
         if not spec_text:
-            return None
+            raise _InvalidCGrammarSyntax(f"Invalid parameter declaration: {stripped}")
         name, type_, _storage, _function_specifiers, _direct_function = self._build_declared_type(
             spec_text,
             declarator,
@@ -1724,13 +1732,17 @@ def _parse_parameters(self, parameters_text: str) -> tuple[list[CParameter], boo
 
         parameters: list[CParameter] = []
         variadic = False
-        for item in top_level_split(stripped, ","):
+        items = top_level_split(stripped, ",")
+        for index, item in enumerate(items):
             if item == "...":
+                if variadic or index != len(items) - 1:
+                    raise _InvalidCGrammarSyntax("The variadic marker must be the final function parameter.")
                 variadic = True
                 continue
             parameter = self._parse_parameter(item)
-            if parameter is not None:
-                parameters.append(parameter)
+            if parameter is None:
+                raise _InvalidCGrammarSyntax(f"Invalid parameter declaration: {item}")
+            parameters.append(parameter)
         return parameters, variadic
 
     def _is_knr_definition(self, segment: CTopLevelSegment, parameters_text: str) -> bool:
@@ -1803,7 +1815,7 @@ def _raise_for_unsupported_old_style_definitions(
                         line_number=mapping.line if mapping is not None else index + 1,
                         column=max(line.find(name_match.group(0)) + 1, 1),
                         source_line=source_line,
-                        code="CPARSE002",
+                        code="CPARSE_UNSUPPORTED_KNR_DEFINITION",
                     )
                 if stripped.endswith(";"):
                     saw_old_style_declaration = True
@@ -1821,7 +1833,7 @@ def _raise_for_unsupported_old_style_definitions(
                     line_number=mapping.line if mapping is not None else index + 1,
                     column=max(line.find(name_match.group(0)) + 1, 1),
                     source_line=source_line,
-                    code="CPARSE002",
+                    code="CPARSE_UNSUPPORTED_KNR_DEFINITION",
                 )
 
     def _prototype_style(self, parameters_text: str) -> str:
@@ -1857,7 +1869,7 @@ def _parse_function(self, segment: CTopLevelSegment) -> CFunction | None:
                     line_number=segment.original_start_line,
                     column=segment.original_start_column,
                     source_line=segment.original_source_line,
-                    code="CPARSE002",
+                    code="CPARSE_UNSUPPORTED_KNR_DEFINITION",
                 )
         return self._function_from_type(
             name,
@@ -2196,6 +2208,13 @@ def _parse_fields(
         """Parse struct/union member declarations through the shared backend."""
         members: list[CVariable] = []
         diagnostics: list[CDiagnostic] = []
+        if body.strip() and not body.rstrip().endswith(";"):
+            raise self._invalid_syntax_error(
+                segment,
+                body,
+                context=f"{owner_kind} field declaration",
+                offset=body_offset,
+            )
         for text, field_offset in top_level_split_with_offsets(body, ";"):
             member_offset = body_offset + field_offset
             member_location = self._source_location_at(segment, member_offset)
@@ -2239,17 +2258,21 @@ def _parse_fields(
                     )
                 )
                 continue
+            if "::" in text:
+                raise self._invalid_syntax_error(
+                    segment,
+                    text,
+                    context=f"{owner_kind} field declaration",
+                    offset=member_offset,
+                )
             spec_text, declarator_list = self._split_declaration_specifiers(text)
             if not spec_text or not declarator_list:
-                diagnostics.append(
-                    self._field_diagnostic(
-                        segment,
-                        owner_kind,
-                        "Unsupported field declaration.",
-                        offset=member_offset,
-                    )
+                raise self._invalid_syntax_error(
+                    segment,
+                    text,
+                    context=f"{owner_kind} field declaration",
+                    offset=member_offset,
                 )
-                continue
             for declarator in top_level_split(declarator_list, ","):
                 declaration, _initializer = top_level_partition(declarator, "=")
                 declaration, bit_width = top_level_partition(declaration, ":")
@@ -2293,10 +2316,10 @@ def _parse_enumerators(self, body: str, segment: CTopLevelSegment) -> list[CEnum
             name_text, value = top_level_partition(item, "=")
             identifier = self._read_identifier(name_text.strip(), 0)
             if identifier is None:
-                continue
+                raise self._invalid_syntax_error(segment, item, context="enum member")
             name, end = identifier
             if name_text[end:].strip():
-                continue
+                raise self._invalid_syntax_error(segment, item, context="enum member")
             constants.append(
                 CEnumerator(
                     name=name,
@@ -2402,7 +2425,6 @@ def _parse_declaration(
             not text
             or text.startswith("_Static_assert")
             or self._has_unsupported_declaration_marker(text)
-            or _looks_like_cxx_declaration(text)
         ):
             return [], [], [], []
 
@@ -2418,13 +2440,10 @@ def _unsupported_declaration_diagnostic(self, segment: CTopLevelSegment) -> CDia
         if not text:
             return None
 
-        kind = "unsupported_declaration"
-        message = "Unsupported C declaration form."
+        kind = ""
+        message = ""
 
-        if _looks_like_cxx_declaration(text):
-            kind = "cxx_declaration"
-            message = "C++ declaration syntax is not supported by the C parser."
-        elif text.startswith("struct "):
+        if text.startswith("struct "):
             kind = "struct_definition"
             message = "Struct definitions are not supported yet."
         elif text.startswith("union "):
@@ -2445,6 +2464,8 @@ def _unsupported_declaration_diagnostic(self, segment: CTopLevelSegment) -> CDia
         elif "{" in text or "}" in text:
             kind = "brace_declaration"
             message = "Unsupported declaration containing braces."
+        else:
+            return None
 
         return CDiagnostic(
             code="C_UNSUPPORTED_DECLARATION",
@@ -2521,12 +2542,10 @@ def _parse_translation_unit(
                     )
                 )
                 continue
-            if _looks_like_cxx_declaration(segment.text):
-                unsupported = self._unsupported_declaration_diagnostic(segment)
-                if unsupported is not None:
-                    diagnostics.append(unsupported)
-                continue
-            tag_definition = self._parse_tag_definition(segment)
+            try:
+                tag_definition = self._parse_tag_definition(segment)
+            except _InvalidCGrammarSyntax as error:
+                raise self._invalid_syntax_error(segment, str(error), context="nested declaration") from None
             if tag_definition is not None:
                 aggregate, parsed_functions, parsed_typedefs, parsed_variables, parsed_diagnostics = tag_definition
                 if isinstance(aggregate, CStruct):
@@ -2551,6 +2570,8 @@ def _parse_translation_unit(
                 except _UnsupportedDeclaratorSyntax as error:
                     diagnostics.append(self._declarator_diagnostic(segment, str(error)))
                     continue
+                except _InvalidCGrammarSyntax as error:
+                    raise self._invalid_syntax_error(segment, str(error), context="function declaration") from None
                 if function is not None:
                     function.condition_set = condition_sets.get(
                         segment.original_start_line,
@@ -2562,17 +2583,21 @@ def _parse_translation_unit(
                 unsupported = self._unsupported_declaration_diagnostic(segment)
                 if unsupported is not None:
                     diagnostics.append(unsupported)
-                continue
+                    continue
+                raise self._invalid_syntax_error(segment, segment.text, context="top level")
             forward_tag = self._forward_tag(segment)
             if isinstance(forward_tag, CStruct):
                 structs.append(forward_tag)
                 continue
             if isinstance(forward_tag, CUnion):
                 unions.append(forward_tag)
                 continue
-            parsed_functions, parsed_typedefs, parsed_variables, declarator_diagnostics = self._parse_declaration(
-                segment
-            )
+            try:
+                parsed_functions, parsed_typedefs, parsed_variables, declarator_diagnostics = self._parse_declaration(
+                    segment
+                )
+            except _InvalidCGrammarSyntax as error:
+                raise self._invalid_syntax_error(segment, str(error), context="declaration") from None
             functions.extend(parsed_functions)
             for function in parsed_functions:
                 function.condition_set = condition_sets.get(
@@ -2593,6 +2618,8 @@ def _parse_translation_unit(
                 unsupported = self._unsupported_declaration_diagnostic(segment)
                 if unsupported is not None:
                     diagnostics.append(unsupported)
+                else:
+                    raise self._invalid_syntax_error(segment, segment.text, context="top level")
 
         return functions, structs, unions, enums, typedefs, variables, diagnostics