[mypyc] Support new syntax for generic functions and classes (PEP 695)

mypy/nodes.py

@@ -2535,8 +2535,9 @@ def __init__(
         default: mypy.types.Type,
         variance: int = INVARIANT,
         is_new_style: bool = False,
+        line: int = -1,
     ) -> None:
-        super().__init__()
+        super().__init__(line=line)
         self._name = name
         self._fullname = fullname
         self.upper_bound = upper_bound
@@ -2582,8 +2583,9 @@ def __init__(
         default: mypy.types.Type,
         variance: int = INVARIANT,
         is_new_style: bool = False,
+        line: int = -1,
     ) -> None:
-        super().__init__(name, fullname, upper_bound, default, variance, is_new_style)
+        super().__init__(name, fullname, upper_bound, default, variance, is_new_style, line=line)
         self.values = values
 
     def accept(self, visitor: ExpressionVisitor[T]) -> T:
@@ -2661,8 +2663,9 @@ def __init__(
         default: mypy.types.Type,
         variance: int = INVARIANT,
         is_new_style: bool = False,
+        line: int = -1,
     ) -> None:
-        super().__init__(name, fullname, upper_bound, default, variance, is_new_style)
+        super().__init__(name, fullname, upper_bound, default, variance, is_new_style, line=line)
         self.tuple_fallback = tuple_fallback
 
     def accept(self, visitor: ExpressionVisitor[T]) -> T:

mypy/semanal.py

@@ -1689,7 +1689,7 @@ def push_type_args(
         self.scope_stack.append(SCOPE_ANNOTATION)
         tvs: list[tuple[str, TypeVarLikeExpr]] = []
         for p in type_args:
-            tv = self.analyze_type_param(p)
+            tv = self.analyze_type_param(p, context)
             if tv is None:
                 return None
             tvs.append((p.name, tv))
@@ -1712,7 +1712,9 @@ def is_defined_type_param(self, name: str) -> bool:
                     return True
         return False
 
-    def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None:
+    def analyze_type_param(
+        self, type_param: TypeParam, context: Context
+    ) -> TypeVarLikeExpr | None:
         fullname = self.qualified_name(type_param.name)
         if type_param.upper_bound:
             upper_bound = self.anal_type(type_param.upper_bound)
@@ -1737,6 +1739,7 @@ def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None:
                 default=default,
                 variance=VARIANCE_NOT_READY,
                 is_new_style=True,
+                line=context.line,
             )
         elif type_param.kind == PARAM_SPEC_KIND:
             return ParamSpecExpr(
@@ -1745,6 +1748,7 @@ def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None:
                 upper_bound=upper_bound,
                 default=default,
                 is_new_style=True,
+                line=context.line,
             )
         else:
             assert type_param.kind == TYPE_VAR_TUPLE_KIND
@@ -1757,6 +1761,7 @@ def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None:
                 tuple_fallback=tuple_fallback,
                 default=default,
                 is_new_style=True,
+                line=context.line,
             )
 
     def pop_type_args(self, type_args: list[TypeParam] | None) -> None:

mypyc/codegen/emitfunc.py

@@ -6,14 +6,22 @@
 
 from mypyc.analysis.blockfreq import frequently_executed_blocks
 from mypyc.codegen.emit import DEBUG_ERRORS, Emitter, TracebackAndGotoHandler, c_array_initializer
-from mypyc.common import MODULE_PREFIX, NATIVE_PREFIX, REG_PREFIX, STATIC_PREFIX, TYPE_PREFIX
+from mypyc.common import (
+    MODULE_PREFIX,
+    NATIVE_PREFIX,
+    REG_PREFIX,
+    STATIC_PREFIX,
+    TYPE_PREFIX,
+    TYPE_VAR_PREFIX,
+)
 from mypyc.ir.class_ir import ClassIR
 from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD, FuncDecl, FuncIR, all_values
 from mypyc.ir.ops import (
     ERR_FALSE,
     NAMESPACE_MODULE,
     NAMESPACE_STATIC,
     NAMESPACE_TYPE,
+    NAMESPACE_TYPE_VAR,
     Assign,
     AssignMulti,
     BasicBlock,
@@ -477,6 +485,7 @@ def visit_set_attr(self, op: SetAttr) -> None:
         NAMESPACE_STATIC: STATIC_PREFIX,
         NAMESPACE_TYPE: TYPE_PREFIX,
         NAMESPACE_MODULE: MODULE_PREFIX,
+        NAMESPACE_TYPE_VAR: TYPE_VAR_PREFIX,
     }
 
     def visit_load_static(self, op: LoadStatic) -> None:

mypyc/codegen/emitmodule.py

@@ -41,6 +41,7 @@
     PREFIX,
     RUNTIME_C_FILES,
     TOP_LEVEL_NAME,
+    TYPE_VAR_PREFIX,
     shared_lib_name,
     short_id_from_name,
     use_vectorcall,
@@ -590,6 +591,7 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]:
             self.declare_finals(module_name, module.final_names, declarations)
             for cl in module.classes:
                 generate_class_type_decl(cl, emitter, ext_declarations, declarations)
+            self.declare_type_vars(module_name, module.type_var_names, declarations)
             for fn in module.functions:
                 generate_function_declaration(fn, declarations)
 
@@ -1063,6 +1065,15 @@ def declare_static_pyobject(self, identifier: str, emitter: Emitter) -> None:
         symbol = emitter.static_name(identifier, None)
         self.declare_global("PyObject *", symbol)
 
+    def declare_type_vars(self, module: str, type_var_names: list[str], emitter: Emitter) -> None:
+        for name in type_var_names:
+            static_name = emitter.static_name(name, module, prefix=TYPE_VAR_PREFIX)
+            emitter.context.declarations[static_name] = HeaderDeclaration(
+                f"PyObject *{static_name};",
+                [f"PyObject *{static_name} = NULL;"],
+                needs_export=False,
+            )
+
 
 def sort_classes(classes: list[tuple[str, ClassIR]]) -> list[tuple[str, ClassIR]]:
     mod_name = {ir: name for name, ir in classes}

mypyc/common.py

@@ -13,6 +13,7 @@
 STATIC_PREFIX: Final = "CPyStatic_"  # Static variables (for literals etc.)
 TYPE_PREFIX: Final = "CPyType_"  # Type object struct
 MODULE_PREFIX: Final = "CPyModule_"  # Cached modules
+TYPE_VAR_PREFIX: Final = "CPyTypeVar_"  # Type variables when using new-style Python 3.12 syntax
 ATTR_PREFIX: Final = "_"  # Attributes
 
 ENV_ATTR_NAME: Final = "__mypyc_env__"

mypyc/ir/module_ir.py

@@ -21,12 +21,17 @@ def __init__(
         functions: list[FuncIR],
         classes: list[ClassIR],
         final_names: list[tuple[str, RType]],
+        type_var_names: list[str],
     ) -> None:
         self.fullname = fullname
         self.imports = imports.copy()
         self.functions = functions
         self.classes = classes
         self.final_names = final_names
+        # Names of C statics used for Python 3.12 type variable objects.
+        # These are only visible in the module that defined them, so no need
+        # to serialize.
+        self.type_var_names = type_var_names
 
     def serialize(self) -> JsonDict:
         return {
@@ -45,6 +50,7 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> ModuleIR:
             [ctx.functions[FuncDecl.get_id_from_json(f)] for f in data["functions"]],
             [ClassIR.deserialize(c, ctx) for c in data["classes"]],
             [(k, deserialize_type(t, ctx)) for k, t in data["final_names"]],
+            [],
         )
 
 

mypyc/ir/ops.py

@@ -789,6 +789,9 @@ def accept(self, visitor: OpVisitor[T]) -> T:
 # Namespace for modules
 NAMESPACE_MODULE: Final = "module"
 
+# Namespace for Python 3.12 type variable objects (implicitly created TypeVar instances, etc.)
+NAMESPACE_TYPE_VAR: Final = "typevar"
+
 
 class LoadStatic(RegisterOp):
     """Load a static name (name :: static).

mypyc/irbuild/builder.py

@@ -69,6 +69,7 @@
 from mypyc.ir.func_ir import INVALID_FUNC_DEF, FuncDecl, FuncIR, FuncSignature, RuntimeArg
 from mypyc.ir.ops import (
     NAMESPACE_MODULE,
+    NAMESPACE_TYPE_VAR,
     Assign,
     BasicBlock,
     Branch,
@@ -179,6 +180,7 @@ def __init__(
         self.function_names: set[tuple[str | None, str]] = set()
         self.classes: list[ClassIR] = []
         self.final_names: list[tuple[str, RType]] = []
+        self.type_var_names: list[str] = []
         self.callable_class_names: set[str] = set()
         self.options = options
 
@@ -541,6 +543,21 @@ def load_final_static(
             error_msg=f'value for final name "{error_name}" was not set',
         )
 
+    def init_type_var(self, value: Value, name: str, line: int) -> None:
+        unique_name = name + "___" + str(line)
+        self.type_var_names.append(unique_name)
+        self.add(InitStatic(value, unique_name, self.module_name, namespace=NAMESPACE_TYPE_VAR))
+
+    def load_type_var(self, name: str, line: int) -> Value:
+        return self.add(
+            LoadStatic(
+                object_rprimitive,
+                name + "___" + str(line),
+                self.module_name,
+                namespace=NAMESPACE_TYPE_VAR,
+            )
+        )
+
     def load_literal_value(self, val: int | str | bytes | float | complex | bool) -> Value:
         """Load value of a final name, class-level attribute, or constant folded expression."""
         if isinstance(val, bool):

mypyc/irbuild/classdef.py

@@ -7,6 +7,9 @@
 from typing import Callable, Final
 
 from mypy.nodes import (
+    PARAM_SPEC_KIND,
+    TYPE_VAR_KIND,
+    TYPE_VAR_TUPLE_KIND,
     AssignmentStmt,
     CallExpr,
     ClassDef,
@@ -22,6 +25,7 @@
     StrExpr,
     TempNode,
     TypeInfo,
+    TypeParam,
     is_class_var,
 )
 from mypy.types import ENUM_REMOVED_PROPS, Instance, RawExpressionType, get_proper_type
@@ -63,9 +67,16 @@
 )
 from mypyc.irbuild.util import dataclass_type, get_func_def, is_constant, is_dataclass_decorator
 from mypyc.primitives.dict_ops import dict_new_op, dict_set_item_op
-from mypyc.primitives.generic_ops import py_hasattr_op, py_setattr_op
+from mypyc.primitives.generic_ops import (
+    iter_op,
+    next_op,
+    py_get_item_op,
+    py_hasattr_op,
+    py_setattr_op,
+)
 from mypyc.primitives.misc_ops import (
     dataclass_sleight_of_hand,
+    import_op,
     not_implemented_op,
     py_calc_meta_op,
     pytype_from_template_op,
@@ -405,8 +416,14 @@ def get_type_annotation(self, stmt: AssignmentStmt) -> TypeInfo | None:
 def allocate_class(builder: IRBuilder, cdef: ClassDef) -> Value:
     # OK AND NOW THE FUN PART
     base_exprs = cdef.base_type_exprs + cdef.removed_base_type_exprs
-    if base_exprs:
-        bases = [builder.accept(x) for x in base_exprs]
+    new_style_type_args = cdef.type_args
+    if new_style_type_args:
+        bases = [make_generic_base_class(builder, cdef.fullname, new_style_type_args, cdef.line)]
+    else:
+        bases = []
+
+    if base_exprs or new_style_type_args:
+        bases.extend([builder.accept(x) for x in base_exprs])
         tp_bases = builder.new_tuple(bases, cdef.line)
     else:
         tp_bases = builder.add(LoadErrorValue(object_rprimitive, is_borrowed=True))
@@ -453,6 +470,45 @@ def allocate_class(builder: IRBuilder, cdef: ClassDef) -> Value:
     return tp
 
 
+def make_generic_base_class(
+    builder: IRBuilder, fullname: str, type_args: list[TypeParam], line: int
+) -> Value:
+    """Construct Generic[...] base class object for a new-style generic class (Python 3.12)."""
+    mod = builder.call_c(import_op, [builder.load_str("_typing")], line)
+    tvs = []
+    type_var_imported: Value | None = None
+    for type_param in type_args:
+        unpack = False
+        if type_param.kind == TYPE_VAR_KIND:
+            if type_var_imported:
+                # Reuse previously imported value as a minor optimization
+                tvt = type_var_imported
+            else:
+                tvt = builder.py_get_attr(mod, "TypeVar", line)
+                type_var_imported = tvt
+        elif type_param.kind == TYPE_VAR_TUPLE_KIND:
+            tvt = builder.py_get_attr(mod, "TypeVarTuple", line)
+            unpack = True
+        else:
+            assert type_param.kind == PARAM_SPEC_KIND
+            tvt = builder.py_get_attr(mod, "ParamSpec", line)
+        tv = builder.py_call(tvt, [builder.load_str(type_param.name)], line)
+        builder.init_type_var(tv, type_param.name, line)
+        if unpack:
+            # Evaluate *Ts for a TypeVarTuple
+            it = builder.call_c(iter_op, [tv], line)
+            tv = builder.call_c(next_op, [it], line)
+        tvs.append(tv)
+    gent = builder.py_get_attr(mod, "Generic", line)
+    if len(tvs) == 1:
+        arg = tvs[0]
+    else:
+        arg = builder.new_tuple(tvs, line)
+
+    base = builder.call_c(py_get_item_op, [gent, arg], line)
+    return base
+
+
 # Mypy uses these internally as base classes of TypedDict classes. These are
 # lies and don't have any runtime equivalent.
 MAGIC_TYPED_DICT_CLASSES: Final[tuple[str, ...]] = (

mypyc/irbuild/expression.py

@@ -44,6 +44,7 @@
     TupleExpr,
     TypeApplication,
     TypeInfo,
+    TypeVarLikeExpr,
     UnaryExpr,
     Var,
 )
@@ -106,6 +107,10 @@
 
 
 def transform_name_expr(builder: IRBuilder, expr: NameExpr) -> Value:
+    if isinstance(expr.node, TypeVarLikeExpr) and expr.node.is_new_style:
+        # Reference to Python 3.12 implicit TypeVar/TupleVarTuple/... object.
+        # These are stored in C statics and not visible in Python namespaces.
+        return builder.load_type_var(expr.node.name, expr.node.line)
     if expr.node is None:
         builder.add(
             RaiseStandardError(

mypyc/irbuild/main.py

@@ -99,6 +99,7 @@ def build_ir(
             builder.functions,
             builder.classes,
             builder.final_names,
+            builder.type_var_names,
         )
         result[module.fullname] = module_ir
         class_irs.extend(builder.classes)

mypyc/primitives/generic_ops.py

@@ -178,7 +178,7 @@
 )
 
 # obj1[obj2]
-method_op(
+py_get_item_op = method_op(
     name="__getitem__",
     arg_types=[object_rprimitive, object_rprimitive],
     return_type=object_rprimitive,