Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[mypyc] Support new syntax for generic functions and classes (PEP 695) #17357

Merged
merged 16 commits into from
Jun 11, 2024
9 changes: 6 additions & 3 deletions mypy/nodes.py
Original file line number Diff line number Diff line change
Expand Up @@ -2535,8 +2535,9 @@ def __init__(
default: mypy.types.Type,
variance: int = INVARIANT,
is_new_style: bool = False,
line: int = -1,
) -> None:
super().__init__()
super().__init__(line=line)
self._name = name
self._fullname = fullname
self.upper_bound = upper_bound
Expand Down Expand Up @@ -2582,8 +2583,9 @@ def __init__(
default: mypy.types.Type,
variance: int = INVARIANT,
is_new_style: bool = False,
line: int = -1,
) -> None:
super().__init__(name, fullname, upper_bound, default, variance, is_new_style)
super().__init__(name, fullname, upper_bound, default, variance, is_new_style, line=line)
self.values = values

def accept(self, visitor: ExpressionVisitor[T]) -> T:
Expand Down Expand Up @@ -2661,8 +2663,9 @@ def __init__(
default: mypy.types.Type,
variance: int = INVARIANT,
is_new_style: bool = False,
line: int = -1,
) -> None:
super().__init__(name, fullname, upper_bound, default, variance, is_new_style)
super().__init__(name, fullname, upper_bound, default, variance, is_new_style, line=line)
self.tuple_fallback = tuple_fallback

def accept(self, visitor: ExpressionVisitor[T]) -> T:
Expand Down
9 changes: 7 additions & 2 deletions mypy/semanal.py
Original file line number Diff line number Diff line change
Expand Up @@ -1689,7 +1689,7 @@ def push_type_args(
self.scope_stack.append(SCOPE_ANNOTATION)
tvs: list[tuple[str, TypeVarLikeExpr]] = []
for p in type_args:
tv = self.analyze_type_param(p)
tv = self.analyze_type_param(p, context)
if tv is None:
return None
tvs.append((p.name, tv))
Expand All @@ -1712,7 +1712,9 @@ def is_defined_type_param(self, name: str) -> bool:
return True
return False

def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None:
def analyze_type_param(
self, type_param: TypeParam, context: Context
) -> TypeVarLikeExpr | None:
fullname = self.qualified_name(type_param.name)
if type_param.upper_bound:
upper_bound = self.anal_type(type_param.upper_bound)
Expand All @@ -1737,6 +1739,7 @@ def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None:
default=default,
variance=VARIANCE_NOT_READY,
is_new_style=True,
line=context.line,
)
elif type_param.kind == PARAM_SPEC_KIND:
return ParamSpecExpr(
Expand All @@ -1745,6 +1748,7 @@ def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None:
upper_bound=upper_bound,
default=default,
is_new_style=True,
line=context.line,
)
else:
assert type_param.kind == TYPE_VAR_TUPLE_KIND
Expand All @@ -1757,6 +1761,7 @@ def analyze_type_param(self, type_param: TypeParam) -> TypeVarLikeExpr | None:
tuple_fallback=tuple_fallback,
default=default,
is_new_style=True,
line=context.line,
)

def pop_type_args(self, type_args: list[TypeParam] | None) -> None:
Expand Down
11 changes: 10 additions & 1 deletion mypyc/codegen/emitfunc.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,22 @@

from mypyc.analysis.blockfreq import frequently_executed_blocks
from mypyc.codegen.emit import DEBUG_ERRORS, Emitter, TracebackAndGotoHandler, c_array_initializer
from mypyc.common import MODULE_PREFIX, NATIVE_PREFIX, REG_PREFIX, STATIC_PREFIX, TYPE_PREFIX
from mypyc.common import (
MODULE_PREFIX,
NATIVE_PREFIX,
REG_PREFIX,
STATIC_PREFIX,
TYPE_PREFIX,
TYPE_VAR_PREFIX,
)
from mypyc.ir.class_ir import ClassIR
from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD, FuncDecl, FuncIR, all_values
from mypyc.ir.ops import (
ERR_FALSE,
NAMESPACE_MODULE,
NAMESPACE_STATIC,
NAMESPACE_TYPE,
NAMESPACE_TYPE_VAR,
Assign,
AssignMulti,
BasicBlock,
Expand Down Expand Up @@ -477,6 +485,7 @@ def visit_set_attr(self, op: SetAttr) -> None:
NAMESPACE_STATIC: STATIC_PREFIX,
NAMESPACE_TYPE: TYPE_PREFIX,
NAMESPACE_MODULE: MODULE_PREFIX,
NAMESPACE_TYPE_VAR: TYPE_VAR_PREFIX,
}

def visit_load_static(self, op: LoadStatic) -> None:
Expand Down
11 changes: 11 additions & 0 deletions mypyc/codegen/emitmodule.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
PREFIX,
RUNTIME_C_FILES,
TOP_LEVEL_NAME,
TYPE_VAR_PREFIX,
shared_lib_name,
short_id_from_name,
use_vectorcall,
Expand Down Expand Up @@ -590,6 +591,7 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]:
self.declare_finals(module_name, module.final_names, declarations)
for cl in module.classes:
generate_class_type_decl(cl, emitter, ext_declarations, declarations)
self.declare_type_vars(module_name, module.type_var_names, declarations)
for fn in module.functions:
generate_function_declaration(fn, declarations)

Expand Down Expand Up @@ -1063,6 +1065,15 @@ def declare_static_pyobject(self, identifier: str, emitter: Emitter) -> None:
symbol = emitter.static_name(identifier, None)
self.declare_global("PyObject *", symbol)

def declare_type_vars(self, module: str, type_var_names: list[str], emitter: Emitter) -> None:
for name in type_var_names:
static_name = emitter.static_name(name, module, prefix=TYPE_VAR_PREFIX)
emitter.context.declarations[static_name] = HeaderDeclaration(
f"PyObject *{static_name};",
[f"PyObject *{static_name} = NULL;"],
needs_export=False,
)


def sort_classes(classes: list[tuple[str, ClassIR]]) -> list[tuple[str, ClassIR]]:
mod_name = {ir: name for name, ir in classes}
Expand Down
1 change: 1 addition & 0 deletions mypyc/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
STATIC_PREFIX: Final = "CPyStatic_" # Static variables (for literals etc.)
TYPE_PREFIX: Final = "CPyType_" # Type object struct
MODULE_PREFIX: Final = "CPyModule_" # Cached modules
TYPE_VAR_PREFIX: Final = "CPyTypeVar_" # Type variables when using new-style Python 3.12 syntax
ATTR_PREFIX: Final = "_" # Attributes

ENV_ATTR_NAME: Final = "__mypyc_env__"
Expand Down
6 changes: 6 additions & 0 deletions mypyc/ir/module_ir.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,17 @@ def __init__(
functions: list[FuncIR],
classes: list[ClassIR],
final_names: list[tuple[str, RType]],
type_var_names: list[str],
) -> None:
self.fullname = fullname
self.imports = imports.copy()
self.functions = functions
self.classes = classes
self.final_names = final_names
# Names of C statics used for Python 3.12 type variable objects.
# These are only visible in the module that defined them, so no need
# to serialize.
self.type_var_names = type_var_names

def serialize(self) -> JsonDict:
return {
Expand All @@ -45,6 +50,7 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> ModuleIR:
[ctx.functions[FuncDecl.get_id_from_json(f)] for f in data["functions"]],
[ClassIR.deserialize(c, ctx) for c in data["classes"]],
[(k, deserialize_type(t, ctx)) for k, t in data["final_names"]],
[],
)


Expand Down
3 changes: 3 additions & 0 deletions mypyc/ir/ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -789,6 +789,9 @@ def accept(self, visitor: OpVisitor[T]) -> T:
# Namespace for modules
NAMESPACE_MODULE: Final = "module"

# Namespace for Python 3.12 type variable objects (implicitly created TypeVar instances, etc.)
NAMESPACE_TYPE_VAR: Final = "typevar"


class LoadStatic(RegisterOp):
"""Load a static name (name :: static).
Expand Down
17 changes: 17 additions & 0 deletions mypyc/irbuild/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@
from mypyc.ir.func_ir import INVALID_FUNC_DEF, FuncDecl, FuncIR, FuncSignature, RuntimeArg
from mypyc.ir.ops import (
NAMESPACE_MODULE,
NAMESPACE_TYPE_VAR,
Assign,
BasicBlock,
Branch,
Expand Down Expand Up @@ -179,6 +180,7 @@ def __init__(
self.function_names: set[tuple[str | None, str]] = set()
self.classes: list[ClassIR] = []
self.final_names: list[tuple[str, RType]] = []
self.type_var_names: list[str] = []
self.callable_class_names: set[str] = set()
self.options = options

Expand Down Expand Up @@ -541,6 +543,21 @@ def load_final_static(
error_msg=f'value for final name "{error_name}" was not set',
)

def init_type_var(self, value: Value, name: str, line: int) -> None:
unique_name = name + "___" + str(line)
self.type_var_names.append(unique_name)
self.add(InitStatic(value, unique_name, self.module_name, namespace=NAMESPACE_TYPE_VAR))

def load_type_var(self, name: str, line: int) -> Value:
return self.add(
LoadStatic(
object_rprimitive,
name + "___" + str(line),
self.module_name,
namespace=NAMESPACE_TYPE_VAR,
)
)

def load_literal_value(self, val: int | str | bytes | float | complex | bool) -> Value:
"""Load value of a final name, class-level attribute, or constant folded expression."""
if isinstance(val, bool):
Expand Down
62 changes: 59 additions & 3 deletions mypyc/irbuild/classdef.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@
from typing import Callable, Final

from mypy.nodes import (
PARAM_SPEC_KIND,
TYPE_VAR_KIND,
TYPE_VAR_TUPLE_KIND,
AssignmentStmt,
CallExpr,
ClassDef,
Expand All @@ -22,6 +25,7 @@
StrExpr,
TempNode,
TypeInfo,
TypeParam,
is_class_var,
)
from mypy.types import ENUM_REMOVED_PROPS, Instance, RawExpressionType, get_proper_type
Expand Down Expand Up @@ -63,9 +67,16 @@
)
from mypyc.irbuild.util import dataclass_type, get_func_def, is_constant, is_dataclass_decorator
from mypyc.primitives.dict_ops import dict_new_op, dict_set_item_op
from mypyc.primitives.generic_ops import py_hasattr_op, py_setattr_op
from mypyc.primitives.generic_ops import (
iter_op,
next_op,
py_get_item_op,
py_hasattr_op,
py_setattr_op,
)
from mypyc.primitives.misc_ops import (
dataclass_sleight_of_hand,
import_op,
not_implemented_op,
py_calc_meta_op,
pytype_from_template_op,
Expand Down Expand Up @@ -405,8 +416,14 @@ def get_type_annotation(self, stmt: AssignmentStmt) -> TypeInfo | None:
def allocate_class(builder: IRBuilder, cdef: ClassDef) -> Value:
# OK AND NOW THE FUN PART
base_exprs = cdef.base_type_exprs + cdef.removed_base_type_exprs
if base_exprs:
bases = [builder.accept(x) for x in base_exprs]
new_style_type_args = cdef.type_args
if new_style_type_args:
bases = [make_generic_base_class(builder, cdef.fullname, new_style_type_args, cdef.line)]
else:
bases = []

if base_exprs or new_style_type_args:
bases.extend([builder.accept(x) for x in base_exprs])
tp_bases = builder.new_tuple(bases, cdef.line)
else:
tp_bases = builder.add(LoadErrorValue(object_rprimitive, is_borrowed=True))
Expand Down Expand Up @@ -453,6 +470,45 @@ def allocate_class(builder: IRBuilder, cdef: ClassDef) -> Value:
return tp


def make_generic_base_class(
builder: IRBuilder, fullname: str, type_args: list[TypeParam], line: int
) -> Value:
"""Construct Generic[...] base class object for a new-style generic class (Python 3.12)."""
mod = builder.call_c(import_op, [builder.load_str("_typing")], line)
tvs = []
type_var_imported: Value | None = None
for type_param in type_args:
unpack = False
if type_param.kind == TYPE_VAR_KIND:
if type_var_imported:
# Reuse previously imported value as a minor optimization
tvt = type_var_imported
else:
tvt = builder.py_get_attr(mod, "TypeVar", line)
type_var_imported = tvt
elif type_param.kind == TYPE_VAR_TUPLE_KIND:
tvt = builder.py_get_attr(mod, "TypeVarTuple", line)
unpack = True
else:
assert type_param.kind == PARAM_SPEC_KIND
tvt = builder.py_get_attr(mod, "ParamSpec", line)
tv = builder.py_call(tvt, [builder.load_str(type_param.name)], line)
builder.init_type_var(tv, type_param.name, line)
if unpack:
# Evaluate *Ts for a TypeVarTuple
it = builder.call_c(iter_op, [tv], line)
tv = builder.call_c(next_op, [it], line)
tvs.append(tv)
gent = builder.py_get_attr(mod, "Generic", line)
if len(tvs) == 1:
arg = tvs[0]
else:
arg = builder.new_tuple(tvs, line)

base = builder.call_c(py_get_item_op, [gent, arg], line)
return base


# Mypy uses these internally as base classes of TypedDict classes. These are
# lies and don't have any runtime equivalent.
MAGIC_TYPED_DICT_CLASSES: Final[tuple[str, ...]] = (
Expand Down
5 changes: 5 additions & 0 deletions mypyc/irbuild/expression.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
TupleExpr,
TypeApplication,
TypeInfo,
TypeVarLikeExpr,
UnaryExpr,
Var,
)
Expand Down Expand Up @@ -106,6 +107,10 @@


def transform_name_expr(builder: IRBuilder, expr: NameExpr) -> Value:
if isinstance(expr.node, TypeVarLikeExpr) and expr.node.is_new_style:
# Reference to Python 3.12 implicit TypeVar/TupleVarTuple/... object.
# These are stored in C statics and not visible in Python namespaces.
return builder.load_type_var(expr.node.name, expr.node.line)
if expr.node is None:
builder.add(
RaiseStandardError(
Expand Down
1 change: 1 addition & 0 deletions mypyc/irbuild/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ def build_ir(
builder.functions,
builder.classes,
builder.final_names,
builder.type_var_names,
)
result[module.fullname] = module_ir
class_irs.extend(builder.classes)
Expand Down
2 changes: 1 addition & 1 deletion mypyc/primitives/generic_ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@
)

# obj1[obj2]
method_op(
py_get_item_op = method_op(
name="__getitem__",
arg_types=[object_rprimitive, object_rprimitive],
return_type=object_rprimitive,
Expand Down
Loading
Loading