Support most use cases for PEP 612 with Generic (#817)

Author: Fidget-Spinner

typing_extensions/README.rst

@@ -90,8 +90,8 @@ issues when mixing the differing implementations of modified classes.
 
 Certain types have incorrect runtime behavior due to limitations of older
 versions of the typing module.  For example, ``ParamSpec`` and ``Concatenate``
-will not work with ``get_args``, ``get_origin`` or user-defined ``Generic``\ s
-because they need to be lists to work with older versions of ``Callable``.
+will not work with ``get_args``, ``get_origin``. Certain PEP 612 special cases
+in user-defined ``Generic``\ s are also not available.
 These types are only guaranteed to work for static type checking.
 
 Running tests

typing_extensions/src_py3/test_typing_extensions.py

@@ -2012,34 +2012,71 @@ def test_valid_uses(self):
         P = ParamSpec('P')
         T = TypeVar('T')
         C1 = typing.Callable[P, int]
+        # Callable in Python 3.5.2 might be bugged when collecting __args__.
+        # https://github.com/python/cpython/blob/91185fe0284a04162e0b3425b53be49bdbfad67d/Lib/typing.py#L1026
+        PY_3_5_2 = sys.version_info[:3] == (3, 5, 2)
+        if not PY_3_5_2:
+            self.assertEqual(C1.__args__, (P, int))
+            self.assertEqual(C1.__parameters__, (P,))
         C2 = typing.Callable[P, T]
+        if not PY_3_5_2:
+            self.assertEqual(C2.__args__, (P, T))
+            self.assertEqual(C2.__parameters__, (P, T))
 
-        # Note: no tests for Callable.__args__ and Callable.__parameters__ here
-        # because pre-3.10 Callable sees ParamSpec as a plain list, not a
-        # TypeVar.
 
         # Test collections.abc.Callable too.
         if sys.version_info[:2] >= (3, 9):
+            # Note: no tests for Callable.__parameters__ here
+            # because types.GenericAlias Callable is hardcoded to search
+            # for tp_name "TypeVar" in C.  This was changed in 3.10.
             C3 = collections.abc.Callable[P, int]
+            self.assertEqual(C3.__args__, (P, int))
             C4 = collections.abc.Callable[P, T]
+            self.assertEqual(C4.__args__, (P, T))
 
         # ParamSpec instances should also have args and kwargs attributes.
-        self.assertIn('args', dir(P))
-        self.assertIn('kwargs', dir(P))
+        # Note: not in dir(P) because of __class__ hacks
+        self.assertTrue(hasattr(P, 'args'))
+        self.assertTrue(hasattr(P, 'kwargs'))
 
     def test_args_kwargs(self):
         P = ParamSpec('P')
-        self.assertIn('args', dir(P))
-        self.assertIn('kwargs', dir(P))
+        # Note: not in dir(P) because of __class__ hacks
+        self.assertTrue(hasattr(P, 'args'))
+        self.assertTrue(hasattr(P, 'kwargs'))
         self.assertIsInstance(P.args, ParamSpecArgs)
         self.assertIsInstance(P.kwargs, ParamSpecKwargs)
         self.assertIs(P.args.__origin__, P)
         self.assertIs(P.kwargs.__origin__, P)
         self.assertEqual(repr(P.args), "P.args")
         self.assertEqual(repr(P.kwargs), "P.kwargs")
 
-    # Note: ParamSpec doesn't work for pre-3.10 user-defined Generics due
-    # to type checks inside Generic.
+    def test_user_generics(self):
+        T = TypeVar("T")
+        P = ParamSpec("P")
+        P_2 = ParamSpec("P_2")
+
+        class X(Generic[T, P]):
+            pass
+
+        G1 = X[int, P_2]
+        self.assertEqual(G1.__args__, (int, P_2))
+        self.assertEqual(G1.__parameters__, (P_2,))
+
+        G2 = X[int, Concatenate[int, P_2]]
+        self.assertEqual(G2.__args__, (int, Concatenate[int, P_2]))
+        self.assertEqual(G2.__parameters__, (P_2,))
+
+        # The following are some valid uses cases in PEP 612 that don't work:
+        # These do not work in 3.9, _type_check blocks the list and ellipsis.
+        # G3 = X[int, [int, bool]]
+        # G4 = X[int, ...]
+        # G5 = Z[[int, str, bool]]
+        # Not working because this is special-cased in 3.10.
+        # G6 = Z[int, str, bool]
+
+        class Z(Generic[P]):
+            pass
 
     def test_pickle(self):
         global P, P_co, P_contra

typing_extensions/src_py3/typing_extensions.py

@@ -2329,6 +2329,9 @@ def add_two(x: float, y: float) -> float:
         be pickled.
         """
 
+        # Trick Generic __parameters__.
+        __class__ = TypeVar
+
         @property
         def args(self):
             return ParamSpecArgs(self)
@@ -2377,14 +2380,31 @@ def __reduce__(self):
         def __call__(self, *args, **kwargs):
             pass
 
-        # Note: Can't fake ParamSpec as a TypeVar to get it to work
-        # with Generics. ParamSpec isn't an instance of TypeVar in 3.10.
-        # So encouraging code like isinstance(ParamSpec('P'), TypeVar))
-        # will lead to breakage in 3.10.
-        # This also means no accurate __parameters__ for GenericAliases.
+        if not PEP_560:
+            # Only needed in 3.6 and lower.
+            def _get_type_vars(self, tvars):
+                if self not in tvars:
+                    tvars.append(self)
 
 # Inherits from list as a workaround for Callable checks in Python < 3.9.2.
 class _ConcatenateGenericAlias(list):
+
+    # Trick Generic into looking into this for __parameters__.
+    if PEP_560:
+        __class__ = _GenericAlias
+    elif sys.version_info[:3] == (3, 5, 2):
+        __class__ = typing.TypingMeta
+    else:
+        __class__ = typing._TypingBase
+
+    # Flag in 3.8.
+    _special = False
+    # Attribute in 3.6 and earlier.
+    if sys.version_info[:3] == (3, 5, 2):
+        _gorg = typing.GenericMeta
+    else:
+        _gorg = typing.Generic
+
     def __init__(self, origin, args):
         super().__init__(args)
         self.__origin__ = origin
@@ -2399,6 +2419,20 @@ def __repr__(self):
     def __hash__(self):
         return hash((self.__origin__, self.__args__))
 
+    # Hack to get typing._type_check to pass in Generic.
+    def __call__(self, *args, **kwargs):
+        pass
+
+    @property
+    def __parameters__(self):
+        return tuple(tp for tp in self.__args__ if isinstance(tp, (TypeVar, ParamSpec)))
+
+    if not PEP_560:
+        # Only required in 3.6 and lower.
+        def _get_type_vars(self, tvars):
+            if self.__origin__ and self.__parameters__:
+                typing._get_type_vars(self.__parameters__, tvars)
+
 @_tp_cache
 def _concatenate_getitem(self, parameters):
     if parameters == ():