Flag mapped types with circular property types and do not attempt to print them structurally

src/compiler/checker.ts

@@ -4572,8 +4572,8 @@ namespace ts {
                 }
 
                 function createTypeNodeFromObjectType(type: ObjectType): TypeNode {
-                    if (isGenericMappedType(type)) {
-                        return createMappedTypeNodeFromType(type);
+                    if (isGenericMappedType(type) || (type as MappedType).containsError) {
+                        return createMappedTypeNodeFromType(type as MappedType);
                     }
 
                     const resolved = resolveStructuredTypeMembers(type);
@@ -10309,6 +10309,7 @@ namespace ts {
         function getTypeOfMappedSymbol(symbol: MappedSymbol) {
             if (!symbol.type) {
                 if (!pushTypeResolution(symbol, TypeSystemPropertyName.Type)) {
+                    symbol.mappedType.containsError = true;
                     return errorType;
                 }
                 const templateType = getTemplateTypeFromMappedType(<MappedType>symbol.mappedType.target || symbol.mappedType);

src/compiler/types.ts

@@ -5168,6 +5168,7 @@ namespace ts {
         templateType?: Type;
         modifiersType?: Type;
         resolvedApparentType?: Type;
+        containsError?: boolean;
     }
 
     export interface EvolvingArrayType extends ObjectType {

tests/baselines/reference/recursivelyExpandingUnionNoStackoverflow.errors.txt

@@ -0,0 +1,12 @@
+tests/cases/compiler/recursivelyExpandingUnionNoStackoverflow.ts(3,10): error TS2589: Type instantiation is excessively deep and possibly infinite.
+tests/cases/compiler/recursivelyExpandingUnionNoStackoverflow.ts(3,10): error TS2615: Type of property 'M' circularly references itself in mapped type '{ [P in "M"]: any; }'.
+
+
+==== tests/cases/compiler/recursivelyExpandingUnionNoStackoverflow.ts (2 errors) ====
+    type N<T, K extends string> = T | { [P in K]: N<T, K> }[K];
+
+    type M = N<number, "M">;
+             ~~~~~~~~~~~~~~
+!!! error TS2589: Type instantiation is excessively deep and possibly infinite.
+             ~~~~~~~~~~~~~~
+!!! error TS2615: Type of property 'M' circularly references itself in mapped type '{ [P in "M"]: any; }'.

tests/baselines/reference/recursivelyExpandingUnionNoStackoverflow.js

@@ -0,0 +1,6 @@
+//// [recursivelyExpandingUnionNoStackoverflow.ts]
+type N<T, K extends string> = T | { [P in K]: N<T, K> }[K];
+
+type M = N<number, "M">;
+
+//// [recursivelyExpandingUnionNoStackoverflow.js]

tests/baselines/reference/recursivelyExpandingUnionNoStackoverflow.symbols

@@ -0,0 +1,17 @@
+=== tests/cases/compiler/recursivelyExpandingUnionNoStackoverflow.ts ===
+type N<T, K extends string> = T | { [P in K]: N<T, K> }[K];
+>N : Symbol(N, Decl(recursivelyExpandingUnionNoStackoverflow.ts, 0, 0))
+>T : Symbol(T, Decl(recursivelyExpandingUnionNoStackoverflow.ts, 0, 7))
+>K : Symbol(K, Decl(recursivelyExpandingUnionNoStackoverflow.ts, 0, 9))
+>T : Symbol(T, Decl(recursivelyExpandingUnionNoStackoverflow.ts, 0, 7))
+>P : Symbol(P, Decl(recursivelyExpandingUnionNoStackoverflow.ts, 0, 37))
+>K : Symbol(K, Decl(recursivelyExpandingUnionNoStackoverflow.ts, 0, 9))
+>N : Symbol(N, Decl(recursivelyExpandingUnionNoStackoverflow.ts, 0, 0))
+>T : Symbol(T, Decl(recursivelyExpandingUnionNoStackoverflow.ts, 0, 7))
+>K : Symbol(K, Decl(recursivelyExpandingUnionNoStackoverflow.ts, 0, 9))
+>K : Symbol(K, Decl(recursivelyExpandingUnionNoStackoverflow.ts, 0, 9))
+
+type M = N<number, "M">;
+>M : Symbol(M, Decl(recursivelyExpandingUnionNoStackoverflow.ts, 0, 59))
+>N : Symbol(N, Decl(recursivelyExpandingUnionNoStackoverflow.ts, 0, 0))
+

tests/baselines/reference/recursivelyExpandingUnionNoStackoverflow.types

@@ -0,0 +1,7 @@
+=== tests/cases/compiler/recursivelyExpandingUnionNoStackoverflow.ts ===
+type N<T, K extends string> = T | { [P in K]: N<T, K> }[K];
+>N : N<T, K>
+
+type M = N<number, "M">;
+>M : any
+

tests/cases/compiler/recursivelyExpandingUnionNoStackoverflow.ts

@@ -0,0 +1,3 @@
+type N<T, K extends string> = T | { [P in K]: N<T, K> }[K];
+
+type M = N<number, "M">;