Skip to content

Add control flow complexity limiter #30931

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 6 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 24 additions & 14 deletions src/compiler/checker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -540,7 +540,8 @@ namespace ts {
let flowLoopStart = 0;
let flowLoopCount = 0;
let sharedFlowCount = 0;
let flowAnalysisDisabled = false;
let flowNodeCount = -1;
let flowAnalysisErrors = false;

const emptyStringType = getLiteralType("");
const zeroType = getLiteralType(0);
Expand Down Expand Up @@ -15949,17 +15950,14 @@ namespace ts {
return false;
}

function reportFlowControlError(node: Node) {
const block = <Block | ModuleBlock | SourceFile>findAncestor(node, isFunctionOrModuleBlock);
const sourceFile = getSourceFileOfNode(node);
const span = getSpanOfTokenAtPosition(sourceFile, block.statements.pos);
diagnostics.add(createFileDiagnostic(sourceFile, span.start, span.length, Diagnostics.The_containing_function_or_module_body_is_too_large_for_control_flow_analysis));
function getContainingBlock(node: Node) {
return <Block | ModuleBlock | SourceFile>findAncestor(node, isFunctionOrModuleBlock);
}

function getFlowTypeOfReference(reference: Node, declaredType: Type, initialType = declaredType, flowContainer?: Node, couldBeUninitialized?: boolean) {
let key: string | undefined;
let flowDepth = 0;
if (flowAnalysisDisabled) {
if (flowAnalysisErrors && getNodeLinks(getContainingBlock(reference)).flags & NodeCheckFlags.FlowAnalysisError) {
return errorType;
}
if (!reference.flowNode || !couldBeUninitialized && !(declaredType.flags & TypeFlags.Narrowable)) {
Expand All @@ -15979,14 +15977,23 @@ namespace ts {
return resultType;

function getTypeAtFlowNode(flow: FlowNode): FlowType {
if (flowDepth === 2000) {
// We have made 2000 recursive invocations. To avoid overflowing the call stack we report an error
// and disable further control flow analysis in the containing function or module body.
flowAnalysisDisabled = true;
reportFlowControlError(reference);
if (flowDepth === 2000 || flowNodeCount === 500000) {
// We have made 2000 recursive invocations or visited 500000 control flow nodes while analyzing
// the containing function or module body, the limit at which we consider the function or module
// body is too complex and disable further control flow analysis.
const block = getContainingBlock(reference);
const nodeLinks = getNodeLinks(block);
if (!(nodeLinks.flags & NodeCheckFlags.FlowAnalysisError)) {
nodeLinks.flags |= NodeCheckFlags.FlowAnalysisError;
const sourceFile = getSourceFileOfNode(block);
const span = getSpanOfTokenAtPosition(sourceFile, block.statements.pos);
diagnostics.add(createFileDiagnostic(sourceFile, span.start, span.length, Diagnostics.The_containing_function_or_module_body_is_too_complex_for_control_flow_analysis));
flowAnalysisErrors = true;
}
return errorType;
}
flowDepth++;
if (flowNodeCount >= 0) flowNodeCount++;
while (true) {
const flags = flow.flags;
if (flags & FlowFlags.Shared) {
Expand Down Expand Up @@ -26026,9 +26033,10 @@ namespace ts {
checkGrammarStatementInAmbientContext(node);
}
if (isFunctionOrModuleBlock(node)) {
const saveFlowAnalysisDisabled = flowAnalysisDisabled;
const saveFlowNodeCount = flowNodeCount;
flowNodeCount = 0;
forEach(node.statements, checkSourceElement);
flowAnalysisDisabled = saveFlowAnalysisDisabled;
flowNodeCount = saveFlowNodeCount;
}
else {
forEach(node.statements, checkSourceElement);
Expand Down Expand Up @@ -28861,7 +28869,9 @@ namespace ts {

function checkSourceFile(node: SourceFile) {
performance.mark("beforeCheck");
flowNodeCount = 0;
checkSourceFileWorker(node);
flowNodeCount = -1;
performance.mark("afterCheck");
performance.measure("Check", "beforeCheck", "afterCheck");
}
Expand Down
2 changes: 1 addition & 1 deletion src/compiler/diagnosticMessages.json
Original file line number Diff line number Diff line change
Expand Up @@ -2044,7 +2044,7 @@
"category": "Error",
"code": 2562
},
"The containing function or module body is too large for control flow analysis.": {
"The containing function or module body is too complex for control flow analysis.": {
"category": "Error",
"code": 2563
},
Expand Down
1 change: 1 addition & 0 deletions src/compiler/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3855,6 +3855,7 @@ namespace ts {
AssignmentsMarked = 0x00800000, // Parameter assignments have been marked
ClassWithConstructorReference = 0x01000000, // Class that contains a binding to its constructor inside of the class body.
ConstructorReferenceInClass = 0x02000000, // Binding to a class constructor inside of the class's body.
FlowAnalysisError = 0x04000000, // Control flow analysis error in this block
}

/* @internal */
Expand Down
163 changes: 163 additions & 0 deletions tests/baselines/reference/complexControlFlowGraph.errors.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
tests/cases/compiler/complexControlFlowGraph.ts(4,3): error TS2563: The containing function or module body is too complex for control flow analysis.


==== tests/cases/compiler/complexControlFlowGraph.ts (1 errors) ====
// Repro from #29926

const foo = function (this: any) {
var a, b, c, d, ab, bc, cd, da, blocks = this.blocks;
~~~
!!! error TS2563: The containing function or module body is too complex for control flow analysis.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Am I crazy for thinking that this input code doesn't actually look that complicated and that we're probably doing too much work somewhere? Like, this is all binary arithmetic - the result is quite literally always a number - I'm not even sure why control flow comes into play.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

eeeyyyeaaahhh; In reusing checkExpression verbatim within control flow, we're missing any fast-path-type-caluclation-only type savings we could get on statements like these. Without even looking at the operands we could pretty easily assume they're of type number. I mean, technically we can't straight up assume, thanks to bigints; but if a binary expression tree contains number literals the result is either an error or number, and likewise for bigint literals (and if both are in a binary expression tree then the result is definitely an error), and the error case should get flagged by the normal statement walk. We could probably get serious savings by skipping calculation of a ton of references in control flow with a simple syntactic walk like that, since it'd mean that for any binary arithmetic expression with numeric literals in it, we'd always be able to avoid following the reference chain.

Copy link
Member

@weswigham weswigham Apr 15, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could even flag such trees during binding (like we do for other subtree containment checks), so all we need to do is check a flag on the node during checking, hmmmmm...

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We're not using checkExpression verbatim, we're using getTypeOfExpression, the intent of which is to obtain the type of an expression with as little work as possible (with checkExpression being the fallback). We already optimize for non-generic call expressions with no overloads and assertion expressions, and we could certainly explore more optimizations.

The particular input code that led to this PR may indeed have many operations for which we can obtain a type just from looking at literals and operators, and, with some work, we might even be able to process the code without hitting the limiter. But of course someone could write the example to use named constants instead, and we'd be right back where we started. So, one way or the other, we need an overall work limiter.

The thing that could really make a difference is better caching of intermediate types. But knowing when it is safe to cache is tricky. In particular, code with back edges from loops is hard.

Copy link
Member

@weswigham weswigham Apr 15, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This code contains zero loops (and zero generics and zero overloads), however, which is why this code, specifically, being problematic to us seems so odd to me. I definitely understand the desire to have some sort of graceful-failure-like hard cap on work done here, however I don't think this example is a good justification for it - the actual complexity of the given code is relatively low; it just currently triggers an unoptimized worst case (deep chains of apparently dependent statements with uncached checks on them). IMO, that itself merits some fixing, at which point this test won't be that useful for testing this limiter. :(

Although maybe this is also just a good example of why we should have a merge-able hierarchical cache; we call getTypeOfExpression with caching disabled in control flow because we don't know if we're in an inferential/contextual context or not, and caching control flow results in those contexts does bad things to the types we produce - if we could simply "always cache" and have call resolution merge/toss the cache layer as appropriate we'd probably not have this problem, at least in this case.

Really, the heart of it is that while I believe the limiter is probably needed for more graceful failure modes, I don't think it's really a "fix" - error'ing on completely valid input code should be a last resort for us, yeah? To be used when resources are truly stretched thin? This is just a poignant example of something we need to optimize, especially when the input code, to a human like myself, seems so simple.

Copy link
Member

@weswigham weswigham Apr 15, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Per our discussion earlier, I've opened #30945 with an optimization that allows us to actually check this input code. With that, we'll have to find another example that actually triggers this limiter for the test to be useful (probably not hard - just strip out all the literals). ❤️

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

#31003 is now a much more complete fix with minimal to no perf impact on non-degenerate cases. ❤️


if (this.first) {
a = blocks[0] - 1;
a = (a << 3) | (a >>> 29);
d = ((a & 0xefcdab89) | (~a & 0x98badcfe)) + blocks[1] + 271733878;
d = (d << 7) | (d >>> 25);
c = ((d & a) | (~d & 0xefcdab89)) + blocks[2] - 1732584194;
c = (c << 11) | (c >>> 21);
b = ((c & d) | (~c & a)) + blocks[3] - 271733879;
b = (b << 19) | (b >>> 13);
} else {
a = this.h0;
b = this.h1;
c = this.h2;
d = this.h3;
a += ((b & c) | (~b & d)) + blocks[0];
a = (a << 3) | (a >>> 29);
d += ((a & b) | (~a & c)) + blocks[1];
d = (d << 7) | (d >>> 25);
c += ((d & a) | (~d & b)) + blocks[2];
c = (c << 11) | (c >>> 21);
b += ((c & d) | (~c & a)) + blocks[3];
b = (b << 19) | (b >>> 13);
}

a += ((b & c) | (~b & d)) + blocks[4];
a = (a << 3) | (a >>> 29);
d += ((a & b) | (~a & c)) + blocks[5];
d = (d << 7) | (d >>> 25);
c += ((d & a) | (~d & b)) + blocks[6];
c = (c << 11) | (c >>> 21);
b += ((c & d) | (~c & a)) + blocks[7];
b = (b << 19) | (b >>> 13);
a += ((b & c) | (~b & d)) + blocks[8];
a = (a << 3) | (a >>> 29);
d += ((a & b) | (~a & c)) + blocks[9];
d = (d << 7) | (d >>> 25);
c += ((d & a) | (~d & b)) + blocks[10];
c = (c << 11) | (c >>> 21);
b += ((c & d) | (~c & a)) + blocks[11];
b = (b << 19) | (b >>> 13);
a += ((b & c) | (~b & d)) + blocks[12];
a = (a << 3) | (a >>> 29);
d += ((a & b) | (~a & c)) + blocks[13];
d = (d << 7) | (d >>> 25);
c += ((d & a) | (~d & b)) + blocks[14];
c = (c << 11) | (c >>> 21);
b += ((c & d) | (~c & a)) + blocks[15];
b = (b << 19) | (b >>> 13);

bc = b & c;
a += (bc | (b & d) | (c & d)) + blocks[0] + 1518500249;
a = (a << 3) | (a >>> 29);
ab = a & b;
d += (ab | (a & c) | bc) + blocks[4] + 1518500249;
d = (d << 5) | (d >>> 27);
da = d & a;
c += (da | (d & b) | ab) + blocks[8] + 1518500249;
c = (c << 9) | (c >>> 23);
cd = c & d;
b += (cd | (c & a) | da) + blocks[12] + 1518500249;
b = (b << 13) | (b >>> 19);
bc = b & c;
a += (bc | (b & d) | cd) + blocks[1] + 1518500249;
a = (a << 3) | (a >>> 29);
ab = a & b;
d += (ab | (a & c) | bc) + blocks[5] + 1518500249;
d = (d << 5) | (d >>> 27);
da = d & a;
c += (da | (d & b) | ab) + blocks[9] + 1518500249;
c = (c << 9) | (c >>> 23);
cd = c & d;
b += (cd | (c & a) | da) + blocks[13] + 1518500249;
b = (b << 13) | (b >>> 19);
bc = b & c;
a += (bc | (b & d) | cd) + blocks[2] + 1518500249;
a = (a << 3) | (a >>> 29);
ab = a & b;
d += (ab | (a & c) | bc) + blocks[6] + 1518500249;
d = (d << 5) | (d >>> 27);
da = d & a;
c += (da | (d & b) | ab) + blocks[10] + 1518500249;
c = (c << 9) | (c >>> 23);
cd = c & d;
b += (cd | (c & a) | da) + blocks[14] + 1518500249;
b = (b << 13) | (b >>> 19);
bc = b & c;
a += (bc | (b & d) | cd) + blocks[3] + 1518500249;
a = (a << 3) | (a >>> 29);
ab = a & b;
d += (ab | (a & c) | bc) + blocks[7] + 1518500249;
d = (d << 5) | (d >>> 27);
da = d & a;
c += (da | (d & b) | ab) + blocks[11] + 1518500249;
c = (c << 9) | (c >>> 23);
b += ((c & d) | (c & a) | da) + blocks[15] + 1518500249;
b = (b << 13) | (b >>> 19);

bc = b ^ c;
a += (bc ^ d) + blocks[0] + 1859775393;
a = (a << 3) | (a >>> 29);
d += (bc ^ a) + blocks[8] + 1859775393;
d = (d << 9) | (d >>> 23);
da = d ^ a;
c += (da ^ b) + blocks[4] + 1859775393;
c = (c << 11) | (c >>> 21);
b += (da ^ c) + blocks[12] + 1859775393;
b = (b << 15) | (b >>> 17);
bc = b ^ c;
a += (bc ^ d) + blocks[2] + 1859775393;
a = (a << 3) | (a >>> 29);
d += (bc ^ a) + blocks[10] + 1859775393;
d = (d << 9) | (d >>> 23);
da = d ^ a;
c += (da ^ b) + blocks[6] + 1859775393;
c = (c << 11) | (c >>> 21);
b += (da ^ c) + blocks[14] + 1859775393;
b = (b << 15) | (b >>> 17);
bc = b ^ c;
a += (bc ^ d) + blocks[1] + 1859775393;
a = (a << 3) | (a >>> 29);
d += (bc ^ a) + blocks[9] + 1859775393;
d = (d << 9) | (d >>> 23);
da = d ^ a;
c += (da ^ b) + blocks[5] + 1859775393;
c = (c << 11) | (c >>> 21);
b += (da ^ c) + blocks[13] + 1859775393;
b = (b << 15) | (b >>> 17);
bc = b ^ c;
a += (bc ^ d) + blocks[3] + 1859775393;
a = (a << 3) | (a >>> 29);
d += (bc ^ a) + blocks[11] + 1859775393;
d = (d << 9) | (d >>> 23);
da = d ^ a;
c += (da ^ b) + blocks[7] + 1859775393;
c = (c << 11) | (c >>> 21);
b += (da ^ c) + blocks[15] + 1859775393;
b = (b << 15) | (b >>> 17);

if (this.first) {
this.h0 = a + 1732584193 << 0;
this.h1 = b - 271733879 << 0;
this.h2 = c - 1732584194 << 0;
this.h3 = d + 271733878 << 0;
this.first = false;
} else {
this.h0 = this.h0 + a << 0;
this.h1 = this.h1 + b << 0;
this.h2 = this.h2 + c << 0;
this.h3 = this.h3 + d << 0;
}
};

Loading