Ensure JIT_Dbl2ULng correctly handles NaN

tannergooding · tannergooding · commit 329834538e26 · 2022-05-27T08:59:18.000-07:00
diff --git a/src/coreclr/vm/jithelpers.cpp b/src/coreclr/vm/jithelpers.cpp
@@ -595,10 +595,14 @@ HCIMPL1_V(UINT64, JIT_Dbl2ULng, double val)
     if (val < two63) {
         ret = FastDbl2Lng(val);
     }
-    else {
+    else if (!_isnan(val)) {
         // subtract 0x8000000000000000, do the convert then add it back again
         ret = FastDbl2Lng(val - two63) + I64(0x8000000000000000);
     }
+    else {
+        // specially handle NaN otherwise we return (Int64.MaxValue + 1)
+        ret = 0;
+    }
     return ret;
 }
 HCIMPLEND

Original file line number	Diff line number	Diff line change
`@@ -595,10 +595,14 @@ HCIMPL1_V(UINT64, JIT_Dbl2ULng, double val)`
`595`	`595`	`if (val < two63) {`
`596`	`596`	`ret = FastDbl2Lng(val);`
`597`	`597`	`}`
`598`		`- else {`
	`598`	`+ else if (!_isnan(val)) {`
`599`	`599`	`// subtract 0x8000000000000000, do the convert then add it back again`
`600`	`600`	`ret = FastDbl2Lng(val - two63) + I64(0x8000000000000000);`
`601`	`601`	`}`
	`602`	`+ else {`
	`603`	`+ // specially handle NaN otherwise we return (Int64.MaxValue + 1)`
	`604`	`+ ret = 0;`
	`605`	`+ }`
`602`	`606`	`return ret;`
`603`	`607`	`}`
`604`	`608`	`HCIMPLEND`