Fix SoftMax precision by utilizing double in the internal calculations (#3676)

najeeb-kazmi · web-flow · commit 715ba89e92e6 · 2019-05-15T13:40:10.000-07:00
* Fix SoftMax precision by utilizing double in the internal calculations * Brew install libomp 7.0.0 only. (#3721) * Brew install libomp 7.0.0 only. * Brew install libomp 7.0.0 only. * Brew install libomp 7.0.0 only. * Brew install libomp 7.0.0 only. * Fix SoftMax precision by utilizing double in the internal calculations
diff --git a/src/Microsoft.ML.StandardTrainers/Standard/MulticlassClassification/OneVersusAllTrainer.cs b/src/Microsoft.ML.StandardTrainers/Standard/MulticlassClassification/OneVersusAllTrainer.cs
@@ -740,15 +740,17 @@ public override ValueMapper<VBuffer<float>, VBuffer<float>> GetMapper()
 
             private void NormalizeSoftmax(float[] scores, int count)
             {
-                float sum = 0;
+                double sum = 0;
+                var score = new double[count];
+
                 for (int i = 0; i < count; i++)
                 {
-                    scores[i] = (float)Math.Exp(scores[i]);
-                    sum += scores[i];
+                    score[i] = Math.Exp(scores[i]);
+                    sum += score[i];
                 }
 
                 for (int i = 0; i < count; i++)
-                    scores[i] = scores[i] / sum;
+                    scores[i] = (float)(score[i] / sum);
             }
 
             public override JToken SaveAsPfa(BoundPfaContext ctx, JToken input)

Original file line number	Diff line number	Diff line change
`@@ -740,15 +740,17 @@ public override ValueMapper<VBuffer<float>, VBuffer<float>> GetMapper()`
`740`	`740`
`741`	`741`	`private void NormalizeSoftmax(float[] scores, int count)`
`742`	`742`	`{`
`743`		`- float sum = 0;`
	`743`	`+ double sum = 0;`
	`744`	`+ var score = new double[count];`
	`745`	`+`
`744`	`746`	`for (int i = 0; i < count; i++)`
`745`	`747`	`{`
`746`		`- scores[i] = (float)Math.Exp(scores[i]);`
`747`		`- sum += scores[i];`
	`748`	`+ score[i] = Math.Exp(scores[i]);`
	`749`	`+ sum += score[i];`
`748`	`750`	`}`
`749`	`751`
`750`	`752`	`for (int i = 0; i < count; i++)`
`751`		`- scores[i] = scores[i] / sum;`
	`753`	`+ scores[i] = (float)(score[i] / sum);`
`752`	`754`	`}`
`753`	`755`
`754`	`756`	`public override JToken SaveAsPfa(BoundPfaContext ctx, JToken input)`