Skip to content

Commit c2eaf75

Browse files
TomFinleyTomFinley
authored
Update PFI tests to use single threaded code where it is important to do so. (#1926)
1 parent 6881700 commit c2eaf75

File tree

1 file changed

+84
-84
lines changed

1 file changed

+84
-84
lines changed

‎test/Microsoft.ML.Tests/PermutationFeatureImportanceTests.cs

Lines changed: 84 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -40,18 +40,18 @@ public void TestPfiRegressionOnDenseFeatures()
4040
// X4Rand: 3
4141

4242
// For the following metrics lower is better, so maximum delta means more important feature, and vice versa
43-
Assert.True(MinDeltaIndex(pfi, m => m.L1) == 3);
44-
Assert.True(MaxDeltaIndex(pfi, m => m.L1) == 1);
43+
Assert.Equal(3, MinDeltaIndex(pfi, m => m.L1));
44+
Assert.Equal(1, MaxDeltaIndex(pfi, m => m.L1));
4545

46-
Assert.True(MinDeltaIndex(pfi, m => m.L2) == 3);
47-
Assert.True(MaxDeltaIndex(pfi, m => m.L2) == 1);
46+
Assert.Equal(3, MinDeltaIndex(pfi, m => m.L2));
47+
Assert.Equal(1, MaxDeltaIndex(pfi, m => m.L2));
4848

49-
Assert.True(MinDeltaIndex(pfi, m => m.Rms) == 3);
50-
Assert.True(MaxDeltaIndex(pfi, m => m.Rms) == 1);
49+
Assert.Equal(3, MinDeltaIndex(pfi, m => m.Rms));
50+
Assert.Equal(1, MaxDeltaIndex(pfi, m => m.Rms));
5151

5252
// For the following metrics higher is better, so minimum delta means more important feature, and vice versa
53-
Assert.True(MaxDeltaIndex(pfi, m => m.RSquared) == 3);
54-
Assert.True(MinDeltaIndex(pfi, m => m.RSquared) == 1);
53+
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.RSquared));
54+
Assert.Equal(1, MinDeltaIndex(pfi, m => m.RSquared));
5555

5656
Done();
5757
}
@@ -76,18 +76,18 @@ public void TestPfiRegressionOnSparseFeatures()
7676

7777
// Permuted X2VBuffer-Slot-1 lot (f2) should have min impact on SGD metrics, X3Important -- max impact.
7878
// For the following metrics lower is better, so maximum delta means more important feature, and vice versa
79-
Assert.True(MinDeltaIndex(results, m => m.L1) == 2);
80-
Assert.True(MaxDeltaIndex(results, m => m.L1) == 5);
79+
Assert.Equal(2, MinDeltaIndex(results, m => m.L1));
80+
Assert.Equal(5, MaxDeltaIndex(results, m => m.L1));
8181

82-
Assert.True(MinDeltaIndex(results, m => m.L2) == 2);
83-
Assert.True(MaxDeltaIndex(results, m => m.L2) == 5);
82+
Assert.Equal(2, MinDeltaIndex(results, m => m.L2));
83+
Assert.Equal(5, MaxDeltaIndex(results, m => m.L2));
8484

85-
Assert.True(MinDeltaIndex(results, m => m.Rms) == 2);
86-
Assert.True(MaxDeltaIndex(results, m => m.Rms) == 5);
85+
Assert.Equal(2, MinDeltaIndex(results, m => m.Rms));
86+
Assert.Equal(5, MaxDeltaIndex(results, m => m.Rms));
8787

8888
// For the following metrics higher is better, so minimum delta means more important feature, and vice versa
89-
Assert.True(MaxDeltaIndex(results, m => m.RSquared) == 2);
90-
Assert.True(MinDeltaIndex(results, m => m.RSquared) == 5);
89+
Assert.Equal(2, MaxDeltaIndex(results, m => m.RSquared));
90+
Assert.Equal(5, MinDeltaIndex(results, m => m.RSquared));
9191
}
9292

9393
#endregion
@@ -100,7 +100,7 @@ public void TestPfiRegressionOnSparseFeatures()
100100
public void TestPfiBinaryClassificationOnDenseFeatures()
101101
{
102102
var data = GetDenseDataset(TaskType.BinaryClassification);
103-
var model = ML.BinaryClassification.Trainers.LogisticRegression().Fit(data);
103+
var model = ML.BinaryClassification.Trainers.LogisticRegression(advancedSettings: args => args.NumThreads = 1).Fit(data);
104104
var pfi = ML.BinaryClassification.PermutationFeatureImportance(model, data);
105105

106106
// Pfi Indices:
@@ -110,22 +110,22 @@ public void TestPfiBinaryClassificationOnDenseFeatures()
110110
// X4Rand: 3
111111

112112
// For the following metrics higher is better, so minimum delta means more important feature, and vice versa
113-
Assert.True(MaxDeltaIndex(pfi, m => m.Auc) == 3);
114-
Assert.True(MinDeltaIndex(pfi, m => m.Auc) == 1);
115-
Assert.True(MaxDeltaIndex(pfi, m => m.Accuracy) == 3);
116-
Assert.True(MinDeltaIndex(pfi, m => m.Accuracy) == 1);
117-
Assert.True(MaxDeltaIndex(pfi, m => m.PositivePrecision) == 3);
118-
Assert.True(MinDeltaIndex(pfi, m => m.PositivePrecision) == 1);
119-
Assert.True(MaxDeltaIndex(pfi, m => m.PositiveRecall) == 3);
120-
Assert.True(MinDeltaIndex(pfi, m => m.PositiveRecall) == 1);
121-
Assert.True(MaxDeltaIndex(pfi, m => m.NegativePrecision) == 3);
122-
Assert.True(MinDeltaIndex(pfi, m => m.NegativePrecision) == 1);
123-
Assert.True(MaxDeltaIndex(pfi, m => m.NegativeRecall) == 3);
124-
Assert.True(MinDeltaIndex(pfi, m => m.NegativeRecall) == 1);
125-
Assert.True(MaxDeltaIndex(pfi, m => m.F1Score) == 3);
126-
Assert.True(MinDeltaIndex(pfi, m => m.F1Score) == 1);
127-
Assert.True(MaxDeltaIndex(pfi, m => m.Auprc) == 3);
128-
Assert.True(MinDeltaIndex(pfi, m => m.Auprc) == 1);
113+
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.Auc));
114+
Assert.Equal(1, MinDeltaIndex(pfi, m => m.Auc));
115+
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.Accuracy));
116+
Assert.Equal(1, MinDeltaIndex(pfi, m => m.Accuracy));
117+
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.PositivePrecision));
118+
Assert.Equal(1, MinDeltaIndex(pfi, m => m.PositivePrecision));
119+
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.PositiveRecall));
120+
Assert.Equal(1, MinDeltaIndex(pfi, m => m.PositiveRecall));
121+
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.NegativePrecision));
122+
Assert.Equal(1, MinDeltaIndex(pfi, m => m.NegativePrecision));
123+
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.NegativeRecall));
124+
Assert.Equal(1, MinDeltaIndex(pfi, m => m.NegativeRecall));
125+
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.F1Score));
126+
Assert.Equal(1, MinDeltaIndex(pfi, m => m.F1Score));
127+
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.Auprc));
128+
Assert.Equal(1, MinDeltaIndex(pfi, m => m.Auprc));
129129

130130
Done();
131131
}
@@ -137,7 +137,7 @@ public void TestPfiBinaryClassificationOnDenseFeatures()
137137
public void TestPfiBinaryClassificationOnSparseFeatures()
138138
{
139139
var data = GetSparseDataset(TaskType.BinaryClassification);
140-
var model = ML.BinaryClassification.Trainers.LogisticRegression().Fit(data);
140+
var model = ML.BinaryClassification.Trainers.LogisticRegression(advancedSettings: args => args.NumThreads = 1).Fit(data);
141141
var pfi = ML.BinaryClassification.PermutationFeatureImportance(model, data);
142142

143143
// Pfi Indices:
@@ -149,22 +149,22 @@ public void TestPfiBinaryClassificationOnSparseFeatures()
149149
// X3Important: 5
150150

151151
// For the following metrics higher is better, so minimum delta means more important feature, and vice versa
152-
Assert.True(MaxDeltaIndex(pfi, m => m.Auc) == 2);
153-
Assert.True(MinDeltaIndex(pfi, m => m.Auc) == 5);
154-
Assert.True(MaxDeltaIndex(pfi, m => m.Accuracy) == 2);
155-
Assert.True(MinDeltaIndex(pfi, m => m.Accuracy) == 5);
156-
Assert.True(MaxDeltaIndex(pfi, m => m.PositivePrecision) == 2);
157-
Assert.True(MinDeltaIndex(pfi, m => m.PositivePrecision) == 5);
158-
Assert.True(MaxDeltaIndex(pfi, m => m.PositiveRecall) == 2);
159-
Assert.True(MinDeltaIndex(pfi, m => m.PositiveRecall) == 5);
160-
Assert.True(MaxDeltaIndex(pfi, m => m.NegativePrecision) == 2);
161-
Assert.True(MinDeltaIndex(pfi, m => m.NegativePrecision) == 5);
162-
Assert.True(MaxDeltaIndex(pfi, m => m.NegativeRecall) == 2);
163-
Assert.True(MinDeltaIndex(pfi, m => m.NegativeRecall) == 5);
164-
Assert.True(MaxDeltaIndex(pfi, m => m.F1Score) == 2);
165-
Assert.True(MinDeltaIndex(pfi, m => m.F1Score) == 5);
166-
Assert.True(MaxDeltaIndex(pfi, m => m.Auprc) == 2);
167-
Assert.True(MinDeltaIndex(pfi, m => m.Auprc) == 5);
152+
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.Auc));
153+
Assert.Equal(5, MinDeltaIndex(pfi, m => m.Auc));
154+
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.Accuracy));
155+
Assert.Equal(5, MinDeltaIndex(pfi, m => m.Accuracy));
156+
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.PositivePrecision));
157+
Assert.Equal(5, MinDeltaIndex(pfi, m => m.PositivePrecision));
158+
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.PositiveRecall));
159+
Assert.Equal(5, MinDeltaIndex(pfi, m => m.PositiveRecall));
160+
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.NegativePrecision));
161+
Assert.Equal(5, MinDeltaIndex(pfi, m => m.NegativePrecision));
162+
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.NegativeRecall));
163+
Assert.Equal(5, MinDeltaIndex(pfi, m => m.NegativeRecall));
164+
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.F1Score));
165+
Assert.Equal(5, MinDeltaIndex(pfi, m => m.F1Score));
166+
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.Auprc));
167+
Assert.Equal(5, MinDeltaIndex(pfi, m => m.Auprc));
168168

169169
Done();
170170
}
@@ -188,21 +188,21 @@ public void TestPfiMulticlassClassificationOnDenseFeatures()
188188
// X4Rand: 3
189189

190190
// For the following metrics higher is better, so minimum delta means more important feature, and vice versa
191-
Assert.True(MaxDeltaIndex(pfi, m => m.AccuracyMicro) == 3);
192-
Assert.True(MinDeltaIndex(pfi, m => m.AccuracyMicro) == 1);
193-
Assert.True(MaxDeltaIndex(pfi, m => m.AccuracyMacro) == 3);
194-
Assert.True(MinDeltaIndex(pfi, m => m.AccuracyMacro) == 1);
195-
Assert.True(MaxDeltaIndex(pfi, m => m.LogLossReduction) == 3);
196-
Assert.True(MinDeltaIndex(pfi, m => m.LogLossReduction) == 1);
191+
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.AccuracyMicro));
192+
Assert.Equal(1, MinDeltaIndex(pfi, m => m.AccuracyMicro));
193+
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.AccuracyMacro));
194+
Assert.Equal(1, MinDeltaIndex(pfi, m => m.AccuracyMacro));
195+
Assert.Equal(3, MaxDeltaIndex(pfi, m => m.LogLossReduction));
196+
Assert.Equal(1, MinDeltaIndex(pfi, m => m.LogLossReduction));
197197

198198
// For the following metrics-delta lower is better, so maximum delta means more important feature, and vice versa
199199
// Because they are _negative_, the difference will be positive for worse classifiers.
200-
Assert.True(MaxDeltaIndex(pfi, m => m.LogLoss) == 1);
201-
Assert.True(MinDeltaIndex(pfi, m => m.LogLoss) == 3);
200+
Assert.Equal(1, MaxDeltaIndex(pfi, m => m.LogLoss));
201+
Assert.Equal(3, MinDeltaIndex(pfi, m => m.LogLoss));
202202
for (int i = 0; i < pfi[0].PerClassLogLoss.Length; i++)
203203
{
204-
Assert.True(MaxDeltaIndex(pfi, m => m.PerClassLogLoss[i]) == 1);
205-
Assert.True(MinDeltaIndex(pfi, m => m.PerClassLogLoss[i]) == 3);
204+
Assert.Equal(1, MaxDeltaIndex(pfi, m => m.PerClassLogLoss[i]));
205+
Assert.Equal(3, MinDeltaIndex(pfi, m => m.PerClassLogLoss[i]));
206206
}
207207

208208
Done();
@@ -227,21 +227,21 @@ public void TestPfiMulticlassClassificationOnSparseFeatures()
227227
// X3Important: 5 // Most important
228228

229229
// For the following metrics higher is better, so minimum delta means more important feature, and vice versa
230-
Assert.True(MaxDeltaIndex(pfi, m => m.AccuracyMicro) == 2);
231-
Assert.True(MinDeltaIndex(pfi, m => m.AccuracyMicro) == 5);
232-
Assert.True(MaxDeltaIndex(pfi, m => m.AccuracyMacro) == 2);
233-
Assert.True(MinDeltaIndex(pfi, m => m.AccuracyMacro) == 5);
234-
Assert.True(MaxDeltaIndex(pfi, m => m.LogLossReduction) == 2);
235-
Assert.True(MinDeltaIndex(pfi, m => m.LogLossReduction) == 5);
230+
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.AccuracyMicro));
231+
Assert.Equal(5, MinDeltaIndex(pfi, m => m.AccuracyMicro));
232+
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.AccuracyMacro));
233+
Assert.Equal(5, MinDeltaIndex(pfi, m => m.AccuracyMacro));
234+
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.LogLossReduction));
235+
Assert.Equal(5, MinDeltaIndex(pfi, m => m.LogLossReduction));
236236

237237
// For the following metrics-delta lower is better, so maximum delta means more important feature, and vice versa
238238
// Because they are negative metrics, the _difference_ will be positive for worse classifiers.
239-
Assert.True(MaxDeltaIndex(pfi, m => m.LogLoss) == 5);
240-
Assert.True(MinDeltaIndex(pfi, m => m.LogLoss) == 2);
239+
Assert.Equal(5, MaxDeltaIndex(pfi, m => m.LogLoss));
240+
Assert.Equal(2, MinDeltaIndex(pfi, m => m.LogLoss));
241241
for (int i = 0; i < pfi[0].PerClassLogLoss.Length; i++)
242242
{
243-
Assert.True(MaxDeltaIndex(pfi, m => m.PerClassLogLoss[i]) == 5);
244-
Assert.True(MinDeltaIndex(pfi, m => m.PerClassLogLoss[i]) == 2);
243+
Assert.Equal(5, MaxDeltaIndex(pfi, m => m.PerClassLogLoss[i]));
244+
Assert.Equal(2, MinDeltaIndex(pfi, m => m.PerClassLogLoss[i]));
245245
}
246246

247247
Done();
@@ -268,13 +268,13 @@ public void TestPfiRankingOnDenseFeatures()
268268
// For the following metrics higher is better, so minimum delta means more important feature, and vice versa
269269
for (int i = 0; i < pfi[0].Dcg.Length; i++)
270270
{
271-
Assert.True(MaxDeltaIndex(pfi, m => m.Dcg[i]) == 0);
272-
Assert.True(MinDeltaIndex(pfi, m => m.Dcg[i]) == 1);
271+
Assert.Equal(0, MaxDeltaIndex(pfi, m => m.Dcg[i]));
272+
Assert.Equal(1, MinDeltaIndex(pfi, m => m.Dcg[i]));
273273
}
274274
for (int i = 0; i < pfi[0].Ndcg.Length; i++)
275275
{
276-
Assert.True(MaxDeltaIndex(pfi, m => m.Ndcg[i]) == 0);
277-
Assert.True(MinDeltaIndex(pfi, m => m.Ndcg[i]) == 1);
276+
Assert.Equal(0, MaxDeltaIndex(pfi, m => m.Ndcg[i]));
277+
Assert.Equal(1, MinDeltaIndex(pfi, m => m.Ndcg[i]));
278278
}
279279

280280
Done();
@@ -301,13 +301,13 @@ public void TestPfiRankingOnSparseFeatures()
301301
// For the following metrics higher is better, so minimum delta means more important feature, and vice versa
302302
for (int i = 0; i < pfi[0].Dcg.Length; i++)
303303
{
304-
Assert.True(MaxDeltaIndex(pfi, m => m.Dcg[i]) == 2);
305-
Assert.True(MinDeltaIndex(pfi, m => m.Dcg[i]) == 5);
304+
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.Dcg[i]));
305+
Assert.Equal(5, MinDeltaIndex(pfi, m => m.Dcg[i]));
306306
}
307307
for (int i = 0; i < pfi[0].Ndcg.Length; i++)
308308
{
309-
Assert.True(MaxDeltaIndex(pfi, m => m.Ndcg[i]) == 2);
310-
Assert.True(MinDeltaIndex(pfi, m => m.Ndcg[i]) == 5);
309+
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.Ndcg[i]));
310+
Assert.Equal(5, MinDeltaIndex(pfi, m => m.Ndcg[i]));
311311
}
312312

313313
Done();
@@ -325,8 +325,8 @@ public void TestPfiClusteringOnDenseFeatures()
325325

326326
var preview = data.Preview();
327327

328-
var model = ML.Clustering.Trainers.KMeans("Features", clustersCount: 5,
329-
advancedSettings: args =>{ args.NormalizeFeatures = NormalizeOption.No;})
328+
var model = ML.Clustering.Trainers.KMeans("Features", clustersCount: 5,
329+
advancedSettings: args =>{ args.NormalizeFeatures = NormalizeOption.No; args.NumThreads = 1; })
330330
.Fit(data);
331331
var pfi = ML.Clustering.PermutationFeatureImportance(model, data);
332332

@@ -336,12 +336,12 @@ public void TestPfiClusteringOnDenseFeatures()
336336
// X3: 2 -- Least important for clustering (smallest range)
337337

338338
// For the following metrics lower is better, so maximum delta means more important feature, and vice versa
339-
Assert.True(MinDeltaIndex(pfi, m => m.AvgMinScore) == 0);
340-
Assert.True(MaxDeltaIndex(pfi, m => m.AvgMinScore) == 2);
339+
Assert.Equal(0, MinDeltaIndex(pfi, m => m.AvgMinScore));
340+
Assert.Equal(2, MaxDeltaIndex(pfi, m => m.AvgMinScore));
341341

342342
// For the following metrics higher is better, so minimum delta means more important feature, and vice versa
343-
Assert.True(MinDeltaIndex(pfi, m => m.Nmi) == 2);
344-
Assert.True(MaxDeltaIndex(pfi, m => m.Nmi) == 0);
343+
Assert.Equal(2, MinDeltaIndex(pfi, m => m.Nmi));
344+
Assert.Equal(0, MaxDeltaIndex(pfi, m => m.Nmi));
345345

346346
Done();
347347
}

0 commit comments

Comments
 (0)