Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Extract TestFrameworkCommon Project, remove dependency of TestFramework from FT #4346

Merged
merged 16 commits into from
Oct 30, 2019
Merged

Extract TestFrameworkCommon Project, remove dependency of TestFramework from FT #4346

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
e74e74e
extract TestFrameworkCommon project and remove reference to TestFrame…
frank-dong-ms Oct 15, 2019
26f5861
update on TestFrameworkCommon
frank-dong-ms Oct 17, 2019
49f3e66
update on TestFrameworkCommon project
frank-dong-ms Oct 17, 2019
f74cbd0
minor fix
frank-dong-ms Oct 17, 2019
60f8717
Merge https://github.com/dotnet/machinelearning into nightly-build
frank-dong-ms Oct 17, 2019
9e06a20
fix reference
frank-dong-ms Oct 17, 2019
e534d04
Merge https://github.com/dotnet/machinelearning into nightly-build
frank-dong-ms Oct 17, 2019
19c10ed
fix project reference
frank-dong-ms Oct 17, 2019
3e1b2fa
fix CI build error
frank-dong-ms Oct 17, 2019
be83960
Merge https://github.com/dotnet/machinelearning into nightly-build
frank-dong-ms Oct 17, 2019
d807e69
sync latest code and resolve
frank-dong-ms Oct 18, 2019
1a4f223
take comments and made change
frank-dong-ms Oct 22, 2019
feaf409
take comments and fix
frank-dong-ms Oct 22, 2019
6b7baf7
Merge https://github.com/dotnet/machinelearning into nightly-build
frank-dong-ms Oct 23, 2019
1f899ac
sync and resolve conflict
frank-dong-ms Oct 30, 2019
1d972fc
modify sln file to build on azure pipeline
frank-dong-ms Oct 30, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions Microsoft.ML.sln
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.AutoML.Samples
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Samples.GPU", "docs\samples\Microsoft.ML.Samples.GPU\Microsoft.ML.Samples.GPU.csproj", "{3C8F910B-7F23-4D25-B521-6D5AC9570ADD}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.TestFrameworkCommon", "test\Microsoft.ML.TestFrameworkCommon\Microsoft.ML.TestFrameworkCommon.csproj", "{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -1632,6 +1634,30 @@ Global
{3C8F910B-7F23-4D25-B521-6D5AC9570ADD}.Release-netfx|Any CPU.Build.0 = Release-netfx|Any CPU
{3C8F910B-7F23-4D25-B521-6D5AC9570ADD}.Release-netfx|x64.ActiveCfg = Release-netfx|Any CPU
{3C8F910B-7F23-4D25-B521-6D5AC9570ADD}.Release-netfx|x64.Build.0 = Release-netfx|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Debug|x64.ActiveCfg = Debug|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Debug|x64.Build.0 = Debug|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Debug-netcoreapp3_0|Any CPU.ActiveCfg = Debug-netcoreapp3_0|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Debug-netcoreapp3_0|Any CPU.Build.0 = Debug-netcoreapp3_0|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Debug-netcoreapp3_0|x64.ActiveCfg = Debug-netcoreapp3_0|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Debug-netcoreapp3_0|x64.Build.0 = Debug-netcoreapp3_0|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Debug-netfx|Any CPU.ActiveCfg = Debug-netfx|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Debug-netfx|Any CPU.Build.0 = Debug-netfx|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Debug-netfx|x64.ActiveCfg = Debug-netfx|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Debug-netfx|x64.Build.0 = Debug-netfx|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Release|Any CPU.Build.0 = Release|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Release|x64.ActiveCfg = Release|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Release|x64.Build.0 = Release|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Release-netcoreapp3_0|Any CPU.ActiveCfg = Release-netcoreapp3_0|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Release-netcoreapp3_0|Any CPU.Build.0 = Release-netcoreapp3_0|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Release-netcoreapp3_0|x64.ActiveCfg = Release-netcoreapp3_0|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Release-netcoreapp3_0|x64.Build.0 = Release-netcoreapp3_0|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Release-netfx|Any CPU.ActiveCfg = Release-netfx|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Release-netfx|Any CPU.Build.0 = Release-netfx|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Release-netfx|x64.ActiveCfg = Release-netfx|Any CPU
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB}.Release-netfx|x64.Build.0 = Release-netfx|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down Expand Up @@ -1718,6 +1744,7 @@ Global
{F5D11F71-2D61-4AE9-99D7-0F0B54649B15} = {D3D38B03-B557-484D-8348-8BADEE4DF592}
{A6924919-9E37-4023-8B7F-E85C8E3CC9B3} = {DA452A53-2E94-4433-B08C-041EDEC729E6}
{3C8F910B-7F23-4D25-B521-6D5AC9570ADD} = {DA452A53-2E94-4433-B08C-041EDEC729E6}
{6EA76538-5B49-4B8B-8695-E9F59AA6ECEB} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}
Expand Down
1 change: 1 addition & 0 deletions test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
using System.Linq;
using Microsoft.ML.Data;
using Microsoft.ML.RunTests;
using Microsoft.ML.TestFrameworkCommon;
using Xunit;

namespace Microsoft.ML.AutoML.Test
Expand Down
2 changes: 1 addition & 1 deletion test/Microsoft.ML.Benchmarks/Numeric/Ranking.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@
using BenchmarkDotNet.Attributes;
using Microsoft.ML.Data;
using Microsoft.ML.Trainers.LightGbm;
using Microsoft.ML.RunTests;
using Microsoft.ML.TestFramework;
using Microsoft.ML.Trainers;
using Microsoft.ML.Trainers.FastTree;
using Microsoft.ML.Transforms;
using Microsoft.ML.TestFrameworkCommon;

namespace Microsoft.ML.Benchmarks
{
Expand Down
1 change: 1 addition & 0 deletions test/Microsoft.ML.Benchmarks/RffTransform.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using Microsoft.ML.Data;
using Microsoft.ML.RunTests;
using Microsoft.ML.TestFramework;
using Microsoft.ML.TestFrameworkCommon;
using Microsoft.ML.Transforms;

namespace Microsoft.ML.Benchmarks
Expand Down
1 change: 1 addition & 0 deletions test/Microsoft.ML.Benchmarks/Text/MultiClassClassification.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
using Microsoft.ML.TestFramework;
using Microsoft.ML.Trainers;
using Microsoft.ML.Transforms;
using Microsoft.ML.TestFrameworkCommon;

namespace Microsoft.ML.Benchmarks
{
Expand Down
82 changes: 4 additions & 78 deletions test/Microsoft.ML.Core.Tests/UnitTests/CoreBaseTestClass.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using Microsoft.ML.Internal.Utilities;
using Microsoft.ML.RunTests;
using Microsoft.ML.Runtime;
using Microsoft.ML.TestFrameworkCommon;
using Xunit.Abstractions;

namespace Microsoft.ML.Core.Tests.UnitTests
Expand All @@ -25,16 +26,6 @@ protected bool Failed()
return false;
}

protected bool EqualTypes(DataViewType type1, DataViewType type2, bool exactTypes)
{
Contracts.AssertValue(type1);
Contracts.AssertValue(type2);

if (type1.Equals(type2))
return true;
return !exactTypes && type1 is VectorDataViewType vt1 && type2 is VectorDataViewType vt2 && vt1.ItemType.Equals(vt2.ItemType) && vt1.Size == vt2.Size;
}

protected Func<bool> GetIdComparer(DataViewRow r1, DataViewRow r2, out ValueGetter<DataViewRowId> idGetter)
{
var g1 = r1.GetIdGetter();
Expand Down Expand Up @@ -85,75 +76,10 @@ protected Func<bool> GetComparerVec<T>(DataViewRow r1, DataViewRow r2, int col,
{
g1(ref v1);
g2(ref v2);
return CompareVec<T>(in v1, in v2, size, fn);
return CommonUtilities.CompareVec<T>(in v1, in v2, size, fn);
};
}

protected bool CompareVec<T>(in VBuffer<T> v1, in VBuffer<T> v2, int size, Func<T, T, bool> fn)
{
return CompareVec(in v1, in v2, size, (i, x, y) => fn(x, y));
}

protected bool CompareVec<T>(in VBuffer<T> v1, in VBuffer<T> v2, int size, Func<int, T, T, bool> fn)
{
Contracts.Assert(size == 0 || v1.Length == size);
Contracts.Assert(size == 0 || v2.Length == size);
Contracts.Assert(v1.Length == v2.Length);

var v1Values = v1.GetValues();
var v2Values = v2.GetValues();

if (v1.IsDense && v2.IsDense)
{
for (int i = 0; i < v1.Length; i++)
{
var x1 = v1Values[i];
var x2 = v2Values[i];
if (!fn(i, x1, x2))
return false;
}
return true;
}

Contracts.Assert(!v1.IsDense || !v2.IsDense);
int iiv1 = 0;
int iiv2 = 0;
var v1Indices = v1.GetIndices();
var v2Indices = v2.GetIndices();
for (; ; )
{
int iv1 = v1.IsDense ? iiv1 : iiv1 < v2Indices.Length ? v1Indices[iiv1] : v1.Length;
Copy link
Contributor Author

@frank-dong-ms-zz frank-dong-ms-zz Oct 17, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

v2Indices [](start = 53, length = 9)

Seems like a bug to me, should be v1Indices #Resolved

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch! Yes, it does appear to be a bug. Can you please check the callers who are using this function and how those tests are passing?

In reply to: 336252686 [](ancestors = 336252686)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TestEntryPoints class is using this method, these tests will pass as long as v1 and v2 has same indices length, this happens to be true in these tests

In reply to: 336584593 [](ancestors = 336584593,336252686)

int iv2 = v2.IsDense ? iiv2 : iiv2 < v2Indices.Length ? v2Indices[iiv2] : v2.Length;
T x1, x2;
int iv;
if (iv1 == iv2)
{
if (iv1 == v1.Length)
return true;
x1 = v1Values[iiv1];
x2 = v2Values[iiv2];
iv = iv1;
iiv1++;
iiv2++;
}
else if (iv1 < iv2)
{
x1 = v1Values[iiv1];
x2 = default(T);
iv = iv1;
iiv1++;
}
else
{
x1 = default(T);
x2 = v2Values[iiv2];
iv = iv2;
iiv2++;
}
if (!fn(iv, x1, x2))
return false;
}
}
protected Func<bool> GetColumnComparer(DataViewRow r1, DataViewRow r2, int col, DataViewType type, bool exactDoubles)
{
if (type is VectorDataViewType vecType)
Expand Down Expand Up @@ -331,7 +257,7 @@ protected bool CheckSameValues(DataViewRowCursor curs1, DataViewRowCursor curs2,
{
var type1 = curs1.Schema[col].Type;
var type2 = curs2.Schema[col].Type;
if (!EqualTypes(type1, type2, exactTypes))
if (!CommonUtilities.EqualTypes(type1, type2, exactTypes))
{
Fail($"Different types {type1} and {type2}");
return Failed();
Expand Down Expand Up @@ -420,7 +346,7 @@ protected bool CheckSameValues(DataViewRowCursor curs1, IDataView view2, bool ex
Contracts.Assert(cursors[col] != null);
var type1 = curs1.Schema[col].Type;
var type2 = cursors[col].Schema[col].Type;
if (!EqualTypes(type1, type2, exactTypes))
if (!CommonUtilities.EqualTypes(type1, type2, exactTypes))
{
Fail("Different types");
return Failed();
Expand Down
1 change: 1 addition & 0 deletions test/Microsoft.ML.Core.Tests/UnitTests/TestEntryPoints.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
using Microsoft.ML.Model.OnnxConverter;
using Microsoft.ML.Runtime;
using Microsoft.ML.TestFramework.Attributes;
using Microsoft.ML.TestFrameworkCommon;
using Microsoft.ML.Trainers;
using Microsoft.ML.Trainers.Ensemble;
using Microsoft.ML.Trainers.FastTree;
Expand Down
8 changes: 4 additions & 4 deletions test/Microsoft.ML.Functional.Tests/DataIO.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

using System.IO;
using Microsoft.ML.Functional.Tests.Datasets;
using Microsoft.ML.TestFramework;
using Microsoft.ML.TestFrameworkCommon;
using Xunit;
using Xunit.Abstractions;

Expand All @@ -13,7 +13,7 @@ namespace Microsoft.ML.Functional.Tests
/// <summary>
/// Test data input and output formats.
/// </summary>
public class DataIO : BaseTestClass
public class DataIO : FunctionalTestBaseClass
{
// Separators to test
private readonly char[] _separators;
Expand Down Expand Up @@ -118,7 +118,7 @@ public void WriteAndReadAFromABinaryFile()

private string SerializeDatasetToFile(MLContext mlContext, IDataView data, char separator)
{
var filePath = GetOutputPath(Path.GetRandomFileName());
var filePath = CommonUtilities.GetOutputPath(Path.GetRandomFileName(), OutDir);
using (var file = File.Create(filePath))
mlContext.Data.SaveAsText(data, file, separatorChar: separator, headerRow: true);

Expand All @@ -127,7 +127,7 @@ private string SerializeDatasetToFile(MLContext mlContext, IDataView data, char

private string SerializeDatasetToBinaryFile(MLContext mlContext, IDataView data)
{
var filePath = GetOutputPath(Path.GetRandomFileName());
var filePath = CommonUtilities.GetOutputPath(Path.GetRandomFileName(), OutDir);
using (var file = File.Create(filePath))
mlContext.Data.SaveAsBinary(data, file);

Expand Down
13 changes: 6 additions & 7 deletions test/Microsoft.ML.Functional.Tests/DataTransformation.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,15 @@

using System;
using Microsoft.ML.Functional.Tests.Datasets;
using Microsoft.ML.RunTests;
using Microsoft.ML.TestFramework;
using Microsoft.ML.TestFrameworkCommon;
using Microsoft.ML.Trainers;
using Microsoft.ML.Transforms.Text;
using Xunit;
using Xunit.Abstractions;

namespace Microsoft.ML.Functional.Tests
{
public class DataTransformation : BaseTestClass
public class DataTransformation : FunctionalTestBaseClass
{
public DataTransformation(ITestOutputHelper output) : base(output)
{
Expand All @@ -30,7 +29,7 @@ void ExtensibilityAddAColumnAsAFunctionOfMultipleColumns()

// Load the Iris dataset
var data = mlContext.Data.LoadFromTextFile<Iris>(
GetDataPath(TestDatasets.iris.trainFilename),
CommonUtilities.GetDataPath(TestDatasets.iris.trainFilename, DataDir),
hasHeader: TestDatasets.iris.fileHasHeader,
separatorChar: TestDatasets.iris.fileSeparator);

Expand Down Expand Up @@ -83,7 +82,7 @@ void ExtensibilityAddingTwoColumns()

// Load the Iris dataset
var data = mlContext.Data.LoadFromTextFile<Iris>(
GetDataPath(TestDatasets.iris.trainFilename),
CommonUtilities.GetDataPath(TestDatasets.iris.trainFilename, DataDir),
hasHeader: TestDatasets.iris.fileHasHeader,
separatorChar: TestDatasets.iris.fileSeparator);

Expand Down Expand Up @@ -129,7 +128,7 @@ void ExtensibilityModifyTextFeaturization()
// Concurrency must be 1 to assure that the mapping is done sequentially.
var mlContext = new MLContext(seed: 1);

var data = mlContext.Data.LoadFromTextFile<TweetSentiment>(GetDataPath(TestDatasets.Sentiment.trainFilename),
var data = mlContext.Data.LoadFromTextFile<TweetSentiment>(CommonUtilities.GetDataPath(TestDatasets.Sentiment.trainFilename, DataDir),
hasHeader: TestDatasets.Sentiment.fileHasHeader,
separatorChar: TestDatasets.Sentiment.fileSeparator);

Expand Down Expand Up @@ -167,7 +166,7 @@ void ExtensibilityNormalizeColumns()

// Load the Iris dataset.
var data = mlContext.Data.LoadFromTextFile<Iris>(
GetDataPath(TestDatasets.iris.trainFilename),
CommonUtilities.GetDataPath(TestDatasets.iris.trainFilename, DataDir),
hasHeader: TestDatasets.iris.fileHasHeader,
separatorChar: TestDatasets.iris.fileSeparator);

Expand Down
11 changes: 5 additions & 6 deletions test/Microsoft.ML.Functional.Tests/Debugging.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,15 @@
using System.Collections.Generic;
using Microsoft.ML.Data;
using Microsoft.ML.Functional.Tests.Datasets;
using Microsoft.ML.RunTests;
using Microsoft.ML.TestFramework;
using Microsoft.ML.TestFrameworkCommon;
using Microsoft.ML.Trainers;
using Microsoft.ML.Transforms.Text;
using Xunit;
using Xunit.Abstractions;

namespace Microsoft.ML.Functional.Tests
{
public class Debugging : BaseTestClass
public class Debugging : FunctionalTestBaseClass
{
public Debugging(ITestOutputHelper output) : base(output)
{
Expand Down Expand Up @@ -101,7 +100,7 @@ public void InspectPipelineSchema()
var mlContext = new MLContext(seed: 1);

// Get the dataset.
var data = mlContext.Data.LoadFromTextFile<HousingRegression>(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true);
var data = mlContext.Data.LoadFromTextFile<HousingRegression>(CommonUtilities.GetDataPath(TestDatasets.housing.trainFilename, DataDir), hasHeader: true);

// Define a pipeline
var pipeline = mlContext.Transforms.Concatenate("Features", HousingRegression.Features)
Expand Down Expand Up @@ -130,7 +129,7 @@ public void InspectSchemaUponLoadingData()
var mlContext = new MLContext(seed: 1);

// Get the dataset.
var data = mlContext.Data.LoadFromTextFile<HousingRegression>(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true);
var data = mlContext.Data.LoadFromTextFile<HousingRegression>(CommonUtilities.GetDataPath(TestDatasets.housing.trainFilename, DataDir), hasHeader: true);

// Verify the column names.
int i = 0;
Expand Down Expand Up @@ -168,7 +167,7 @@ public void ViewTrainingOutput()
mlContext.Log += logWatcher.ObserveEvent;

// Get the dataset.
var data = mlContext.Data.LoadFromTextFile<HousingRegression>(GetDataPath(TestDatasets.housing.trainFilename), hasHeader: true);
var data = mlContext.Data.LoadFromTextFile<HousingRegression>(CommonUtilities.GetDataPath(TestDatasets.housing.trainFilename, DataDir), hasHeader: true);

// Define a pipeline
var pipeline = mlContext.Transforms.Concatenate("Features", HousingRegression.Features)
Expand Down
Loading