Merge branch 'master' into faster-calls-2

Author: sharwell

.vsts-dotnet-ci.yml

@@ -64,6 +64,7 @@ jobs:
         _includeBenchmarkData: true
         _targetFramework: netcoreapp3.0
     innerLoop: true
+    vsTestConfiguration: "/Framework:.NETCoreApp,Version=v3.0"
     pool:
       name: Hosted VS2017
 
@@ -72,6 +73,7 @@ jobs:
     name: Windows_x64_NetCoreApp21
     buildScript: build.cmd
     innerLoop: true
+    vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1"
     pool:
       name: Hosted VS2017
 
@@ -91,6 +93,7 @@ jobs:
         _includeBenchmarkData: false
         _targetFramework: win-x64
     innerLoop: true
+    vsTestConfiguration: "/Framework:.NETCoreApp,Version=v4.0"
     pool:
       name: Hosted VS2017
 
@@ -100,5 +103,6 @@ jobs:
     architecture: x86
     buildScript: build.cmd
     innerLoop: true
+    vsTestConfiguration: "/Framework:.NETCoreApp,Version=v2.1"
     pool:
       name: Hosted VS2017

build/.night-build.yml

@@ -38,10 +38,12 @@ jobs:
       Debug_Build:
         _configuration: Debug-netcoreapp3_0
         _config_short: DI
+        _targetFramework: netcoreapp3.0
         _includeBenchmarkData: false
       Release_Build:
         _configuration: Release-netcoreapp3_0
         _config_short: RI
+        _targetFramework: netcoreapp3.0
         _includeBenchmarkData: true
     nightlyBuild: true
     pool:

build/BranchInfo.props

@@ -31,12 +31,12 @@
     <MajorVersion>1</MajorVersion>
     <MinorVersion>5</MinorVersion>
     <PatchVersion>0</PatchVersion>
-    <PreReleaseLabel>preview2</PreReleaseLabel>
+    <PreReleaseLabel>preview3</PreReleaseLabel>
   </PropertyGroup>
   <PropertyGroup Condition="'$(IsStableProject)' != 'true'">
     <MajorVersion>0</MajorVersion>
     <MinorVersion>17</MinorVersion>
     <PatchVersion>0</PatchVersion>
-    <PreReleaseLabel>preview2</PreReleaseLabel>
+    <PreReleaseLabel>preview3</PreReleaseLabel>
   </PropertyGroup>
 </Project>

build/Dependencies.props

@@ -16,7 +16,7 @@
     <GoogleProtobufPackageVersion>3.10.1</GoogleProtobufPackageVersion>
     <LightGBMPackageVersion>2.2.3</LightGBMPackageVersion>
     <MicrosoftExtensionsPackageVersion>2.1.0</MicrosoftExtensionsPackageVersion>
-    <MicrosoftMLOnnxRuntimePackageVersion>1.1.2</MicrosoftMLOnnxRuntimePackageVersion>
+    <MicrosoftMLOnnxRuntimePackageVersion>1.2</MicrosoftMLOnnxRuntimePackageVersion>
     <MlNetMklDepsPackageVersion>0.0.0.9</MlNetMklDepsPackageVersion>
     <ParquetDotNetPackageVersion>2.1.3</ParquetDotNetPackageVersion>
     <SystemDrawingCommonPackageVersion>4.5.0</SystemDrawingCommonPackageVersion>

build/ci/job-template.yml

@@ -9,11 +9,12 @@ parameters:
   innerLoop: false
   runSpecific: false
   container: ''
+  useVSTestTask: false
 
 jobs:
   - job: ${{ parameters.name }}
     ${{ if eq(parameters.nightlyBuild, 'true') }}:
-      timeoutInMinutes: 20
+      timeoutInMinutes: 30
     ${{ if and(eq(parameters.nightlyBuild, 'false'), eq(parameters.codeCoverage, 'false')) }}:
       timeoutInMinutes: 75
     ${{ if eq(parameters.codeCoverage, 'true') }}:
@@ -23,9 +24,10 @@ jobs:
       dotnetPath: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet
       nugetFeed: https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json
       nightlyBuildProjPath: $(Build.SourcesDirectory)/test/Microsoft.ML.NightlyBuild.Tests/Microsoft.ML.NightlyBuild.Tests.csproj
-      nightlyBuildRunPath: (Build.SourcesDirectory)/bin/AnyCPU.(_configuration)/Microsoft.ML.NightlyBuild.Tests/(_targetFramework)
+      nightlyBuildRunPath: $(Build.SourcesDirectory)/bin/AnyCPU.$(_configuration)/Microsoft.ML.NightlyBuild.Tests/$(_targetFramework)
       packageUpdaterProjPath: $(Build.SourcesDirectory)/test/Microsoft.ML.NugetPackageVersionUpdater/Microsoft.ML.NugetPackageVersionUpdater.csproj
       versionFilePath: $(Build.SourcesDirectory)/test/Microsoft.ML.NugetPackageVersionUpdater/latest_versions.txt
+      PROCDUMP_PATH: '$(Build.SourcesDirectory)/Tools/procdump/'
     strategy:
       matrix:
         ${{ if eq(parameters.customMatrixes, '') }}:
@@ -52,7 +54,7 @@ jobs:
         displayName: Install build dependencies
     - ${{ if and( eq(parameters.nightlyBuild, 'true'), eq(parameters.pool.name, 'Hosted Ubuntu 1604')) }}:
       - bash: echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(nightlyBuildRunPath):$LD_LIBRARY_PATH"
-        displayName: Set LD_LIBRARY_PATH for Ubuntu to locate Native shared library in current running path
+        displayName: Set LD_LIBRARY_PATH for Ubuntu and CentOS to locate Native shared library in current running path
     - script: ${{ parameters.buildScript }} -$(_configuration) -buildArch=${{ parameters.architecture }}
       displayName: Build
     - ${{ if eq(parameters.nightlyBuild, 'true') }}:
@@ -62,6 +64,15 @@ jobs:
         displayName: List latest package versions
       - script: $(dotnetPath) run --project $(packageUpdaterProjPath)
         displayName: Update package versions for nightly build
+      - ${{ if eq(parameters.buildScript, 'build.cmd') }}:
+        - powershell: |
+                    Get-ChildItem -Path  '.\bin\AnyCPU.*' -Recurse |
+                    Select -ExpandProperty FullName |
+                    Where {$_ -notlike '*\Microsoft.ML.NightlyBuild.Tests*'} |
+                    sort length -Descending | 
+                    Remove-Item -force 
+                    Write-Output "Done cleaning up usless project..."
+          displayName: Clean up useless project
       - script: $(dotnetPath) msbuild -restore $(nightlyBuildProjPath) /p:ReferenceTypeForTestFramework="Nuget" /p:Configuration=$(_configuration) /p:TargetArchitecture=${{ parameters.architecture }}
         displayName: Build Nightly-Build Project with latest package versions
       - script: ${{ parameters.buildScript }} -$(_configuration) -runnightlybuildtests
@@ -70,12 +81,33 @@ jobs:
       - script: ${{ parameters.buildScript }} -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=$(_includeBenchmarkData)
         displayName: Download Benchmark Data
       - ${{ if eq(parameters.innerLoop, 'false') }}:
-        - ${{ if eq(parameters.runSpecific, 'false') }}:
+        - ${{ if and(eq(parameters.runSpecific, 'false'), eq(parameters.useVSTestTask, 'false')) }}:
           - script: ${{ parameters.buildScript }} -$(_configuration) -runtests -coverage=${{ parameters.codeCoverage }}
             displayName: Run All Tests.
-        - ${{ if eq(parameters.runSpecific, 'true') }}:
+        - ${{ if and(eq(parameters.runSpecific, 'true'), eq(parameters.useVSTestTask, 'false')) }}:
           - script: ${{ parameters.buildScript }} -$(_configuration) -runSpecificTests -coverage=${{ parameters.codeCoverage }}
             displayName: Run Specific Tests.
+        - ${{ if and(eq(parameters.buildScript, 'build.cmd'), eq(parameters.useVSTestTask, 'true')) }}:
+          - task: VSTest@2
+            displayName: Run Tests with VSTest
+            inputs:
+              testSelector: testAssemblies
+              testAssemblyVer2: |
+                **\*test.dll
+                **\*tests.dll
+                !**\obj\** 
+              runSettingsFile: $(Build.SourcesDirectory)/tools-local/vstest.runsettings
+              searchFolder: '$(System.DefaultWorkingDirectory)'
+              vstestLocationMethod: 'version' 
+              vsTestVersion: 'latest'
+              runInParallel: False
+              runTestsInIsolation: True
+              codeCoverageEnabled: ${{ parameters.codeCoverage }}
+              otherConsoleOptions: ${{ parameters.vsTestConfiguration }}
+              dontDistribute: True
+              diagnosticsEnabled: False
+              collectDumpOn: onAbortOnly
+              publishRunAttachments: true
       - ${{ if eq(parameters.innerLoop, 'true') }}:
         - script: ${{ parameters.buildScript }} -$(_configuration) -runCITests -coverage=${{ parameters.codeCoverage }}
           displayName: Run CI Tests.
@@ -114,7 +146,7 @@ jobs:
       inputs:
         sourceFolder: $(Build.SourcesDirectory)
         contents: |
-          CrashDumps/*.dmp
+          *.dmp
           bin/**/*.pdb
         targetFolder: $(Build.ArtifactStagingDirectory)
     - task: PublishBuildArtifacts@1

docs/api-reference/time-series-pvalue.md

@@ -7,7 +7,7 @@ The lower its value, the more likely it is a spike. The p-value score is always
 
 This score is the p-value of the current computed raw score according to a distribution of raw scores.
 Here, the distribution is estimated based on the most recent raw score values up to certain depth back in the history.
-More specifically, this distribution is estimated using [kernel density estimation (https://en.wikipedia.org/wiki/Kernel_density_estimation) with the Gaussian [kernels](https://en.wikipedia.org/wiki/Kernel_(statistics)#In_non-parametric_statistics) of adaptive bandwidth.
+More specifically, this distribution is estimated using [kernel density estimation](https://en.wikipedia.org/wiki/Kernel_density_estimation) with the Gaussian [kernels](https://en.wikipedia.org/wiki/Kernel_(statistics)#In_non-parametric_statistics) of adaptive bandwidth.
 
 If the p-value score exceeds $1 - \frac{\text{confidence}}{100}$, the associated timestamp may get a non-zero alert value in spike detection, which means a spike point is detected.
 Note that $\text{confidence}$ is defined in the signatures of [DetectIidSpike](xref:Microsoft.ML.TimeSeriesCatalog.DetectIidSpike(Microsoft.ML.TransformsCatalog,System.String,System.String,System.Int32,System.Int32,Microsoft.ML.Transforms.TimeSeries.AnomalySide))

docs/project-docs/developer-guide.md

@@ -78,3 +78,13 @@ Steps to update `core_manifest.json` and `core_ep-list.tsv`:
 3. Verify the changes to `core_manifest.json` and `core_ep-list.tsv` are correct.
 4. Re-enable the skip attribute on the `RegenerateEntryPointCatalog` test.
 5. Commit the updated `core_manifest.json` and `core_ep-list.tsv` files to your branch.
+
+### Running unit tests through VSTest Task & Collecting memory dumps
+
+During development, there may also arise a need to debug hanging tests. In this scenario, it can be beneficial to collect the memory dump while a given test is hanging.
+
+In this case, the given needs needs to be implemented according to the Microsoft test framework. Please check out the [Microsoft test framework walkthrough](https://docs.microsoft.com/en-us/visualstudio/test/walkthrough-creating-and-running-unit-tests-for-managed-code?view=vs-2019) and the VSTest [sample](https://github.com/dotnet/samples/tree/master/core/getting-started/unit-testing-using-mstest) demonstrating the "TestClass", "TestMethod", "DataTestMethod", and "DataRow" attributes.
+
+Once the unit test(s) are implemented according to VSTest and ready to be debugged, the `useVSTestTask` parameter in `build\ci\job-template.yml` needs to be set to `True`. Once these steps are completed and pushed in your pull request, the unit test(s) will run and produce a full memory dump. At the end of a run, the memory dump `.dmp` file will be availible for downloading and inspection in the published artifacts of the build, in the folder `TestResults`.
+
+Note: this is only supported on Windows builds, as [ProcDump](https://docs.microsoft.com/en-us/sysinternals/downloads/procdump) is officially only available on Windows.

docs/release-notes/1.5.0-preview2/release-1.5.0-preview2.md

@@ -0,0 +1,89 @@
+# [ML.NET](http://dot.net/ml) 1.5.0-preview2
+
+## **New Features (IN-PREVIEW, please provide feedback)**
+- **TimeSeriesImputer** ([#4623](https://github.com/dotnet/machinelearning/pull/4623)) This data transformer can be used to impute missing rows in time series data.
+- **LDSVM Trainer** ([#4060](https://github.com/dotnet/machinelearning/pull/4060)) The "Local Deep SVM" usess trees as its SVM kernel to create a non-linear binary trainer. A sample can be found [here](https://github.com/dotnet/machinelearning/blob/c819d77e9250c68883713d5f1cd79b8971a11faf/docs/samples/Microsoft.ML.Samples/Dynamic/Trainers/BinaryClassification/LdSvm.cs).
+- **Onnxruntime updated to v1.2** This also includes support for GPU execution of onnx models
+- **Export-to-ONNX for below components:**
+    - SlotsDroppingTransformer ([#4562](https://github.com/dotnet/machinelearning/pull/4562))
+    - ColumnSelectingTransformer ([#4590](https://github.com/dotnet/machinelearning/pull/4590))
+    - VectorWhiteningTransformer ([#4577](https://github.com/dotnet/machinelearning/pull/4577))
+    - NaiveBayesMulticlassTrainer ([#4636](https://github.com/dotnet/machinelearning/pull/4636))
+  	- PlattCalibratorTransformer ([#4699](https://github.com/dotnet/machinelearning/pull/4699))
+  	- TokenizingByCharactersTransformer ([#4805](https://github.com/dotnet/machinelearning/pull/4805))
+  	- TextNormalizingTransformer ([#4781](https://github.com/dotnet/machinelearning/pull/4781))
+
+  
+## **Bug Fixes**
+  - Fix issue in WaiterWaiter caused by race condition ([#4829](https://github.com/dotnet/machinelearning/pull/4829))
+	- Onnx Export change to allow for running inference on multiple rows in OnnxRuntime ([#4783](https://github.com/dotnet/machinelearning/pull/4783))
+  - Data splits to default to MLContext seed when not specified ([#4764](https://github.com/dotnet/machinelearning/pull/4764))
+  - Add Seed property to MLContext and use as default for data splits ([#4775](https://github.com/dotnet/machinelearning/pull/4775))
+- **Onnx bug fixes**
+  - Updating onnxruntime version ([#4882](https://github.com/dotnet/machinelearning/pull/4882))
+  - Calculate ReduceSum row by row in ONNX model from OneVsAllTrainer ([#4904](https://github.com/dotnet/machinelearning/pull/4904))
+  - Several onnx export fixes related to KeyToValue and ValueToKey transformers ([#4900](https://github.com/dotnet/machinelearning/pull/4900), [#4866](https://github.com/dotnet/machinelearning/pull/4866), [#4841](https://github.com/dotnet/machinelearning/pull/4841), [#4889](https://github.com/dotnet/machinelearning/pull/4889), [#4878](https://github.com/dotnet/machinelearning/pull/4878), [#4797](https://github.com/dotnet/machinelearning/pull/4797))
+  - Fixes to onnx export for text related transforms ([#4891](https://github.com/dotnet/machinelearning/pull/4891), [#4813](https://github.com/dotnet/machinelearning/pull/4813))
+  - Fixed bugs in OptionalColumnTransform and ColumnSelecting ([#4887](https://github.com/dotnet/machinelearning/pull/4887), [#4815](https://github.com/dotnet/machinelearning/pull/4815))
+  - Alternate solution for ColumnConcatenatingTransformer ([#4875](https://github.com/dotnet/machinelearning/pull/4875))
+  - Added slot names support for OnnxTransformer ([#4857](https://github.com/dotnet/machinelearning/pull/4857))
+  - Fixed output schema of OnnxTransformer ([#4849](https://github.com/dotnet/machinelearning/pull/4849))
+  - Changed Binarizer node to be cast to the type of the predicted label … ([#4818](https://github.com/dotnet/machinelearning/pull/4818))
+  - Fix for OneVersusAllTrainer ([#4698](https://github.com/dotnet/machinelearning/pull/4698)) 
+  - Enable OnnxTransformer to accept KeyDataViewTypes as if they were UInt32 ([#4824](https://github.com/dotnet/machinelearning/pull/4824))
+  - Fix off by 1 error with the cats_int64s attribute for the OneHotEncoder ONNX operator ([#4827](https://github.com/dotnet/machinelearning/pull/4827))
+  - Changed Binarizer node to be cast to the type of the predicted label … ([#4818](https://github.com/dotnet/machinelearning/pull/4818))
+  - Updated handling of missing values with LightGBM, and added ability to use (0) as missing value ([#4695](https://github.com/dotnet/machinelearning/pull/4695))
+  - Double cast to float for some onnx estimators  ([#4745](https://github.com/dotnet/machinelearning/pull/4745))
+  - Fix onnx output name for GcnTransform ([#4786](https://github.com/dotnet/machinelearning/pull/4786))
+- Added support to run PFI on uncalibrated binary classification models ([#4587](https://github.com/dotnet/machinelearning/pull/4587))
+- Fix bug in WordBagEstimator when training on empty data ([#4696](https://github.com/dotnet/machinelearning/pull/4696))
+- Added Cancellation mechanism to Image Classification (through the experimental nuget) (fixes #4632) ([#4650](https://github.com/dotnet/machinelearning/pull/4650))
+-	Changed F1 score to return 0 instead of NaN when Precision + Recall is 0 ([#4674](https://github.com/dotnet/machinelearning/pull/4674))
+-	TextLoader, BinaryLoader and SvmLightLoader now check the existence of the input file before training ([#4665](https://github.com/dotnet/machinelearning/pull/4665))
+- ImageLoadingTransformer now checks the existence of input folder before training ([#4691]( https://github.com/dotnet/machinelearning/pull/4691))
+- Use random file name for AutoML experiment folder ([#4657](https://github.com/dotnet/machinelearning/pull/4657))
+- Using invariance culture when converting to string  ([#4635](https://github.com/dotnet/machinelearning/pull/4635))
+- Fix NullReferenceException when it comes to Recommendation in AutoML and CodeGenerator ([#4774](https://github.com/dotnet/machinelearning/pull/4774))
+
+## **Enhancements**
+- Added in support for System.DateTime type for the DateTimeTransformer ([#4661](https://github.com/dotnet/machinelearning/pull/4661))
+- Additional changes to ExpressionTransformer ([#4614](https://github.com/dotnet/machinelearning/pull/4614))
+- Optimize generic MethodInfo for Func<TResult> ([#4588](https://github.com/dotnet/machinelearning/pull/4588))
+- Data splits to default to MLContext seed when not specified ([#4764](https://github.com/dotnet/machinelearning/pull/4764))
+- Added in DateTime type support for TimeSeriesImputer ([#4812](https://github.com/dotnet/machinelearning/pull/4812))
+
+## **Test updates**
+- **Code analysis updates**
+  - Update analyzer test library ([#4740](https://github.com/dotnet/machinelearning/pull/4740))
+  - Enable the internal code analyzer for test projects ([#4731](https://github.com/dotnet/machinelearning/pull/4731))
+  - Implement MSML_ExtendBaseTestClass (Test classes should be derived from BaseTestClass) ([#4746](https://github.com/dotnet/machinelearning/pull/4746))
+  - Enable MSML_TypeParamName for the full solution ([#4762](https://github.com/dotnet/machinelearning/pull/4762))
+  - Enable MSML_ParameterLocalVarName for the full solution ([#4833](https://github.com/dotnet/machinelearning/pull/4833))
+  - Enable MSML_SingleVariableDeclaration for the full solution ([#4765](https://github.com/dotnet/machinelearning/pull/4765))
+- **Better logging from tests**
+  - Ensure tests capture the full log ([#4710](https://github.com/dotnet/machinelearning/pull/4710))
+  - Fix failure to capture test failures ([#4716](https://github.com/dotnet/machinelearning/pull/4716))
+  - Collect crash dump upload dump and pdb to artifact ([#4666](https://github.com/dotnet/machinelearning/pull/4666))
+- Enable Conditional Numerical Reproducibility for tests ([#4569](https://github.com/dotnet/machinelearning/pull/4569))
+- Changed all MLContext creation to include a fixed seed ([#4736](https://github.com/dotnet/machinelearning/pull/4736))
+- Fix incorrect SynchronizationContext use in TestSweeper ([#4779](https://github.com/dotnet/machinelearning/pull/4779))
+
+## **Documentation Updates**
+- Update cookbook to latest API ([#4706](https://github.com/dotnet/machinelearning/pull/4706))
+- Update documentation to stop mentioning interfaces that no longer exist ([#4673](https://github.com/dotnet/machinelearning/pull/4673))
+- Roadmap update ([#4704](https://github.com/dotnet/machinelearning/pull/4704))
+- Added release process documentation to README.md ([#4402](https://github.com/dotnet/machinelearning/pull/4402))
+- Fix documentation of SvmLightLoader ([#4616](https://github.com/dotnet/machinelearning/pull/4616))
+- Correct KMeans scoring function doc ([#4705](https://github.com/dotnet/machinelearning/pull/4705))
+- Several typo fixes thanks to [@MaherJendoubi](https://github.com/MaherJendoubi) ([#4627](https://github.com/dotnet/machinelearning/pull/4627), [#4631](https://github.com/dotnet/machinelearning/pull/4631), [#4626](https://github.com/dotnet/machinelearning/pull/4626) [#4617](https://github.com/dotnet/machinelearning/pull/4617), [#4633](https://github.com/dotnet/machinelearning/pull/4633), [#4629](https://github.com/dotnet/machinelearning/pull/4629), [#4642](https://github.com/dotnet/machinelearning/pull/4642))
+- Other typo fixes: ([#4628](https://github.com/dotnet/machinelearning/pull/4628), [#4685](https://github.com/dotnet/machinelearning/pull/4685), [#4885](https://github.com/dotnet/machinelearning/pull/4885))
+
+
+## **Breaking Changes**
+- None
+
+
+
+
+