grab latest from master

.vsts-dotnet-ci.yml

@@ -10,11 +10,12 @@ resources:
   - container: UbuntuContainer
     image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-mlnet-207e097-20190312152303
 
-phases:
-- template: /build/ci/phase-template.yml
+jobs:
+- template: /build/ci/job-template.yml
   parameters:
     name: Centos_x64_NetCoreApp30
     buildScript: ./build.sh
+    container: CentosContainer
     customMatrixes:
       Debug_Build:
         _configuration: Debug-Intrinsics
@@ -24,26 +25,25 @@ phases:
         _configuration: Release-Intrinsics
         _config_short: RI
         _includeBenchmarkData: true
-    queue:
+    pool:
       name: Hosted Ubuntu 1604
-      container: CentosContainer
 
-- template: /build/ci/phase-template.yml
+- template: /build/ci/job-template.yml
   parameters:
     name: Ubuntu_x64_NetCoreApp21
     buildScript: ./build.sh
-    queue:
+    container: UbuntuContainer
+    pool:
       name: Hosted Ubuntu 1604
-      container: UbuntuContainer
 
-- template: /build/ci/phase-template.yml
+- template: /build/ci/job-template.yml
   parameters:
     name: MacOS_x64_NetCoreApp21
     buildScript: ./build.sh
-    queue:
+    pool:
       name: Hosted macOS
 
-- template: /build/ci/phase-template.yml
+- template: /build/ci/job-template.yml
   parameters:
     name: Windows_x64_NetCoreApp30
     buildScript: build.cmd
@@ -56,17 +56,18 @@ phases:
         _configuration: Release-Intrinsics
         _config_short: RI
         _includeBenchmarkData: true
-    queue:
-      name: Hosted VS2017
+    pool:
+      name: NetCorePublic-Pool
+      queue: buildpool.windows.10.amd64.vs2017.open
 
-- template: /build/ci/phase-template.yml
+- template: /build/ci/job-template.yml
   parameters:
     name: Windows_x64_NetCoreApp21
     buildScript: build.cmd
-    queue:
+    pool:
       name: Hosted VS2017
 
-- template: /build/ci/phase-template.yml
+- template: /build/ci/job-template.yml
   parameters:
     name: Windows_x64_NetFx461
     buildScript: build.cmd
@@ -79,13 +80,13 @@ phases:
         _configuration: Release-netfx
         _config_short: RFX
         _includeBenchmarkData: false
-    queue:
+    pool:
       name: Hosted VS2017
 
-- template: /build/ci/phase-template.yml
+- template: /build/ci/job-template.yml
   parameters:
     name: Windows_x86_NetCoreApp21
     architecture: x86
     buildScript: build.cmd
-    queue:
+    pool:
       name: Hosted VS2017

Directory.Build.targets

@@ -21,6 +21,14 @@
       </NativeAssemblyReference>
     </ItemGroup>
 
+    <ItemGroup>
+      <NativeAssemblyReferenceWithMajorVersion>
+          <!-- Tensorflow has a different naming scheme for v1.14.0. Those binaries need to be copied along with the standard names -->
+          <AssemblyPathWithMajorVersion Condition="'$(OS)' != 'Windows_NT'">$(NativeOutputPath)$(LibPrefix)%(NativeAssemblyReferenceWithMajorVersion.Identity)$(LibExtension).%(NativeAssemblyReferenceWithMajorVersion.MajorVersion)</AssemblyPathWithMajorVersion>
+          <AssemblyPathWithMajorVersion Condition="$([MSBuild]::IsOSPlatform('osx'))">$(NativeOutputPath)$(LibPrefix)%(NativeAssemblyReferenceWithMajorVersion.Identity).%(NativeAssemblyReferenceWithMajorVersion.MajorVersion)$(LibExtension)</AssemblyPathWithMajorVersion>
+      </NativeAssemblyReferenceWithMajorVersion>
+    </ItemGroup>
+
     <Copy SourceFiles = "@(NativeAssemblyReference->'%(FullAssemblyPath)')"
           DestinationFolder="$(OutputPath)"
           OverwriteReadOnlyFiles="$(OverwriteReadOnlyFiles)"
@@ -30,7 +38,19 @@
           UseSymboliclinksIfPossible="$(CreateSymbolicLinksForPublishFilesIfPossible)">
       <Output TaskParameter="DestinationFiles" ItemName="FileWrites"/>
     </Copy>
-
+
+    <!-- Optionally copy the native binaries that have a version number attended (Only tensorflow right now) -->
+    <Copy Condition="'$(OS)' != 'Windows_NT'"
+          SourceFiles = "@(NativeAssemblyReferenceWithMajorVersion->'%(AssemblyPathWithMajorVersion)')"
+          DestinationFolder="$(OutputPath)"
+          OverwriteReadOnlyFiles="$(OverwriteReadOnlyFiles)"
+          Retries="$(CopyRetryCount)"
+          RetryDelayMilliseconds="$(CopyRetryDelayMilliseconds)"
+          UseHardlinksIfPossible="$(CreateHardLinksForPublishFilesIfPossible)"
+          UseSymboliclinksIfPossible="$(CreateSymbolicLinksForPublishFilesIfPossible)">
+      <Output TaskParameter="DestinationFiles" ItemName="FileWrites"/>
+    </Copy>
+
   </Target>
 
   <Import Project="$(ToolsDir)/versioning.targets" Condition="Exists('$(ToolsDir)/versioning.targets')" />

build/BranchInfo.props

@@ -15,20 +15,23 @@
       Microsoft.ML.LightGbm;
       Microsoft.ML.Mkl.Components;
       Microsoft.ML.Mkl.Redist;
+      Microsoft.ML.TimeSeries;
+      Microsoft.ML.TensorFlow;
+      Microsoft.ML.OnnxTransformer;
     </StableProjects>
     <IsStableProject Condition="'$(MSBuildProjectName.Contains(.symbols))' == 'false'">$(StableProjects.Contains($(MSBuildProjectName)))</IsStableProject>
     <IsStableProject Condition="'$(MSBuildProjectName.Contains(.symbols))' == 'true'">$(StableProjects.Contains($(MSBuildProjectName.Substring(0, $(MSBuildProjectName.IndexOf(.symbols))))))</IsStableProject>
     <IsStableProject Condition="'$(UseStableVersionForNativeAssets)' == 'true'">true</IsStableProject>
   </PropertyGroup>
   <PropertyGroup Condition="'$(IsStableProject)' == 'true'">
     <MajorVersion>1</MajorVersion>
-    <MinorVersion>2</MinorVersion>
+    <MinorVersion>3</MinorVersion>
     <PatchVersion>0</PatchVersion>
     <PreReleaseLabel>preview</PreReleaseLabel>
   </PropertyGroup>
   <PropertyGroup Condition="'$(IsStableProject)' != 'true'">
     <MajorVersion>0</MajorVersion>
-    <MinorVersion>14</MinorVersion>
+    <MinorVersion>15</MinorVersion>
     <PatchVersion>0</PatchVersion>
     <PreReleaseLabel>preview</PreReleaseLabel>
   </PropertyGroup>

build/Dependencies.props

@@ -15,13 +15,14 @@
     <GoogleProtobufPackageVersion>3.5.1</GoogleProtobufPackageVersion>
     <LightGBMPackageVersion>2.2.3</LightGBMPackageVersion>
     <MicrosoftExtensionsPackageVersion>2.1.0</MicrosoftExtensionsPackageVersion>
-    <MicrosoftMLOnnxRuntimePackageVersion>0.3.0</MicrosoftMLOnnxRuntimePackageVersion>
+    <MicrosoftMLOnnxRuntimePackageVersion>0.4.0</MicrosoftMLOnnxRuntimePackageVersion>
     <MlNetMklDepsPackageVersion>0.0.0.9</MlNetMklDepsPackageVersion>
     <ParquetDotNetPackageVersion>2.1.3</ParquetDotNetPackageVersion>
     <SystemDrawingCommonPackageVersion>4.5.0</SystemDrawingCommonPackageVersion>
     <SystemIOFileSystemAccessControl>4.5.0</SystemIOFileSystemAccessControl>
     <SystemSecurityPrincipalWindows>4.5.0</SystemSecurityPrincipalWindows>
-    <TensorFlowVersion>1.13.1</TensorFlowVersion>
+    <TensorFlowVersion>1.14.0</TensorFlowVersion>
+    <TensorFlowMajorVersion>1</TensorFlowMajorVersion>
   </PropertyGroup>
 
   <!-- Code Analyzer Dependencies -->
@@ -44,9 +45,9 @@
   <PropertyGroup>
     <BenchmarkDotNetVersion>0.11.3</BenchmarkDotNetVersion>
     <MicrosoftCodeAnalysisTestingVersion>1.0.0-beta1-63812-02</MicrosoftCodeAnalysisTestingVersion>
-    <MicrosoftMLTestModelsPackageVersion>0.0.4-test</MicrosoftMLTestModelsPackageVersion>
+    <MicrosoftMLTestModelsPackageVersion>0.0.5-test</MicrosoftMLTestModelsPackageVersion>
     <MicrosoftMLTensorFlowTestModelsVersion>0.0.11-test</MicrosoftMLTensorFlowTestModelsVersion>
-    <MicrosoftMLOnnxTestModelsVersion>0.0.4-test</MicrosoftMLOnnxTestModelsVersion>
+    <MicrosoftMLOnnxTestModelsVersion>0.0.5-test</MicrosoftMLOnnxTestModelsVersion>
   </PropertyGroup>
 
 </Project>

build/ci/phase-template.yml → build/ci/job-template.yml

@@ -2,23 +2,18 @@ parameters:
   name: ''
   architecture: x64
   buildScript: ''
-  queue: {}
+  pool: {}
   customMatrixes: ''
   codeCoverage: false
+  container: ''
 
-phases:
-  - phase: ${{ parameters.name }}
-    variables:
-      _buildScript: ${{ parameters.buildScript }}
-      _phaseName: ${{ parameters.name }}
-      _arch: ${{ parameters.architecture }}
-      _codeCoverage: ${{ parameters.codeCoverage }}
-    queue:
-      ${{ if eq(variables._codeCoverage, 'false') }}:
-        timeoutInMinutes: 30
-      ${{ if eq(variables._codeCoverage, 'true') }}:
-        timeoutInMinutes: 60
-      parallel: 99
+jobs:
+  - job: ${{ parameters.name }}
+    ${{ if eq(parameters.codeCoverage, 'false') }}:
+      timeoutInMinutes: 40
+    ${{ if eq(parameters.codeCoverage, 'true') }}:
+      timeoutInMinutes: 60
+    strategy:
       matrix:
         ${{ if eq(parameters.customMatrixes, '') }}:
           Debug_Build:
@@ -31,28 +26,32 @@ phases:
             _includeBenchmarkData: true
         ${{ if ne(parameters.customMatrixes, '') }}:
           ${{ insert }}: ${{ parameters.customMatrixes }}
-      ${{ insert }}: ${{ parameters.queue }}
+
+    pool: ${{ parameters.pool }}
+    ${{ if ne(parameters.container, '') }}:
+      container: ${{ parameters.container }}
+
     steps:
-    - ${{ if eq(parameters.queue.name, 'Hosted macOS') }}:
+    - ${{ if eq(parameters.pool.name, 'Hosted macOS') }}:
       - script: brew update && brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/f5b1ac99a7fba27c19cee0bc4f036775c889b359/Formula/libomp.rb && brew install mono-libgdiplus gettext && brew link gettext --force && brew link libomp --force
         displayName: Install build dependencies
-    - script: $(_buildScript) -$(_configuration) -buildArch=$(_arch)
+    - script: ${{ parameters.buildScript }} -$(_configuration) -buildArch=${{ parameters.architecture }}
       displayName: Build
-    - script: $(_buildScript) -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=$(_includeBenchmarkData)
+    - script: ${{ parameters.buildScript }} -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=$(_includeBenchmarkData)
       displayName: Download Benchmark Data
-    - script: $(_buildScript) -$(_configuration) -runtests -coverage=$(_codeCoverage)
+    - script: ${{ parameters.buildScript }} -$(_configuration) -runtests -coverage=${{ parameters.codeCoverage }}
       displayName: Run Tests.
     - script: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet msbuild build/Codecoverage.proj /p:CodeCovToken=$(CODECOV_TOKEN)
       displayName: Upload coverage to codecov.io
-      condition: and(succeeded(), eq(variables._codeCoverage, 'true'))
+      condition: and(succeeded(), eq(${{ parameters.codeCoverage }}, True))
     - task: PublishTestResults@2
       displayName: Publish Test Results
       condition: succeededOrFailed()
       inputs:
         testRunner: 'vSTest'
         searchFolder: '$(System.DefaultWorkingDirectory)/bin'
         testResultsFiles: '**/*.trx'
-        testRunTitle: Machinelearning_Tests_$(_phaseName)_$(_configuration)_$(Build.BuildNumber)
+        testRunTitle: Machinelearning_Tests_${{ parameters.name }}_$(_configuration)_$(Build.BuildNumber)
         configuration: $(_configuration)
         mergeTestResults: true
     - task: CopyFiles@2
@@ -78,5 +77,5 @@ phases:
         pathToPublish: $(Build.ArtifactStagingDirectory)
         artifactName: ${{ parameters.name }} $(_config_short)
         artifactType: container
-    - script: $(_buildScript) -buildPackages
+    - script: ${{ parameters.buildScript }} -buildPackages
       displayName: Build Packages

build/codecoverage-ci.yml

@@ -3,7 +3,7 @@
 ################################################################################
 
 phases:
-- template: /build/ci/phase-template.yml
+- template: /build/ci/job-template.yml
   parameters:
     name: Windows_x64
     buildScript: build.cmd

docs/api-reference/io-columns-tree-featurization-binary-classification.md

@@ -0,0 +1,14 @@
+### Input and Output Columns
+The input label column data must be <xref:System.Boolean>.
+The input features column data must be a known-sized vector of <xref:System.Single>.
+
+This estimator outputs the following columns:
+
+| Output Column Name | Column Type | Description|
+| -- | -- | -- |
+| `Trees` | Known-sized vector of <xref:System.Single> | The output values of all trees. Its size is identical to the total number of trees in the tree ensemble model. |
+| `Leaves` | Known-sized vector of <xref:System.Single> | 0-1 vector representation to the IDs of all leaves where the input feature vector falls into. Its size is the number of total leaves in the tree ensemble model. |
+| `Paths` | Known-sized vector of <xref:System.Single> | 0-1 vector representation to the paths the input feature vector passed through to reach the leaves. Its size is the number of non-leaf nodes in the tree ensemble model. |
+
+Those output columns are all optional and user can change their names.
+Please set the names of skipped columns to null so that they would not be produced.

docs/api-reference/io-columns-tree-featurization-ranking.md

@@ -0,0 +1,20 @@
+### Input and Output Columns
+The input label data type must be [key](xref:Microsoft.ML.Data.KeyDataViewType)
+type or <xref:System.Single>. The value of the label determines relevance, where
+higher values indicate higher relevance. If the label is a
+[key](xref:Microsoft.ML.Data.KeyDataViewType) type, then the key index is the
+relevance value, where the smallest index is the least relevant. If the label is a
+<xref:System.Single>, larger values indicate higher relevance. The feature
+column must be a known-sized vector of <xref:System.Single> and input row group
+column must be [key](xref:Microsoft.ML.Data.KeyDataViewType) type.
+
+This estimator outputs the following columns:
+
+| Output Column Name | Column Type | Description|
+| -- | -- | -- |
+| `Trees` | Known-sized vector of <xref:System.Single> | The output values of all trees. Its size is identical to the total number of trees in the tree ensemble model. |
+| `Leaves` | Known-sized vector of <xref:System.Single> | 0-1 vector representation to the IDs of all leaves where the input feature vector falls into. Its size is the number of total leaves in the tree ensemble model. |
+| `Paths` | Known-sized vector of <xref:System.Single> | 0-1 vector representation to the paths the input feature vector passed through to reach the leaves. Its size is the number of non-leaf nodes in the tree ensemble model. |
+
+Those output columns are all optional and user can change their names.
+Please set the names of skipped columns to null so that they would not be produced.

docs/api-reference/io-columns-tree-featurization-regression.md

@@ -0,0 +1,14 @@
+### Input and Output Columns
+The input label column data must be <xref:System.Single>.
+The input features column data must be a known-sized vector of <xref:System.Single>.
+
+This estimator outputs the following columns:
+
+| Output Column Name | Column Type | Description|
+| -- | -- | -- |
+| `Trees` | Known-sized vector of <xref:System.Single> | The output values of all trees. Its size is identical to the total number of trees in the tree ensemble model. |
+| `Leaves` | Known-sized vector of <xref:System.Single> | 0-1 vector representation to the IDs of all leaves where the input feature vector falls into. Its size is the number of total leaves in the tree ensemble model. |
+| `Paths` | Known-sized vector of <xref:System.Single> | 0-1 vector representation to the paths the input feature vector passed through to reach the leaves. Its size is the number of non-leaf nodes in the tree ensemble model. |
+
+Those output columns are all optional and user can change their names.
+Please set the names of skipped columns to null so that they would not be produced.

docs/api-reference/io-time-series-ssa-forecast.md

@@ -0,0 +1,5 @@
+### Input and Output Columns
+There is only one input column.
+The input column must be <xref:System.Single> where a <xref:System.Single> value indicates a value at a timestamp in the time series.
+
+It produces either just one vector of forecasted values or three vectors: a vector of forecasted values, a vector of confidence lower bounds and a vector of confidence upper bounds.

docs/api-reference/regularization-l1-l2.md

@@ -1,6 +1,6 @@
-This class uses [empricial risk minimization](https://en.wikipedia.org/wiki/Empirical_risk_minimization) (i.e., ERM)
+This class uses [empirical risk minimization](https://en.wikipedia.org/wiki/Empirical_risk_minimization) (i.e., ERM)
 to formulate the optimization problem built upon collected data.
-Note that empricial risk is usually measured by applying a loss function on the model's predictions on collected data points.
+Note that empirical risk is usually measured by applying a loss function on the model's predictions on collected data points.
 If the training data does not contain enough data points
 (for example, to train a linear model in $n$-dimensional space, we need at least $n$ data points),
 [overfitting](https://en.wikipedia.org/wiki/Overfitting) may happen so that

docs/api-reference/tree-featurization-prediction.md

@@ -0,0 +1,25 @@
+### Prediction Details
+This estimator produces several output columns from a tree ensemble model. Assume that the model contains only one decision tree:
+
+                   Node 0
+                   /    \
+                 /        \
+               /            \
+             /                \
+           Node 1            Node 2
+           /    \            /    \
+         /        \        /        \
+       /            \     Leaf -3  Node 3
+      Leaf -1      Leaf -2         /    \
+                                 /        \
+                                Leaf -4  Leaf -5
+
+Assume that the input feature vector falls into `Leaf -1`. The output `Trees` may be a 1-element vector where
+the only value is the decision value carried by `Leaf -1`. The output `Leaves` is a 0-1 vector. If the reached
+leaf is the $i$-th (indexed by $-(i+1)$ so the first leaf is `Leaf -1`) leaf in the tree, the $i$-th value in `Leaves`
+would be 1 and all other values would be 0. The output `Paths` is a 0-1 representation of the nodes passed
+through before reaching the leaf. The $i$-th element in `Paths` indicates if the $i$-th node (indexed by $i$) is touched.
+For example, reaching `Leaf -1` lead to $[1, 1, 0, 0]$ as the `Paths`. If there are multiple trees, this estimator
+just concatenates `Trees`'s, `Leaves`'s, `Paths`'s from all trees (first tree's information comes first in the concatenated vectors).
+
+Check the See Also section for links to usage examples.