justinormont · justinormont · Jul 29, 2018 · Jun 12, 2018 · Jun 12, 2018 · Jun 12, 2018
diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml
@@ -0,0 +1,15 @@
+phases:
+- template: /build/ci/phase-template.yml
+  parameters:
+    name: Linux
+    buildScript: ./build.sh
+    dockerImage: microsoft/dotnet-buildtools-prereqs:centos-7-b46d863-20180719033416
+
+- template: /build/ci/phase-template.yml
+  parameters:
+    name: Windows_NT
+    buildScript: build.cmd
+    queue:
+      name: Hosted VS2017
+      demands:
+        - agent.os -equals Windows_NT
diff --git a/Directory.Build.props b/Directory.Build.props
@@ -14,13 +14,15 @@
   <PropertyGroup>
     <RestoreSources>
       https://api.nuget.org/v3/index.json;
+      https://dotnet.myget.org/F/dotnet-core/api/v3/index.json;
     </RestoreSources>
   </PropertyGroup>
 
   <!-- Common repo directories -->
   <PropertyGroup>
     <RepoRoot>$(MSBuildThisFileDirectory)</RepoRoot>
     <SourceDir>$(RepoRoot)src/</SourceDir>
+    <PkgDir>$(RepoRoot)pkg/</PkgDir>
 
     <!-- Output directories -->
     <BinDir Condition="'$(BinDir)'==''">$(RepoRoot)bin/</BinDir>

diff --git a/Directory.Build.targets b/Directory.Build.targets
@@ -5,5 +5,33 @@
            Text="The tools directory [$(ToolsDir)] does not exist. Please run build in the root of the repo to ensure the tools are installed before attempting to build an individual project." />
   </Target>
 
+  <Target Name="CopyNativeAssemblies"
+          BeforeTargets="PrepareForRun">
+
+    <PropertyGroup>
+      <LibPrefix Condition="'$(OS)' != 'Windows_NT'">lib</LibPrefix>
+      <LibExtension Condition="'$(OS)' == 'Windows_NT'">.dll</LibExtension>
+      <LibExtension Condition="'$(OS)' != 'Windows_NT'">.so</LibExtension>
+      <LibExtension Condition="$([MSBuild]::IsOSPlatform('osx'))">.dylib</LibExtension>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <NativeAssemblyReference>
+        <FullAssemblyPath>$(NativeOutputPath)$(LibPrefix)%(NativeAssemblyReference.Identity)$(LibExtension)</FullAssemblyPath>
+      </NativeAssemblyReference>
+    </ItemGroup>
+
+    <Copy SourceFiles = "@(NativeAssemblyReference->'%(FullAssemblyPath)')"
+          DestinationFolder="$(OutputPath)"
+          OverwriteReadOnlyFiles="$(OverwriteReadOnlyFiles)"
+          Retries="$(CopyRetryCount)"
+          RetryDelayMilliseconds="$(CopyRetryDelayMilliseconds)"
+          UseHardlinksIfPossible="$(CreateHardLinksForPublishFilesIfPossible)"
+          UseSymboliclinksIfPossible="$(CreateSymbolicLinksForPublishFilesIfPossible)">
+      <Output TaskParameter="DestinationFiles" ItemName="FileWrites"/>
+    </Copy>
+
+  </Target>
+
   <Import Project="$(ToolsDir)/versioning.targets" Condition="Exists('$(ToolsDir)/versioning.targets')" />
 </Project>
diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln
diff --git a/README.md b/README.md
@@ -18,7 +18,7 @@ Along with these ML capabilities this first release of ML.NET also brings the fi
 
 ML.NET runs on Windows, Linux, and macOS - any platform where 64 bit [.NET Core](https://github.com/dotnet/core) or later is available.
 
-The current release is 0.2. Check out the [release notes](docs/release-notes/0.2/release-0.2.md).
+The current release is 0.3. Check out the [release notes](docs/release-notes/0.3/release-0.3.md).
 
 First ensure you have installed [.NET Core 2.0](https://www.microsoft.com/net/learn/get-started) or later. ML.NET also works on the .NET Framework. Note that ML.NET currently must run in a 64 bit process.
 
@@ -44,9 +44,9 @@ To build ML.NET from source please visit our [developers guide](docs/project-doc
 
 |    | x64 Debug | x64 Release |
 |:---|----------------:|------------------:|
-|**Linux**|[![x64-debug](https://ci2.dot.net/job/dotnet_machinelearning/job/master/job/linux_debug/badge/icon)](https://ci2.dot.net/job/dotnet_machinelearning/job/master/job/linux_debug/lastCompletedBuild)|[![x64-release](https://ci2.dot.net/job/dotnet_machinelearning/job/master/job/linux_release/badge/icon)](https://ci2.dot.net/job/dotnet_machinelearning/job/master/job/linux_release/lastCompletedBuild)|
+|**Linux**|[![x64-debug](https://dotnet.visualstudio.com/public/_apis/build/status/104?branch=master)](https://dotnet.visualstudio.com/DotNet-Public/_build/latest?definitionId=104&branch=master)|[![x64-release](https://dotnet.visualstudio.com/public/_apis/build/status/104?branch=master)](https://dotnet.visualstudio.com/DotNet-Public/_build/latest?definitionId=104&branch=master)|
 |**macOS**|[![x64-debug](https://ci2.dot.net/job/dotnet_machinelearning/job/master/job/osx10.13_debug/badge/icon)](https://ci2.dot.net/job/dotnet_machinelearning/job/master/job/osx10.13_debug/lastCompletedBuild)|[![x64-release](https://ci2.dot.net/job/dotnet_machinelearning/job/master/job/osx10.13_release/badge/icon)](https://ci2.dot.net/job/dotnet_machinelearning/job/master/job/osx10.13_release/lastCompletedBuild)|
-|**Windows**|[![x64-debug](https://ci2.dot.net/job/dotnet_machinelearning/job/master/job/windows_nt_debug/badge/icon)](https://ci2.dot.net/job/dotnet_machinelearning/job/master/job/windows_nt_debug/lastCompletedBuild)|[![x64-release](https://ci2.dot.net/job/dotnet_machinelearning/job/master/job/windows_nt_release/badge/icon)](https://ci2.dot.net/job/dotnet_machinelearning/job/master/job/windows_nt_release/lastCompletedBuild)|
+|**Windows**|[![x64-debug](https://dotnet.visualstudio.com/public/_apis/build/status/104?branch=master)](https://dotnet.visualstudio.com/DotNet-Public/_build/latest?definitionId=104&branch=master)|[![x64-release](https://dotnet.visualstudio.com/public/_apis/build/status/104?branch=master)](https://dotnet.visualstudio.com/DotNet-Public/_build/latest?definitionId=104&branch=master)|
 
 ## Contributing
 
@@ -84,6 +84,9 @@ SentimentPrediction prediction = model.Predict(data);
 
 Console.WriteLine("prediction: " + prediction.Sentiment);
 ```
+## Samples
+
+We have a [repo of samples](https://github.com/dotnet/machinelearning-samples) that you can look at.
 
 ## License
 

diff --git a/build/BranchInfo.props b/build/BranchInfo.props
@@ -1,7 +1,7 @@
 <Project>
   <PropertyGroup>
     <MajorVersion>0</MajorVersion>
-    <MinorVersion>3</MinorVersion>
+    <MinorVersion>4</MinorVersion>
     <PatchVersion>0</PatchVersion>
     <PreReleaseLabel>preview</PreReleaseLabel>
   </PropertyGroup>

diff --git a/build/Dependencies.props b/build/Dependencies.props
@@ -6,7 +6,7 @@
     <SystemThreadingTasksDataflowPackageVersion>4.8.0</SystemThreadingTasksDataflowPackageVersion>
     <SystemCodeDomPackageVersion>4.4.0</SystemCodeDomPackageVersion>
     <SystemReflectionEmitLightweightPackageVersion>4.3.0</SystemReflectionEmitLightweightPackageVersion>
-    <SystemValueTupleVersion>4.4.0</SystemValueTupleVersion>
     <PublishSymbolsPackageVersion>1.0.0-beta-62824-02</PublishSymbolsPackageVersion>
+    <LightGBMPackageVersion>2.1.2.2</LightGBMPackageVersion>
   </PropertyGroup>
 </Project>
diff --git a/build/ci/phase-template.yml b/build/ci/phase-template.yml
@@ -0,0 +1,37 @@
+parameters:
+  name: ''
+  buildScript: ''
+  dockerImage: ''
+  queue: {}
+
+phases:
+  - phase: ${{ parameters.name }}
+    variables:
+      _buildScript: ${{ parameters.buildScript }}
+      _phaseName: ${{ parameters.name }}
+      # if dockerImage is not equal to '' then run under docker container
+      ${{ if ne(parameters.dockerImage, '') }}:
+        _PREVIEW_VSTS_DOCKER_IMAGE: ${{ parameters.dockerImage }}
+    queue:
+      parallel: 2
+      matrix:
+        Build_Debug:
+          _configuration: Debug
+        Build_Release:
+          _configuration: Release 
+      ${{ insert }}: ${{ parameters.queue }}
+    steps:
+    - script: $(_buildScript) -$(_configuration) -runtests
+      displayName: Build and Test
+    - task: PublishTestResults@2
+      displayName: Publish Test Results
+      condition: succeededOrFailed()
+      inputs:
+        testRunner: 'vSTest'
+        searchFolder: '$(System.DefaultWorkingDirectory)/bin'
+        testResultsFiles: '**/*.trx'
+        testRunTitle: Machinelearning_Tests_$(_phaseName)_$(_configuration)_$(Build.BuildNumber)
+        configuration: $(_configuration)
+        mergeTestResults: true
+    - script: $(_buildScript) -buildPackages
+      displayName: Build Packages
diff --git a/docs/building/unix-instructions.md b/docs/building/unix-instructions.md
@@ -42,9 +42,11 @@ macOS 10.12 or higher is needed to build dotnet/machinelearning.
 
 On macOS a few components are needed which are not provided by a default developer setup:
 * cmake 3.10.3
+* gcc
 * All the requirements necessary to run .NET Core 2.0 applications. To view macOS prerequisites click [here](https://docs.microsoft.com/en-us/dotnet/core/macos-prerequisites?tabs=netcore2x).
 
-One way of obtaining CMake is via [Homebrew](http://brew.sh):
+One way of obtaining CMake and gcc is via [Homebrew](http://brew.sh):
 ```sh
 $ brew install cmake
+$ brew install gcc
 ```
diff --git a/docs/code/EntryPoints.md b/docs/code/EntryPoints.md
@@ -0,0 +1,231 @@
+# Entry Points And Helper Classes 
+
+## Overview
+
+Entry points are a way to interface with ML.NET components, by specifying an execution graph of connected inputs and outputs of those components.
+Both the manifest describing available components and their inputs/outputs, and an "experiment" graph description, are expressed in JSON. 
+The recommended way of interacting with ML.NET through other, non-.NET programming languages, is by composing, and exchanging pipelines or experiment graphs.  
+
+Through the documentation, we also refer to entry points as 'entry points nodes', and that is because they are the nodes of the graph representing the experiment. 
+The graph 'variables', the various values of the experiment graph JSON properties, serve to describe the relationship between the entry point nodes. 
+The 'variables' are therefore the edges of the DAG (Directed Acyclic Graph). 
+
+All of ML.NET entry points are described by their manifest. The manifest is another JSON object that documents and describes the structure of an entry points. 
+Manifests are referenced to understand what an entry point does, and how it should be constructed, in a graph.  
+
+This document briefly describes the structure of the entry points, the structure of an entry point manifest, and mentions the ML.NET classes that help construct an entry point graph.
+
+## EntryPoint manifest - the definition of an entry point
+
+The components manifest is build by scanning the ML.NET assemblies through reflection and searching for types having the: `SignatureEntryPointModule` signature in their `LoadableClass` assembly  attribute definition. 
+An example of an entry point manifest object, specifically for the `ColumnTypeConverter` transform, is:
+
+```javascript
+{
+    "Name": "Transforms.ColumnTypeConverter",
+    "Desc": "Converts a column to a different type, using standard conversions.",
+    "FriendlyName": "Convert Transform",
+    "ShortName": "Convert",
+    "Inputs": [
+        {   "Name": "Column",
+            "Type": {
+                "Kind": "Array",
+                "ItemType": {
+                    "Kind": "Struct",
+                    "Fields": [
+                        {
+                            "Name": "ResultType",
+                            "Type": {
+                                "Kind": "Enum",
+                                "Values": [ "I1","I2","U2","I4","U4","I8","U8","R4","Num","R8","TX","Text","TXT","BL","Bool","TimeSpan","TS","DT","DateTime","DZ","DateTimeZone","UG","U16" ]
+                            },
+                            "Desc": "The result type",
+                            "Aliases": [ "type" ],
+                            "Required": false,
+                            "SortOrder": 150,
+                            "IsNullable": true,
+                            "Default": null
+                        },
+                        {   "Name": "Range",
+                            "Type": "String",
+                            "Desc": "For a key column, this defines the range of values",
+                            "Aliases": [ "key" ],
+                            "Required": false,
+                            "SortOrder": 150,
+                            "IsNullable": false,
+                            "Default": null
+                        },
+                        {   "Name": "Name",
+                            "Type": "String",
+                            "Desc": "Name of the new column",
+                            "Aliases": [ "name" ],
+                            "Required": false,
+                            "SortOrder": 150,
+                            "IsNullable": false,
+                            "Default": null
+                        },
+                        {   "Name": "Source",
+                            "Type": "String",
+                            "Desc": "Name of the source column",
+                            "Aliases": [ "src" ],
+                            "Required": false,
+                            "SortOrder": 150,
+                            "IsNullable": false,
+                            "Default": null
+                        }
+                    ]
+                }
+            },
+            "Desc": "New column definition(s) (optional form: name:type:src)",
+            "Aliases": [ "col" ],
+            "Required": true,
+            "SortOrder": 1,
+            "IsNullable": false
+        },
+        {   "Name": "Data",
+            "Type": "DataView",
+            "Desc": "Input dataset",
+            "Required": true,
+            "SortOrder": 2,
+            "IsNullable": false
+        },
+        {   "Name": "ResultType",
+            "Type": {
+                "Kind": "Enum",
+                "Values": [ "I1","I2","U2","I4","U4","I8","U8","R4","Num","R8","TX","Text","TXT","BL","Bool","TimeSpan","TS","DT","DateTime","DZ","DateTimeZone","UG","U16" ]
+            },
+            "Desc": "The result type",
+            "Aliases": [ "type" ],
+            "Required": false,
+            "SortOrder": 2,
+            "IsNullable": true,
+            "Default": null
+        },
+        {   "Name": "Range",
+            "Type": "String",
+            "Desc": "For a key column, this defines the range of values",
+            "Aliases": [ "key" ],
+            "Required": false,
+            "SortOrder": 150,
+            "IsNullable": false,
+            "Default": null
+        }
+    ],
+    "Outputs": [
+	    {
+            "Name": "OutputData",
+            "Type": "DataView",
+            "Desc": "Transformed dataset" 
+        },
+        {
+            "Name": "Model",
+            "Type": "TransformModel",
+            "Desc": "Transform model"
+        }
+    ],
+    "InputKind": ["ITransformInput" ],
+    "OutputKind": [ "ITransformOutput" ]
+}
+```
+
+The respective entry point, constructed based on this manifest would be:
+
+```javascript
+    {
+        "Name": "Transforms.ColumnTypeConverter",
+        "Inputs": {
+            "Column": [{ 
+            "Name": "Features",
+                    "Source": "Features"
+                }],
+            "Data": "$data0",
+            "ResultType": "R4"
+        },
+        "Outputs": {
+            "OutputData": "$Convert_Output",
+            "Model": "$Convert_TransformModel"
+        }
+    }
+```
+
+## `EntryPointGraph`
+
+This class encapsulates the list of nodes (`EntryPointNode`) and edges
+(`EntryPointVariable` inside a `RunContext`) of the graph.
+
+## `EntryPointNode`
+
+This class represents a node in the graph, and wraps an entry point call. It
+has methods for creating and running entry points. It also has a reference to
+the `RunContext` to allow it to get and set values from `EntryPointVariable`s.
+
+To express the inputs that are set through variables, a set of dictionaries
+are used. The `InputBindingMap` maps an input parameter name to a list of
+`ParameterBinding`s. The `InputMap` maps a `ParameterBinding` to a
+`VariableBinding`.  For example, if the JSON looks like this:
+
+```javascript
+'foo': '$bar'
+```
+
+the `InputBindingMap` will have one entry that maps the string "foo" to a list
+that has only one element, a `SimpleParameterBinding` with the name "foo" and
+the `InputMap` will map the `SimpleParameterBinding` to a
+`SimpleVariableBinding` with the name "bar". For a more complicated example,
+let's say we have this JSON:
+
+```javascript
+'foo': [ '$bar[3]', '$baz']
+```
+
+the `InputBindingMap` will have one entry that maps the string "foo" to a list
+that has two elements, an `ArrayIndexParameterBinding` with the name "foo" and
+index 0 and another one with index 1. The `InputMap` will map the first
+`ArrayIndexParameterBinding` to an `ArrayIndexVariableBinding` with name "bar"
+and index 3 and the second `ArrayIndexParameterBinding` to a
+`SimpleVariableBinding` with the name "baz".
+
+For outputs, a node assumes that an output is mapped to a variable, so the
+`OutputMap` is a simple dictionary from string to string.
+
+## `EntryPointVariable`
+
+This class represents an edge in the entry point graph. It has a name, a type
+and a value. Variables can be simple, arrays and/or dictionaries. Currently,
+only data views, file handles, predictor models and transform models are
+allowed as element types for a variable.
+
+## `RunContext`
+
+This class is just a container for all the variables in a graph.
+
+## `VariableBinding` and Derived Classes
+
+The abstract base class represents a "pointer to a (part of a) variable". It
+is used in conjunction with `ParameterBinding`s to specify inputs to an entry
+point node. The `SimpleVariableBinding` is a pointer to an entire variable,
+the `ArrayIndexVariableBinding` is a pointer to a specific index in an array
+variable, and the `DictionaryKeyVariableBinding` is a pointer to a specific
+key in a dictionary variable.
+
+## `ParameterBinding` and Derived Classes
+
+The abstract base class represents a "pointer to a (part of a) parameter". It
+parallels the `VariableBinding` hierarchy and it is used to specify the inputs
+to an entry point node. The `SimpleParameterBinding` is a pointer to a
+non-array, non-dictionary parameter, the `ArrayIndexParameterBinding` is a
+pointer to a specific index of an array parameter and the
+`DictionaryKeyParameterBinding` is a pointer to a specific key of a dictionary
+parameter.
+
+## How to create an entry point for an existing ML.NET component
+
+The steps to take, to create an entry point for an existing ML.NET component, are:
+1. Add the `SignatureEntryPointModule` signature to the `LoadableClass` assembly attribute.  
+2. Create a public static method, that:
+    a. Takes as input, among others, an object representing the arguments of the component you want to expose. 
+    b. Initializes and run the components, returning one of the nested classes of  `Microsoft.ML.Runtime.EntryPoints.CommonOutputs`
+    c. Is annotated with the `TlcModule.EntryPoint` attribute 
+
+Based on the type of entry point being created, there are further conventions on the name of the method, for example, the Trainers entry points are typically called: 'TrainMultiClass', 'TrainBinary' etc, based on the task. 
+Look at [OnlineGradientDescent](../../src/Microsoft.ML.StandardLearners/Standard/Online/OnlineGradientDescent.cs) for an example of a component and its entry point.