Upgrade to ML.NET version 1.0.0

build.cmd

@@ -46,7 +46,7 @@ if /i [%1] == [--skipDotNetBridge]     (
 echo "Usage: build.cmd [--configuration <Configuration>] [--runTests] [--buildDotNetBridgeOnly] [--skipDotNetBridge]"
 echo ""
 echo "Options:"
-echo "  --configuration <Configuration>   Build Configuration (DbgWinPy3.6,DbgWinPy3.5,DbgWinPy2.7,RlsWinPy3.6,RlsWinPy3.5,RlsWinPy2.7)"
+echo "  --configuration <Configuration>   Build Configuration (DbgWinPy3.7,DbgWinPy3.6,DbgWinPy3.5,DbgWinPy2.7,RlsWinPy3.7,RlsWinPy3.6,RlsWinPy3.5,RlsWinPy2.7)"
 echo "  --runTests                        Run tests after build"
 echo "  --buildDotNetBridgeOnly           Build only DotNetBridge"
 echo "  --skipDotNetBridge                Build everything except DotNetBridge"

build/ci/phase-template.yml

@@ -24,7 +24,7 @@ phases:
     - script: $(_buildScript) --configuration $(_configuration) --runTests
   # Mac phases
   - ${{ if eq(parameters.name, 'Mac') }}:
-    - script: brew update && brew install libomp mono-libgdiplus gettext && brew link gettext --force
+    - script: brew update && brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/f5b1ac99a7fba27c19cee0bc4f036775c889b359/Formula/libomp.rb mono-libgdiplus gettext && brew link gettext --force
     - ${{ if eq(parameters.testDistro, 'noTests') }}:
       - script: chmod 777 $(_buildScript) && $(_buildScript) --configuration $(_configuration)
     - ${{ if eq(parameters.testDistro, '') }}:

build/libs_linux.txt

@@ -1,4 +1,3 @@
-Google.Protobuf.dll
 Newtonsoft.Json.dll
 libCpuMathNative.so
 libFactorizationMachineNative.so

build/libs_mac.txt

@@ -1,4 +1,3 @@
-Google.Protobuf.dll
 Newtonsoft.Json.dll
 libCpuMathNative.dylib
 libFactorizationMachineNative.dylib

build/libs_win.txt

@@ -5,6 +5,7 @@ FactorizationMachineNative.dll
 FastTreeNative.dll
 LdaNative.dll
 lib_lightgbm.dll
+libiomp5md.dll
 MklImports.dll
 SymSgdNative.dll
 tensorflow.dll

src/DotNetBridge/Bridge.cs

@@ -10,14 +10,12 @@
 using Microsoft.ML;
 using Microsoft.ML.Data;
 using Microsoft.ML.EntryPoints;
-using Microsoft.ML.ImageAnalytics;
-using Microsoft.ML.LightGBM;
-using Microsoft.ML.Model.Onnx;
+using Microsoft.ML.Model.OnnxConverter;
+using Microsoft.ML.Runtime;
 using Microsoft.ML.Trainers;
+using Microsoft.ML.Trainers.Ensemble;
 using Microsoft.ML.Trainers.FastTree;
-using Microsoft.ML.Trainers.KMeans;
-using Microsoft.ML.Trainers.PCA;
-using Microsoft.ML.Trainers.SymSgd;
+using Microsoft.ML.Trainers.LightGbm;
 using Microsoft.ML.Transforms;
 
 namespace Microsoft.MachineLearning.DotNetBridge
@@ -307,107 +305,110 @@ private static unsafe IntPtr GetFn(FnId id)
         /// </summary>
         private static unsafe int GenericExec(EnvironmentBlock* penv, sbyte* psz, int cdata, DataSourceBlock** ppdata)
         {
-            using (var env = new RmlEnvironment(MarshalDelegate<CheckCancelled>(penv->checkCancel), penv->seed,
-                    verbose: penv != null && penv->verbosity > 3, conc: penv != null ? penv->maxThreadsAllowed : 0))
+            var env = new RmlEnvironment(MarshalDelegate<CheckCancelled>(penv->checkCancel), penv->seed, verbose: penv != null && penv->verbosity > 3);
+            var host = env.Register("ML.NET_Execution");
+
+            env.ComponentCatalog.RegisterAssembly(typeof(TextLoader).Assembly); // ML.Data
+            env.ComponentCatalog.RegisterAssembly(typeof(LinearModelParameters).Assembly); // ML.StandardLearners
+            env.ComponentCatalog.RegisterAssembly(typeof(CategoricalCatalog).Assembly); // ML.Transforms
+            env.ComponentCatalog.RegisterAssembly(typeof(FastTreeRegressionTrainer).Assembly); // ML.FastTree
+
+            //env.ComponentCatalog.RegisterAssembly(typeof(EnsembleModelParameters).Assembly); // ML.Ensemble
+            env.ComponentCatalog.RegisterAssembly(typeof(KMeansModelParameters).Assembly); // ML.KMeansClustering
+            env.ComponentCatalog.RegisterAssembly(typeof(PcaModelParameters).Assembly); // ML.PCA
+            env.ComponentCatalog.RegisterAssembly(typeof(CVSplit).Assembly); // ML.EntryPoints
+
+            env.ComponentCatalog.RegisterAssembly(typeof(OlsModelParameters).Assembly);
+            env.ComponentCatalog.RegisterAssembly(typeof(LightGbmBinaryModelParameters).Assembly);
+            env.ComponentCatalog.RegisterAssembly(typeof(TensorFlowTransformer).Assembly);
+            //env.ComponentCatalog.RegisterAssembly(typeof(SymSgdClassificationTrainer).Assembly);
+            //env.ComponentCatalog.RegisterAssembly(typeof(AutoInference).Assembly); // ML.PipelineInference
+            env.ComponentCatalog.RegisterAssembly(typeof(DataViewReference).Assembly);
+            env.ComponentCatalog.RegisterAssembly(typeof(ImageLoadingTransformer).Assembly);
+            //env.ComponentCatalog.RegisterAssembly(typeof(SaveOnnxCommand).Assembly);
+            //env.ComponentCatalog.RegisterAssembly(typeof(TimeSeriesProcessingEntryPoints).Assembly);
+            //env.ComponentCatalog.RegisterAssembly(typeof(ParquetLoader).Assembly);
+
+            using (var ch = host.Start("Executing"))
             {
-                var host = env.Register("ML.NET_Execution");
-                env.ComponentCatalog.RegisterAssembly(typeof(TextLoader).Assembly); // ML.Data
-                env.ComponentCatalog.RegisterAssembly(typeof(StochasticGradientDescentClassificationTrainer).Assembly); // ML.StandardLearners
-                env.ComponentCatalog.RegisterAssembly(typeof(CategoricalCatalog).Assembly); // ML.Transforms
-                env.ComponentCatalog.RegisterAssembly(typeof(FastTreeRegressionTrainer).Assembly); // ML.FastTree
-                env.ComponentCatalog.RegisterAssembly(typeof(KMeansPlusPlusTrainer).Assembly); // ML.KMeansClustering
-                env.ComponentCatalog.RegisterAssembly(typeof(RandomizedPcaTrainer).Assembly); // ML.PCA
-                //env.ComponentCatalog.RegisterAssembly(typeof(Experiment).Assembly); // ML.Legacy
-                env.ComponentCatalog.RegisterAssembly(typeof(LightGbmRegressorTrainer).Assembly);
-                env.ComponentCatalog.RegisterAssembly(typeof(TensorFlowTransformer).Assembly);
-                env.ComponentCatalog.RegisterAssembly(typeof(ImageLoaderTransformer).Assembly);
-                env.ComponentCatalog.RegisterAssembly(typeof(SymSgdClassificationTrainer).Assembly);
-                //env.ComponentCatalog.RegisterAssembly(typeof(AutoInference).Assembly); // ML.PipelineInference
-                env.ComponentCatalog.RegisterAssembly(typeof(OnnxExportExtensions).Assembly); // ML.Onnx
-                env.ComponentCatalog.RegisterAssembly(typeof(DataViewReference).Assembly);
-                //env.ComponentCatalog.RegisterAssembly(typeof(EnsemblePredictor).Assembly); // // ML.Ensemble BUG https://github.com/dotnet/machinelearning/issues/1078 Ensemble isn't in a NuGet package
-
-                using (var ch = host.Start("Executing"))
+                var sw = new System.Diagnostics.Stopwatch();
+                sw.Start();
+                try
                 {
-                    var sw = new System.Diagnostics.Stopwatch();
-                    sw.Start();
-                    try
-                    {
-                        // code, pszIn, and pszOut can be null.
-                        ch.Trace("Checking parameters");
+                    // code, pszIn, and pszOut can be null.
+                    ch.Trace("Checking parameters");
 
-                        host.CheckParam(penv != null, nameof(penv));
-                        host.CheckParam(penv->messageSink != null, "penv->message");
+                    host.CheckParam(penv != null, nameof(penv));
+                    host.CheckParam(penv->messageSink != null, "penv->message");
 
-                        host.CheckParam(psz != null, nameof(psz));
+                    host.CheckParam(psz != null, nameof(psz));
 
-                        ch.Trace("Converting graph operands");
-                        var graph = BytesToString(psz);
+                    ch.Trace("Converting graph operands");
+                    var graph = BytesToString(psz);
 
-                        ch.Trace("Wiring message sink");
-                        var message = MarshalDelegate<MessageSink>(penv->messageSink);
-                        var messageValidator = new MessageValidator(host);
-                        var lk = new object();
-                        Action<IMessageSource, ChannelMessage> listener =
-                            (sender, msg) =>
+                    ch.Trace("Wiring message sink");
+                    var message = MarshalDelegate<MessageSink>(penv->messageSink);
+                    var messageValidator = new MessageValidator(host);
+                    var lk = new object();
+                    Action<IMessageSource, ChannelMessage> listener =
+                        (sender, msg) =>
+                        {
+                            byte[] bs = StringToNullTerminatedBytes(sender.FullName);
+                            string m = messageValidator.Validate(msg);
+                            if (!string.IsNullOrEmpty(m))
                             {
-                                byte[] bs = StringToNullTerminatedBytes(sender.FullName);
-                                string m = messageValidator.Validate(msg);
-                                if (!string.IsNullOrEmpty(m))
+                                byte[] bm = StringToNullTerminatedBytes(m);
+                                lock (lk)
                                 {
-                                    byte[] bm = StringToNullTerminatedBytes(m);
-                                    lock (lk)
-                                    {
-                                        fixed (byte* ps = bs)
-                                        fixed (byte* pm = bm)
-                                            message(penv, msg.Kind, (sbyte*)ps, (sbyte*)pm);
-                                    }
+                                    fixed (byte* ps = bs)
+                                    fixed (byte* pm = bm)
+                                        message(penv, msg.Kind, (sbyte*)ps, (sbyte*)pm);
                                 }
-                            };
-                        env.AddListener(listener);
+                            }
+                        };
+                    env.AddListener(listener);
 
-                        host.CheckParam(cdata >= 0, nameof(cdata), "must be non-negative");
-                        host.CheckParam(ppdata != null || cdata == 0, nameof(ppdata));
-                        for (int i = 0; i < cdata; i++)
+                    host.CheckParam(cdata >= 0, nameof(cdata), "must be non-negative");
+                    host.CheckParam(ppdata != null || cdata == 0, nameof(ppdata));
+                    for (int i = 0; i < cdata; i++)
+                    {
+                        var pdata = ppdata[i];
+                        host.CheckParam(pdata != null, "pdata");
+                        host.CheckParam(0 <= pdata->ccol && pdata->ccol <= int.MaxValue, "ccol");
+                        host.CheckParam(0 <= pdata->crow && pdata->crow <= long.MaxValue, "crow");
+                        if (pdata->ccol > 0)
                         {
-                            var pdata = ppdata[i];
-                            host.CheckParam(pdata != null, "pdata");
-                            host.CheckParam(0 <= pdata->ccol && pdata->ccol <= int.MaxValue, "ccol");
-                            host.CheckParam(0 <= pdata->crow && pdata->crow <= long.MaxValue, "crow");
-                            if (pdata->ccol > 0)
-                            {
-                                host.CheckParam(pdata->names != null, "names");
-                                host.CheckParam(pdata->kinds != null, "kinds");
-                                host.CheckParam(pdata->keyCards != null, "keyCards");
-                                host.CheckParam(pdata->vecCards != null, "vecCards");
-                                host.CheckParam(pdata->getters != null, "getters");
-                            }
+                            host.CheckParam(pdata->names != null, "names");
+                            host.CheckParam(pdata->kinds != null, "kinds");
+                            host.CheckParam(pdata->keyCards != null, "keyCards");
+                            host.CheckParam(pdata->vecCards != null, "vecCards");
+                            host.CheckParam(pdata->getters != null, "getters");
                         }
+                    }
 
-                        ch.Trace("Validating number of data sources");
+                    ch.Trace("Validating number of data sources");
 
-                        // Wrap the data sets.
-                        ch.Trace("Wrapping native data sources");
-                        ch.Trace("Executing");
-                        ExecCore(penv, host, ch, graph, cdata, ppdata);
-                    }
-                    catch (Exception e)
-                    {
-                        // Dump the exception chain.
-                        var ex = e;
-                        while (ex.InnerException != null)
-                            ex = ex.InnerException;
-                        ch.Error("*** {1}: '{0}'", ex.Message, ex.GetType());
-                        return -1;
-                    }
-                    finally
-                    {
-                        sw.Stop();
-                        if (penv != null && penv->verbosity > 0)
-                            ch.Info("Elapsed time: {0}", sw.Elapsed);
-                        else
-                            ch.Trace("Elapsed time: {0}", sw.Elapsed);
-                    }
+                    // Wrap the data sets.
+                    ch.Trace("Wrapping native data sources");
+                    ch.Trace("Executing");
+                    ExecCore(penv, host, ch, graph, cdata, ppdata);
+                }
+                catch (Exception e)
+                {
+                    // Dump the exception chain.
+                    var ex = e;
+                    while (ex.InnerException != null)
+                        ex = ex.InnerException;
+                    ch.Error("*** {1}: '{0}'", ex.Message, ex.GetType());
+                    return -1;
+                }
+                finally
+                {
+                    sw.Stop();
+                    if (penv != null && penv->verbosity > 0)
+                        ch.Info("Elapsed time: {0}", sw.Elapsed);
+                    else
+                        ch.Trace("Elapsed time: {0}", sw.Elapsed);
                 }
             }
             return 0;

src/DotNetBridge/DotNetBridge.csproj

@@ -31,13 +31,15 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.ML" Version="0.10.0-preview-27310-10" />
-    <PackageReference Include="Microsoft.ML.CpuMath" Version="0.10.0-preview-27310-10" />    
-    <PackageReference Include="Microsoft.ML.EntryPoints" Version="0.10.0-preview-27310-10" />
-    <PackageReference Include="Microsoft.ML.HalLearners" Version="0.10.0-preview-27310-10" />
-    <PackageReference Include="Microsoft.ML.ImageAnalytics" Version="0.10.0-preview-27310-10" />
-    <PackageReference Include="Microsoft.ML.LightGBM" Version="0.10.0-preview-27310-10" />
-    <PackageReference Include="Microsoft.ML.Onnx" Version="0.10.0-preview-27310-10" />
-    <PackageReference Include="Microsoft.ML.TensorFlow" Version="0.10.0-preview-27310-10" />
+    <PackageReference Include="Microsoft.ML" Version="1.0.0" />
+    <PackageReference Include="Microsoft.ML.CpuMath" Version="1.0.0" />    
+    <PackageReference Include="Microsoft.ML.EntryPoints" Version="0.12.0" />
+    <PackageReference Include="Microsoft.ML.Mkl.Components" Version="1.0.0" />
+    <PackageReference Include="Microsoft.ML.Mkl.Redist" Version="1.0.0" />
+    <PackageReference Include="Microsoft.ML.ImageAnalytics" Version="1.0.0" />
+    <PackageReference Include="Microsoft.ML.LightGBM" Version="1.0.0" />
+    <PackageReference Include="Microsoft.ML.OnnxTransformer" Version="0.12.0" />
+    <PackageReference Include="Microsoft.ML.TensorFlow" Version="0.12.0" />
+    <PackageReference Include="Microsoft.ML.Ensemble" Version="0.12.0" />
   </ItemGroup>
 </Project>

src/DotNetBridge/MessageValidator.cs

@@ -5,7 +5,7 @@
 
 using System;
 using System.Globalization;
-using Microsoft.ML;
+using Microsoft.ML.Runtime;
 
 namespace Microsoft.MachineLearning.DotNetBridge
 {