microsoft · pieths · Oct 15, 2019 · Oct 11, 2019 · Oct 11, 2019 · Oct 11, 2019
diff --git a/build.cmd b/build.cmd
@@ -26,6 +26,9 @@ set RunExtendedTests=False
 set BuildDotNetBridgeOnly=False
 set SkipDotNetBridge=False
 set AzureBuild=False
+set BuildManifestGenerator=False
+set UpdateManifest=False
+set VerifyManifest=False
 
 :Arg_Loop
 if [%1] == [] goto :Build
@@ -53,6 +56,10 @@ if /i [%1] == [--skipDotNetBridge]     (
     set SkipDotNetBridge=True
     shift && goto :Arg_Loop
 )
+if /i [%1] == [--updateManifest] (
+    set UpdateManifest=True
+    shift && goto :Arg_Loop
+)
 if /i [%1] == [--azureBuild]     (
     set AzureBuild=True
     shift && goto :Arg_Loop
@@ -68,6 +75,7 @@ echo "  --installPythonPackages           Install python packages after build"
 echo "  --includeExtendedTests            Include the extended tests if the tests are run"
 echo "  --buildDotNetBridgeOnly           Build only DotNetBridge"
 echo "  --skipDotNetBridge                Build everything except DotNetBridge"
+echo "  --updateManifest                  Update manifest.json"
 echo "  --azureBuild                      Building in azure devops (adds dotnet CLI to the path)"
 goto :Exit_Success
 
@@ -189,6 +197,37 @@ if "%BuildDotNetBridgeOnly%" == "True" (
 call "%_dotnet%" build -c %Configuration% --force "%__currentScriptDir%src\Platforms\build.csproj"
 call "%_dotnet%" publish "%__currentScriptDir%src\Platforms\build.csproj" --force --self-contained -r win-x64 -c %Configuration%
 
+
+if "%Configuration:~-5%" == "Py3.7" set VerifyManifest=True
+if "%VerifyManifest%" == "True" set BuildManifestGenerator=True
+if "%UpdateManifest%" == "True" set BuildManifestGenerator=True
+
+if "%BuildManifestGenerator%" == "True" (
+    echo ""
+    echo "#################################"
+    echo "Building Manifest Generator... "
+    echo "#################################"
+    call "%_dotnet%" build -c %Configuration% -o "%BuildOutputDir%%Configuration%"  --force "%__currentScriptDir%src\ManifestGenerator\ManifestGenerator.csproj"
+)
+
+if "%UpdateManifest%" == "True" (
+    echo Updating manifest.json ...
+    call "%_dotnet%" "%BuildOutputDir%%Configuration%\ManifestGenerator.dll" create %__currentScriptDir%\src\python\tools\manifest.json
+    echo manifest.json updated.
+    echo Run entrypoint_compiler.py --generate_api --generate_entrypoints to generate entry points and api files.
+    goto :Exit_Success
+)
+
+if "%VerifyManifest%" == "True" (
+    echo Verifying manifest.json ...
+    call "%_dotnet%" "%BuildOutputDir%%Configuration%\ManifestGenerator.dll" verify %__currentScriptDir%\src\python\tools\manifest.json
+    if errorlevel 1 (
+        echo manifest.json is invalid.
+        echo Run build --updateManifest to update manifest.json.
+        goto :Exit_Error
+    )
+)
+
 echo ""
 echo "#################################"
 echo "Downloading Dependencies "
@@ -392,10 +431,18 @@ if "%RunExtendedTests%" == "True" (
 )
 
 :Exit_Success
+:: Shutdown all dotnet persistent servers so that the
+:: dotnet executable is not left open in the background.
+:: As of dotnet 2.1.3 three servers are left running in
+:: the background. This will shutdown them all down.
+:: See here for more info: https://github.com/dotnet/cli/issues/9458
+call "%_dotnet%" build-server shutdown
 endlocal
 exit /b %ERRORLEVEL%
 
 :Exit_Error
+:: See comment above
+call "%_dotnet%" build-server shutdown
 endlocal
 echo Failed with error %ERRORLEVEL%
 exit /b %ERRORLEVEL%
diff --git a/nimbusml.sln b/nimbusml.sln
@@ -20,6 +20,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		version.txt = version.txt
 	EndProjectSection
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ManifestGenerator", "src\ManifestGenerator\ManifestGenerator.csproj", "{D3AED287-722F-4243-966E-77AD0652B38E}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		DbgLinPy2.7|x64 = DbgLinPy2.7|x64
@@ -65,36 +67,36 @@ Global
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.DbgLinPy3.5|x64.ActiveCfg = DbgLinPy3.5|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.DbgLinPy3.5|x64.Build.0 = DbgLinPy3.5|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.DbgLinPy3.6|x64.ActiveCfg = DbgLinPy3.6|x64
-		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.DbgLinPy3.7|x64.ActiveCfg = DbgLinPy3.7|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.DbgLinPy3.6|x64.Build.0 = DbgLinPy3.6|x64
+		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.DbgLinPy3.7|x64.ActiveCfg = DbgLinPy3.7|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.DbgLinPy3.7|x64.Build.0 = DbgLinPy3.7|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.DbgWinPy2.7|x64.ActiveCfg = DbgWinPy2.7|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.DbgWinPy2.7|x64.Build.0 = DbgWinPy2.7|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.DbgWinPy3.5|x64.ActiveCfg = DbgWinPy3.5|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.DbgWinPy3.5|x64.Build.0 = DbgWinPy3.5|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.DbgWinPy3.6|x64.ActiveCfg = DbgWinPy3.6|x64
-		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.DbgWinPy3.7|x64.ActiveCfg = DbgWinPy3.7|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.DbgWinPy3.6|x64.Build.0 = DbgWinPy3.6|x64
+		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.DbgWinPy3.7|x64.ActiveCfg = DbgWinPy3.7|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.DbgWinPy3.7|x64.Build.0 = DbgWinPy3.7|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.RlsLinPy2.7|x64.ActiveCfg = RlsLinPy2.7|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.RlsLinPy2.7|x64.Build.0 = RlsLinPy2.7|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.RlsLinPy3.5|x64.ActiveCfg = RlsLinPy3.5|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.RlsLinPy3.5|x64.Build.0 = RlsLinPy3.5|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.RlsLinPy3.6|x64.ActiveCfg = RlsLinPy3.6|x64
-		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.RlsLinPy3.7|x64.ActiveCfg = RlsLinPy3.7|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.RlsLinPy3.6|x64.Build.0 = RlsLinPy3.6|x64
+		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.RlsLinPy3.7|x64.ActiveCfg = RlsLinPy3.7|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.RlsLinPy3.7|x64.Build.0 = RlsLinPy3.7|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.RlsMacPy3.6|x64.ActiveCfg = RlsMacPy3.6|x64
-		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.RlsMacPy3.7|x64.ActiveCfg = RlsMacPy3.7|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.RlsMacPy3.6|x64.Build.0 = RlsMacPy3.6|x64
+		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.RlsMacPy3.7|x64.ActiveCfg = RlsMacPy3.7|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.RlsMacPy3.7|x64.Build.0 = RlsMacPy3.7|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.RlsWinPy2.7|x64.ActiveCfg = RlsWinPy2.7|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.RlsWinPy2.7|x64.Build.0 = RlsWinPy2.7|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.RlsWinPy3.5|x64.ActiveCfg = RlsWinPy3.5|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.RlsWinPy3.5|x64.Build.0 = RlsWinPy3.5|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.RlsWinPy3.6|x64.ActiveCfg = RlsWinPy3.6|x64
-		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.RlsWinPy3.7|x64.ActiveCfg = RlsWinPy3.7|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.RlsWinPy3.6|x64.Build.0 = RlsWinPy3.6|x64
+		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.RlsWinPy3.7|x64.ActiveCfg = RlsWinPy3.7|x64
 		{EC58F2CF-A1D5-4E28-97F9-69B1E46F6F63}.RlsWinPy3.7|x64.Build.0 = RlsWinPy3.7|x64
 		{3DA0AF32-A05B-4ECF-8010-83B14612FBB3}.DbgLinPy2.7|x64.ActiveCfg = DbgWinPy2.7|x64
 		{3DA0AF32-A05B-4ECF-8010-83B14612FBB3}.DbgLinPy3.5|x64.ActiveCfg = DbgWinPy3.5|x64
@@ -105,8 +107,8 @@ Global
 		{3DA0AF32-A05B-4ECF-8010-83B14612FBB3}.DbgWinPy3.5|x64.ActiveCfg = DbgWinPy3.5|x64
 		{3DA0AF32-A05B-4ECF-8010-83B14612FBB3}.DbgWinPy3.5|x64.Build.0 = DbgWinPy3.5|x64
 		{3DA0AF32-A05B-4ECF-8010-83B14612FBB3}.DbgWinPy3.6|x64.ActiveCfg = DbgWinPy3.6|x64
-		{3DA0AF32-A05B-4ECF-8010-83B14612FBB3}.DbgWinPy3.7|x64.ActiveCfg = DbgWinPy3.7|x64
 		{3DA0AF32-A05B-4ECF-8010-83B14612FBB3}.DbgWinPy3.6|x64.Build.0 = DbgWinPy3.6|x64
+		{3DA0AF32-A05B-4ECF-8010-83B14612FBB3}.DbgWinPy3.7|x64.ActiveCfg = DbgWinPy3.7|x64
 		{3DA0AF32-A05B-4ECF-8010-83B14612FBB3}.DbgWinPy3.7|x64.Build.0 = DbgWinPy3.7|x64
 		{3DA0AF32-A05B-4ECF-8010-83B14612FBB3}.RlsLinPy2.7|x64.ActiveCfg = RlsWinPy2.7|x64
 		{3DA0AF32-A05B-4ECF-8010-83B14612FBB3}.RlsLinPy3.5|x64.ActiveCfg = RlsWinPy3.5|x64
@@ -119,9 +121,45 @@ Global
 		{3DA0AF32-A05B-4ECF-8010-83B14612FBB3}.RlsWinPy3.5|x64.ActiveCfg = RlsWinPy3.5|x64
 		{3DA0AF32-A05B-4ECF-8010-83B14612FBB3}.RlsWinPy3.5|x64.Build.0 = RlsWinPy3.5|x64
 		{3DA0AF32-A05B-4ECF-8010-83B14612FBB3}.RlsWinPy3.6|x64.ActiveCfg = RlsWinPy3.6|x64
-		{3DA0AF32-A05B-4ECF-8010-83B14612FBB3}.RlsWinPy3.7|x64.ActiveCfg = RlsWinPy3.7|x64
 		{3DA0AF32-A05B-4ECF-8010-83B14612FBB3}.RlsWinPy3.6|x64.Build.0 = RlsWinPy3.6|x64
+		{3DA0AF32-A05B-4ECF-8010-83B14612FBB3}.RlsWinPy3.7|x64.ActiveCfg = RlsWinPy3.7|x64
 		{3DA0AF32-A05B-4ECF-8010-83B14612FBB3}.RlsWinPy3.7|x64.Build.0 = RlsWinPy3.7|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.DbgLinPy2.7|x64.ActiveCfg = DbgLinPy2.7|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.DbgLinPy2.7|x64.Build.0 = DbgLinPy2.7|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.DbgLinPy3.5|x64.ActiveCfg = DbgLinPy3.5|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.DbgLinPy3.5|x64.Build.0 = DbgLinPy3.5|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.DbgLinPy3.6|x64.ActiveCfg = DbgLinPy3.6|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.DbgLinPy3.6|x64.Build.0 = DbgLinPy3.6|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.DbgLinPy3.7|x64.ActiveCfg = DbgLinPy3.7|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.DbgLinPy3.7|x64.Build.0 = DbgLinPy3.7|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.DbgWinPy2.7|x64.ActiveCfg = DbgWinPy2.7|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.DbgWinPy2.7|x64.Build.0 = DbgWinPy2.7|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.DbgWinPy3.5|x64.ActiveCfg = DbgWinPy3.5|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.DbgWinPy3.5|x64.Build.0 = DbgWinPy3.5|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.DbgWinPy3.6|x64.ActiveCfg = DbgWinPy3.6|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.DbgWinPy3.6|x64.Build.0 = DbgWinPy3.6|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.DbgWinPy3.7|x64.ActiveCfg = DbgWinPy3.7|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.DbgWinPy3.7|x64.Build.0 = DbgWinPy3.7|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.RlsLinPy2.7|x64.ActiveCfg = RlsLinPy2.7|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.RlsLinPy2.7|x64.Build.0 = RlsLinPy2.7|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.RlsLinPy3.5|x64.ActiveCfg = RlsLinPy3.5|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.RlsLinPy3.5|x64.Build.0 = RlsLinPy3.5|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.RlsLinPy3.6|x64.ActiveCfg = RlsLinPy3.6|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.RlsLinPy3.6|x64.Build.0 = RlsLinPy3.6|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.RlsLinPy3.7|x64.ActiveCfg = RlsLinPy3.7|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.RlsLinPy3.7|x64.Build.0 = RlsLinPy3.7|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.RlsMacPy3.6|x64.ActiveCfg = RlsMacPy3.6|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.RlsMacPy3.6|x64.Build.0 = RlsMacPy3.6|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.RlsMacPy3.7|x64.ActiveCfg = RlsMacPy3.7|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.RlsMacPy3.7|x64.Build.0 = RlsMacPy3.7|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.RlsWinPy2.7|x64.ActiveCfg = RlsWinPy2.7|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.RlsWinPy2.7|x64.Build.0 = RlsWinPy2.7|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.RlsWinPy3.5|x64.ActiveCfg = RlsWinPy3.5|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.RlsWinPy3.5|x64.Build.0 = RlsWinPy3.5|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.RlsWinPy3.6|x64.ActiveCfg = RlsWinPy3.6|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.RlsWinPy3.6|x64.Build.0 = RlsWinPy3.6|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.RlsWinPy3.7|x64.ActiveCfg = RlsWinPy3.7|x64
+		{D3AED287-722F-4243-966E-77AD0652B38E}.RlsWinPy3.7|x64.Build.0 = RlsWinPy3.7|x64
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

diff --git a/src/DotNetBridge/ManifestUtils.cs b/src/DotNetBridge/ManifestUtils.cs
@@ -0,0 +1,110 @@
+//------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+//------------------------------------------------------------------------------
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Text.RegularExpressions;
+using Microsoft.ML.Data;
+using Microsoft.ML.EntryPoints;
+using Microsoft.ML.Model.OnnxConverter;
+using Microsoft.ML.Runtime;
+using Microsoft.ML.Trainers;
+using Microsoft.ML.Trainers.Ensemble;
+using Microsoft.ML.Trainers.FastTree;
+using Microsoft.ML.Trainers.LightGbm;
+using Microsoft.ML.Transforms;
+using Microsoft.ML.Transforms.TimeSeries;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+
+namespace Microsoft.ML.DotNetBridge
+{
+    public static class ManifestUtils
+    {
+        private static readonly Type[] _types = new Type[]
+        {
+            typeof(TextLoader),
+            typeof(LinearModelParameters),
+            typeof(OneHotEncodingTransformer),
+            typeof(FastTreeBinaryModelParameters),
+            typeof(EnsembleModelParameters),
+            typeof(KMeansModelParameters),
+            typeof(PcaModelParameters),
+            typeof(CVSplit),
+            typeof(LightGbmBinaryModelParameters),
+            typeof(TensorFlowTransformer),
+            typeof(ImageLoadingTransformer),
+            typeof(SymbolicSgdLogisticRegressionBinaryTrainer),
+            typeof(OnnxContext),
+            typeof(SsaForecastingTransformer),
+            typeof(VariableColumnTransform)
+        };
+
+        private static (IEnumerable<string> epListContents, JObject manifest) BuildManifests()
+        {
+            ConsoleEnvironment env = new ConsoleEnvironment();
+
+            foreach (Type type in _types)
+            {
+                env.ComponentCatalog.RegisterAssembly(type.Assembly);
+            }
+
+            var catalog = env.ComponentCatalog;
+
+            var regex = new Regex(@"\r\n?|\n", RegexOptions.Compiled);
+            var epListContents = catalog.AllEntryPoints()
+                .Select(x => string.Join("\t",
+                x.Name,
+                regex.Replace(x.Description, ""),
+                x.Method.DeclaringType,
+                x.Method.Name,
+                x.InputType,
+                x.OutputType)
+                .Replace(Environment.NewLine, "", StringComparison.Ordinal))
+                .OrderBy(x => x);
+
+            var manifest = JsonManifestUtils.BuildAllManifests(env, catalog);
+
+            //clean up the description from the new line characters
+            if (manifest[FieldNames.TopEntryPoints] != null && manifest[FieldNames.TopEntryPoints] is JArray)
+            {
+                foreach (JToken entry in manifest[FieldNames.TopEntryPoints].Children())
+                    if (entry[FieldNames.Desc] != null)
+                        entry[FieldNames.Desc] = regex.Replace(entry[FieldNames.Desc].ToString(), "");
+            }
+
+            return (epListContents, manifest);
+        }
+
+        public static void ShowAssemblyInfo()
+        {
+            foreach (Type type in _types)
+            {
+                Assembly assembly = type.Assembly;
+                Console.WriteLine(assembly.Location);
+            }
+        }
+
+        public static void GenerateManifest(string filePath)
+        {
+            var (epListContents, jObj) = BuildManifests();
+
+            if (!string.IsNullOrWhiteSpace(filePath))
+                File.Delete(filePath);
+
+            using (var file = File.OpenWrite(filePath))
+            using (var writer = new StreamWriter(file))
+            using (var jw = new JsonTextWriter(writer))
+            {
+                jw.Formatting = Formatting.Indented;
+                jObj.WriteTo(jw);
+            }
+        }
+    }
+}