Move Static API extensions to separate assembly

docs/samples/Microsoft.ML.Samples/Static/AveragedPerceptronBinaryClassification.cs

@@ -33,7 +33,7 @@ public static void AveragedPerceptronBinaryClassification()
             var mlContext = new MLContext();
 
             // Creating Data Reader with the initial schema based on the format of the data
-            var reader = TextLoader.CreateReader(
+            var reader = TextLoaderStatic.CreateReader(
                 mlContext,
                 c => (
                     Age: c.LoadFloat(0),

docs/samples/Microsoft.ML.Samples/Static/FastTreeBinaryClassification.cs

@@ -33,7 +33,7 @@ public static void FastTreeBinaryClassification()
             var mlContext = new MLContext();
 
             // Creating Data Reader with the initial schema based on the format of the data
-            var reader = TextLoader.CreateReader(
+            var reader = TextLoaderStatic.CreateReader(
                 mlContext,
                 c => (
                     Age: c.LoadFloat(0),

docs/samples/Microsoft.ML.Samples/Static/FastTreeRegression.cs

@@ -20,7 +20,7 @@ public static void FastTreeRegression()
             var mlContext = new MLContext();
 
             // Creating a data reader, based on the format of the data
-            var reader = TextLoader.CreateReader(mlContext, c => (
+            var reader = TextLoaderStatic.CreateReader(mlContext, c => (
                         label: c.LoadFloat(0),
                         features: c.LoadFloat(1, 6)
                     ),

docs/samples/Microsoft.ML.Samples/Static/FeatureSelectionTransform.cs

@@ -31,7 +31,7 @@ public static void FeatureSelectionTransform()
 
             // First, we define the reader: specify the data columns and where to find them in the text file. Notice that we combine entries from
             // all the feature columns into entries of a vector of a single column named "Features".
-            var reader = TextLoader.CreateReader(ml, c => (
+            var reader = TextLoaderStatic.CreateReader(ml, c => (
                         Label: c.LoadBool(0),
                         Features: c.LoadFloat(1, 9)
                     ),

docs/samples/Microsoft.ML.Samples/Static/LightGBMBinaryClassification.cs

@@ -34,7 +34,7 @@ public static void LightGbmBinaryClassification()
             var mlContext = new MLContext();
 
             // Creating Data Reader with the initial schema based on the format of the data
-            var reader = TextLoader.CreateReader(
+            var reader = TextLoaderStatic.CreateReader(
                 mlContext,
                 c => (
                     Age: c.LoadFloat(0),

docs/samples/Microsoft.ML.Samples/Static/LightGBMRegression.cs

@@ -2,6 +2,7 @@
 using Microsoft.ML.Data;
 using Microsoft.ML.LightGBM;
 using Microsoft.ML.LightGBM.StaticPipe;
+using Microsoft.ML.StaticPipe;
 
 namespace Microsoft.ML.Samples.Static
 {
@@ -19,7 +20,7 @@ public static void LightGbmRegression()
             var mlContext = new MLContext();
 
             // Creating a data reader, based on the format of the data
-            var reader = TextLoader.CreateReader(mlContext, c => (
+            var reader = TextLoaderStatic.CreateReader(mlContext, c => (
                         label: c.LoadFloat(0),
                         features: c.LoadFloat(1, 6)
                     ),

docs/samples/Microsoft.ML.Samples/Static/SDCABinaryClassification.cs

@@ -33,7 +33,7 @@ public static void SdcaBinaryClassification()
             var mlContext = new MLContext();
 
             // Creating Data Reader with the initial schema based on the format of the data
-            var reader = TextLoader.CreateReader(
+            var reader = TextLoaderStatic.CreateReader(
                 mlContext,
                 c => (
                     Age: c.LoadFloat(0),

docs/samples/Microsoft.ML.Samples/Static/SDCARegression.cs

@@ -18,7 +18,7 @@ public static void SdcaRegression()
             var mlContext = new MLContext();
 
             // Creating a data reader, based on the format of the data
-            var reader = TextLoader.CreateReader(mlContext, c => (
+            var reader = TextLoaderStatic.CreateReader(mlContext, c => (
                         label: c.LoadFloat(0),
                         features: c.LoadFloat(1, 6)
                     ),

src/Microsoft.ML.Data/DataLoadSave/DataOperations.cs

@@ -14,6 +14,7 @@ namespace Microsoft.ML
     /// </summary>
     public sealed class DataOperations
     {
+        [BestFriend]
         internal IHostEnvironment Environment { get; }
 
         internal DataOperations(IHostEnvironment env)

src/Microsoft.ML.Data/DataLoadSave/EstimatorExtensions.cs

@@ -143,6 +143,7 @@ public static IEstimator<TTransformer> WithOnFitDelegate<TTransformer>(this IEst
             return new DelegateEstimator<TTransformer>(estimator, onFit);
         }
 
+        [BestFriend]
         internal static T[] AppendElement<T>(this T[] array, T element)
         {
             T[] result = new T[Utils.Size(array) + 1];

src/Microsoft.ML.Data/DataLoadSave/Text/TextLoader.cs

@@ -405,7 +405,7 @@ internal static class DefaultArguments
         /// Used as an input column range.
         /// A variable length segment (extending to the end of the input line) is represented by Lim == SrcLim.
         /// </summary>
-        private struct Segment
+        internal struct Segment
         {
             public int Min;
             public int Lim;
@@ -440,7 +440,7 @@ public Segment(int min)
         /// <summary>
         /// Information for an output column.
         /// </summary>
-        private sealed class ColInfo
+        internal sealed class ColInfo
         {
             public readonly string Name;
             // REVIEW: Fix this for keys.
@@ -1361,6 +1361,8 @@ public void Save(ModelSaveContext ctx)
 
         public IDataView Read(string path) => Read(new MultiFileSource(path));
 
+        public IDataView Read(params string[] path) => Read(new MultiFileSource(path));
+
         internal static TextLoader CreateTextReader<TInput>(IHostEnvironment host,
            bool hasHeader = DefaultArguments.HasHeader,
            char separator = DefaultArguments.Separator,

src/Microsoft.ML.FastTree/FastTreeArguments.cs

@@ -138,12 +138,13 @@ public enum Bundle : Byte
         Adjacent = 2
     }
 
+    [BestFriend]
     internal static class Defaults
     {
-        internal const int NumTrees = 100;
-        internal const int NumLeaves = 20;
-        internal const int MinDocumentsInLeaves = 10;
-        internal const double LearningRates = 0.2;
+        public const int NumTrees = 100;
+        public const int NumLeaves = 20;
+        public const int MinDocumentsInLeaves = 10;
+        public const double LearningRates = 0.2;
     }
 
     public abstract class TreeArgs : LearnerInputBaseWithGroupId

src/Microsoft.ML.FastTree/Properties/AssemblyInfo.cs

@@ -10,4 +10,6 @@
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.LightGBM" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Sweeper" + PublicKey.Value)]
 
+[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.StaticPipe" + PublicKey.Value)]
+
 [assembly: WantsToBeBestFriends]

src/Microsoft.ML.HalLearners.StaticPipe/Microsoft.ML.HalLearners.StaticPipe.csproj

@@ -7,6 +7,7 @@
   <ItemGroup>
     <ProjectReference Include="..\Microsoft.ML.Data\Microsoft.ML.Data.csproj" />
     <ProjectReference Include="..\Microsoft.ML.HalLearners\Microsoft.ML.HalLearners.csproj" />
+    <ProjectReference Include="..\Microsoft.ML.StaticPipe\Microsoft.ML.StaticPipe.csproj" />
   </ItemGroup>
 
 </Project>

src/Microsoft.ML.ImageAnalytics/ImageGrayscaleTransform.cs

@@ -16,8 +16,6 @@
 using Microsoft.ML.ImageAnalytics;
 using Microsoft.ML.Internal.Utilities;
 using Microsoft.ML.Model;
-using Microsoft.ML.StaticPipe;
-using Microsoft.ML.StaticPipe.Runtime;
 
 [assembly: LoadableClass(ImageGrayscaleTransform.Summary, typeof(IDataTransform), typeof(ImageGrayscaleTransform), typeof(ImageGrayscaleTransform.Arguments), typeof(SignatureDataTransform),
     ImageGrayscaleTransform.UserName, "ImageGrayscaleTransform", "ImageGrayscale")]
@@ -239,51 +237,5 @@ public override SchemaShape GetOutputSchema(SchemaShape inputSchema)
 
             return new SchemaShape(result.Values);
         }
-
-        private interface IColInput
-        {
-            PipelineColumn Input { get; }
-        }
-
-        internal sealed class OutPipelineColumn<T> : Custom<T>, IColInput
-        {
-            public PipelineColumn Input { get; }
-
-            public OutPipelineColumn(Custom<T> input)
-                : base(Reconciler.Inst, input)
-            {
-                Contracts.AssertValue(input);
-                Contracts.Assert(typeof(T) == typeof(Bitmap) || typeof(T) == typeof(UnknownSizeBitmap));
-                Input = input;
-            }
-        }
-
-        /// <summary>
-        /// Reconciler to an <see cref="ImageGrayscalingEstimator"/> for the <see cref="PipelineColumn"/>.
-        /// </summary>
-        /// <remarks>Because we want to use the same reconciler for </remarks>
-        /// <see cref="ImageStaticPipe.AsGrayscale(Custom{Bitmap})"/>
-        /// <see cref="ImageStaticPipe.AsGrayscale(Custom{UnknownSizeBitmap})"/>
-        private sealed class Reconciler : EstimatorReconciler
-        {
-            public static Reconciler Inst = new Reconciler();
-
-            private Reconciler() { }
-
-            public override IEstimator<ITransformer> Reconcile(IHostEnvironment env,
-                PipelineColumn[] toOutput,
-                IReadOnlyDictionary<PipelineColumn, string> inputNames,
-                IReadOnlyDictionary<PipelineColumn, string> outputNames,
-                IReadOnlyCollection<string> usedNames)
-            {
-                var cols = new (string input, string output)[toOutput.Length];
-                for (int i = 0; i < toOutput.Length; ++i)
-                {
-                    var outCol = (IColInput)toOutput[i];
-                    cols[i] = (inputNames[outCol.Input], outputNames[toOutput[i]]);
-                }
-                return new ImageGrayscalingEstimator(env, cols);
-            }
-        }
     }
 }

src/Microsoft.ML.ImageAnalytics/ImageLoaderTransform.cs

@@ -16,8 +16,6 @@
 using Microsoft.ML.ImageAnalytics;
 using Microsoft.ML.Internal.Utilities;
 using Microsoft.ML.Model;
-using Microsoft.ML.StaticPipe;
-using Microsoft.ML.StaticPipe.Runtime;
 
 [assembly: LoadableClass(ImageLoaderTransform.Summary, typeof(IDataTransform), typeof(ImageLoaderTransform), typeof(ImageLoaderTransform.Arguments), typeof(SignatureDataTransform),
     ImageLoaderTransform.UserName, "ImageLoaderTransform", "ImageLoader")]
@@ -242,62 +240,5 @@ public override SchemaShape GetOutputSchema(SchemaShape inputSchema)
 
             return new SchemaShape(result.Values);
         }
-
-        internal sealed class OutPipelineColumn : Custom<UnknownSizeBitmap>
-        {
-            private readonly Scalar<string> _input;
-
-            public OutPipelineColumn(Scalar<string> path, string relativeTo)
-                : base(new Reconciler(relativeTo), path)
-            {
-                Contracts.AssertValue(path);
-                _input = path;
-            }
-
-            /// <summary>
-            /// Reconciler to an <see cref="ImageLoadingEstimator"/> for the <see cref="PipelineColumn"/>.
-            /// </summary>
-            /// <remarks>
-            /// We must create a new reconciler per call, because the relative path of <see cref="ImageLoaderTransform.Arguments.ImageFolder"/>
-            /// is considered a transform-wide option, as it is not specified in <see cref="ImageLoaderTransform.Column"/>. However, we still
-            /// implement <see cref="IEquatable{T}"/> so the analyzer can still equate two of these things if they happen to share the same
-            /// path, so we can be a bit more efficient with respect to our estimator declarations.
-            /// </remarks>
-            /// <see cref="ImageStaticPipe.LoadAsImage(Scalar{string}, string)"/>
-            private sealed class Reconciler : EstimatorReconciler, IEquatable<Reconciler>
-            {
-                private readonly string _relTo;
-
-                public Reconciler(string relativeTo)
-                {
-                    Contracts.AssertValueOrNull(relativeTo);
-                    _relTo = relativeTo;
-                }
-
-                public bool Equals(Reconciler other)
-                    => other != null && other._relTo == _relTo;
-
-                public override bool Equals(object obj)
-                    => obj is Reconciler other && Equals(other);
-
-                public override int GetHashCode()
-                    => _relTo?.GetHashCode() ?? 0;
-
-                public override IEstimator<ITransformer> Reconcile(IHostEnvironment env,
-                    PipelineColumn[] toOutput,
-                    IReadOnlyDictionary<PipelineColumn, string> inputNames,
-                    IReadOnlyDictionary<PipelineColumn, string> outputNames,
-                    IReadOnlyCollection<string> usedNames)
-                {
-                    var cols = new (string input, string output)[toOutput.Length];
-                    for (int i = 0; i < toOutput.Length; ++i)
-                    {
-                        var outCol = (OutPipelineColumn)toOutput[i];
-                        cols[i] = (inputNames[outCol._input], outputNames[outCol]);
-                    }
-                    return new ImageLoadingEstimator(env, _relTo, cols);
-                }
-            }
-        }
     }
 }

src/Microsoft.ML.ImageAnalytics/ImagePixelExtractorTransform.cs

@@ -16,8 +16,6 @@
 using Microsoft.ML.ImageAnalytics;
 using Microsoft.ML.Internal.Utilities;
 using Microsoft.ML.Model;
-using Microsoft.ML.StaticPipe;
-using Microsoft.ML.StaticPipe.Runtime;
 
 [assembly: LoadableClass(ImagePixelExtractorTransform.Summary, typeof(IDataTransform), typeof(ImagePixelExtractorTransform), typeof(ImagePixelExtractorTransform.Arguments), typeof(SignatureDataTransform),
     ImagePixelExtractorTransform.UserName, "ImagePixelExtractorTransform", "ImagePixelExtractor")]
@@ -648,70 +646,5 @@ public override SchemaShape GetOutputSchema(SchemaShape inputSchema)
 
             return new SchemaShape(result.Values);
         }
-
-        private interface IColInput
-        {
-            Custom<Bitmap> Input { get; }
-
-            ImagePixelExtractorTransform.ColumnInfo MakeColumnInfo(string input, string output);
-        }
-
-        internal sealed class OutPipelineColumn<T> : Vector<T>, IColInput
-        {
-            public Custom<Bitmap> Input { get; }
-            private static readonly ImagePixelExtractorTransform.Arguments _defaultArgs = new ImagePixelExtractorTransform.Arguments();
-            private readonly ImagePixelExtractorTransform.Column _colParam;
-
-            public OutPipelineColumn(Custom<Bitmap> input, ImagePixelExtractorTransform.Column col)
-                : base(Reconciler.Inst, input)
-            {
-                Contracts.AssertValue(input);
-                Contracts.Assert(typeof(T) == typeof(float) || typeof(T) == typeof(byte));
-                Input = input;
-                _colParam = col;
-            }
-
-            public ImagePixelExtractorTransform.ColumnInfo MakeColumnInfo(string input, string output)
-            {
-                // In principle, the analyzer should only call the the reconciler once for these columns.
-                Contracts.Assert(_colParam.Source == null);
-                Contracts.Assert(_colParam.Name == null);
-
-                _colParam.Name = output;
-                _colParam.Source = input;
-                return new ImagePixelExtractorTransform.ColumnInfo(_colParam, _defaultArgs);
-            }
-        }
-
-        /// <summary>
-        /// Reconciler to an <see cref="ImagePixelExtractingEstimator"/> for the <see cref="PipelineColumn"/>.
-        /// </summary>
-        /// <remarks>Because we want to use the same reconciler for </remarks>
-        /// <see cref="ImageStaticPipe.ExtractPixels(Custom{Bitmap}, bool, bool, bool, bool, bool, float, float)"/>
-        /// <see cref="ImageStaticPipe.ExtractPixelsAsBytes(Custom{Bitmap}, bool, bool, bool, bool, bool)"/>
-        private sealed class Reconciler : EstimatorReconciler
-        {
-            /// <summary>
-            /// Because there are no global settings that cannot be overridden, we can always just use the same reconciler.
-            /// </summary>
-            public static Reconciler Inst = new Reconciler();
-
-            private Reconciler() { }
-
-            public override IEstimator<ITransformer> Reconcile(IHostEnvironment env,
-                PipelineColumn[] toOutput,
-                IReadOnlyDictionary<PipelineColumn, string> inputNames,
-                IReadOnlyDictionary<PipelineColumn, string> outputNames,
-                IReadOnlyCollection<string> usedNames)
-            {
-                var cols = new ImagePixelExtractorTransform.ColumnInfo[toOutput.Length];
-                for (int i = 0; i < toOutput.Length; ++i)
-                {
-                    var outCol = (IColInput)toOutput[i];
-                    cols[i] = outCol.MakeColumnInfo(inputNames[outCol.Input], outputNames[toOutput[i]]);
-                }
-                return new ImagePixelExtractingEstimator(env, cols);
-            }
-        }
     }
 }