Addresses #4226 . Fixes problem when loading NormalizerTransformer from disk.

build/Dependencies.props

@@ -48,7 +48,7 @@
     <BenchmarkDotNetVersion>0.11.3</BenchmarkDotNetVersion>
     <MicrosoftCodeAnalysisTestingVersion>1.0.0-beta1-63812-02</MicrosoftCodeAnalysisTestingVersion>
     <MicrosoftMLTestDatabasesPackageVersion>0.0.5-test</MicrosoftMLTestDatabasesPackageVersion>
-    <MicrosoftMLTestModelsPackageVersion>0.0.5-test</MicrosoftMLTestModelsPackageVersion>
+    <MicrosoftMLTestModelsPackageVersion>0.0.6-test</MicrosoftMLTestModelsPackageVersion>
     <MicrosoftMLTensorFlowTestModelsVersion>0.0.11-test</MicrosoftMLTensorFlowTestModelsVersion>
     <MicrosoftMLOnnxTestModelsVersion>0.0.5-test</MicrosoftMLOnnxTestModelsVersion>
     <SystemDataSqlClientVersion>4.6.1</SystemDataSqlClientVersion>

src/Microsoft.ML.Data/Transforms/Normalizer.cs

@@ -357,7 +357,8 @@ private static VersionInfo GetVersionInfo()
         {
             return new VersionInfo(
                 modelSignature: "NORMALZR",
-                verWrittenCur: 0x00010001, // Initial
+                // verWrittenCur: 0x00010001 // Initial
+                verWrittenCur: 0x00010002, // Support for multidimensional vectors
                 verReadableCur: 0x00010001,
                 verWeCanReadBack: 0x00010001,
                 loaderSignature: LoaderSignature,
@@ -385,39 +386,74 @@ internal ColumnOptions(string name, string inputColumnName, DataViewType inputTy
             internal static DataViewType LoadType(ModelLoadContext ctx)
             {
                 Contracts.AssertValue(ctx);
+
+                if (ctx.Header.ModelVerWritten < 0x00010002)
+                {
+                    // *** Previous Binary format ***
+                    //   - bool: is vector
+                    //   - int: vector size
+                    //   - byte: ItemKind of input column (only R4 and R8 are valid)
+                    bool isVectorOld = ctx.Reader.ReadBoolean();
+                    int vectorSize = ctx.Reader.ReadInt32();
+                    Contracts.CheckDecode(vectorSize >= 0);
+                    Contracts.CheckDecode(vectorSize > 0 || !isVectorOld);
+                    InternalDataKind itemKindOld = (InternalDataKind)ctx.Reader.ReadByte();
+                    Contracts.CheckDecode(itemKindOld == InternalDataKind.R4 || itemKindOld == InternalDataKind.R8);
+                    var itemTypeOld = ColumnTypeExtensions.PrimitiveTypeFromKind(itemKindOld);
+                    return isVectorOld ? (DataViewType)(new VectorDataViewType(itemTypeOld, vectorSize)) : itemTypeOld;
+                }
+
                 // *** Binary format ***
                 //   - bool: is vector
-                //   - int: vector size
                 //   - byte: ItemKind of input column (only R4 and R8 are valid)
-                bool isVector = ctx.Reader.ReadBoolean();
-                int vectorSize = ctx.Reader.ReadInt32();
-                Contracts.CheckDecode(vectorSize >= 0);
-                Contracts.CheckDecode(vectorSize > 0 || !isVector);
+                // If it is a vector:
+                //   - int: number of dimensions
+                //   - ints: as many as dimensions, each one represent the size of each dimension
 
+                bool isVector = ctx.Reader.ReadBoolean();
                 InternalDataKind itemKind = (InternalDataKind)ctx.Reader.ReadByte();
                 Contracts.CheckDecode(itemKind == InternalDataKind.R4 || itemKind == InternalDataKind.R8);
-
                 var itemType = ColumnTypeExtensions.PrimitiveTypeFromKind(itemKind);
-                return isVector ? (DataViewType)(new VectorDataViewType(itemType, vectorSize)) : itemType;
+
+                if (!isVector)
+                    return itemType;
+
+                int ndimensions = ctx.Reader.ReadInt32();
+                Contracts.CheckDecode(ndimensions > 0);
+
+                var dimensions = new int[ndimensions];
+                for (int i = 0; i < ndimensions; i++)
+                    dimensions[i] = ctx.Reader.ReadInt32();
+
+                return new VectorDataViewType(itemType, dimensions);
             }
 
             internal static void SaveType(ModelSaveContext ctx, DataViewType type)
             {
                 Contracts.AssertValue(ctx);
                 // *** Binary format ***
                 //   - bool: is vector
-                //   - int: vector size
                 //   - byte: ItemKind of input column (only R4 and R8 are valid)
+                //   If it is a vector:
+                //   - int: number of dimensions of the vector
+                //   - ints: as many as dimensions, each one represents the size of each dimension
+
                 VectorDataViewType vectorType = type as VectorDataViewType;
                 ctx.Writer.Write(vectorType != null);
 
-                Contracts.Assert(vectorType == null || vectorType.IsKnownSize);
-                ctx.Writer.Write(vectorType?.Size ?? 0);
-
                 DataViewType itemType = vectorType?.ItemType ?? type;
                 itemType.RawType.TryGetDataKind(out InternalDataKind itemKind);
                 Contracts.Assert(itemKind == InternalDataKind.R4 || itemKind == InternalDataKind.R8);
                 ctx.Writer.Write((byte)itemKind);
+
+                Contracts.Assert(vectorType == null || vectorType.IsKnownSize);
+                if (vectorType != null)
+                {
+                    var dims = vectorType.Dimensions;
+                    ctx.Writer.Write(dims.Length);
+                    for (int i = 0; i < dims.Length; i++)
+                            ctx.Writer.Write(dims[i]);
+                }
             }
         }
 

test/Microsoft.ML.Tests/Transformers/NormalizerTests.cs

@@ -14,6 +14,7 @@
 using Microsoft.ML.TestFramework.Attributes;
 using Microsoft.ML.Tools;
 using Microsoft.ML.Transforms;
+using System.Linq;
 using Xunit;
 using Xunit.Abstractions;
 using static Microsoft.ML.Transforms.NormalizingTransformer;
@@ -886,5 +887,93 @@ void TestNormalizeLogMeanVarianceFixZeroVec()
             Assert.Equal(0f, transformedDataArray[2].Features[1]);
             Assert.Equal(0f, transformedDataArray[2].Features[4]);
         }
+
+        [Fact]
+        public void TestNormalizeBackCompatibility2()
+        {
+            // Tests backward compatibility with a normalizing transformer
+            // whose version is "verWrittenCur: 0x00010001"
+
+            string dataPath = GetDataPath(TestDatasets.iris.trainFilename);
+
+            var loader = new TextLoader(Env, new TextLoader.Options
+            {
+                Columns = new[] {
+                    new TextLoader.Column("float1", DataKind.Single, 1),
+                    new TextLoader.Column("float4", DataKind.Single, new[]{new TextLoader.Range(1, 4) }),
+                    new TextLoader.Column("double1", DataKind.Double, 1),
+                    new TextLoader.Column("double4", DataKind.Double, new[]{new TextLoader.Range(1, 4) }),
+                    new TextLoader.Column("int1", DataKind.Int32, 0),
+                    new TextLoader.Column("float0", DataKind.Single, new[]{ new TextLoader.Range { Min = 1, VariableEnd = true } }),
+                },
+                HasHeader = true
+            }, new MultiFileSource(dataPath));
+
+            var data = loader.Load(dataPath);
+
+            var modelPath = Path.Combine("TestModels", "normalizer_verwrit-00010001.zip");
+            var normalizer = ML.Model.Load(modelPath, out var schema);
+
+            var outputPath = GetOutputPath("NormalizerEstimator", "normalized2.tsv");
+            using (var ch = Env.Start("save"))
+            {
+                var saver = new TextSaver(Env, new TextSaver.Arguments { Silent = true });
+                using (var fs = File.Create(outputPath))
+                {
+                    var transformedData = normalizer.Transform(data);
+                    var dataView = ML.Transforms.DropColumns(new[] { "float0" }).Fit(transformedData).Transform(transformedData);
+                    DataSaverUtils.SaveDataView(ch, saver, dataView, fs, keepHidden: true);
+                }
+            }
+
+            CheckEquality("NormalizerEstimator", "normalized2.tsv", "normalized.tsv");
+
+            Done();
+        }
+
+        public class TensorData
+        {
+            private const int dim1 = 2;
+            private const int dim2 = 3;
+            private const int dim3 = 4;
+            private const int size = dim1 * dim2 * dim3;
+
+            [VectorType(dim1, dim2, dim3)]
+            public float[] input { get; set; }
+
+            public static TensorData[] GetTensorData()
+            {
+                var tensor1 = Enumerable.Range(0, size).Select(
+                x => (float)x).ToArray();
+
+                var tensor2 = Enumerable.Range(0, size).Select(
+                x => (float)(x + 10000)).ToArray();
+
+                return new TensorData[]
+                {
+                    new TensorData() { input = tensor1},
+                    new TensorData() { input = tensor2}
+                };
+            }
+        }
+
+        [Fact]
+        void TestSavingNormalizerWithMultidimensionalVectorInput()
+        {
+            var samples = TensorData.GetTensorData();
+            var data = ML.Data.LoadFromEnumerable(samples);
+            var model = ML.Transforms.NormalizeMinMax("output", "input").Fit(data);
+            var transformedData = model.Transform(data);
+
+            var modelAndSchemaPath = GetOutputPath("TestSavingNormalizerWithMultidimensionalVectorInput.zip");
+            ML.Model.Save(model, data.Schema, modelAndSchemaPath);
+            var loadedModel = ML.Model.Load(modelAndSchemaPath, out var schema);
+            var transformedData2 = loadedModel.Transform(data);
+
+            var dimensions1 = (transformedData.Schema["output"].Type as VectorDataViewType).Dimensions;
+            var dimensions2 = (transformedData2.Schema["output"].Type as VectorDataViewType).Dimensions;
+
+            Assert.True(dimensions1.SequenceEqual(dimensions2));
+        }
     }
 }