Add text normalizer transformer to AutoML

src/Microsoft.ML.AutoML/CodeGen/estimator-schema.json

@@ -75,7 +75,8 @@
               "SentenceSimilarity",
               "ObjectDetection",
               "QuestionAnswering",
-              "NamedEntityRecognition"
+              "NamedEntityRecognition",
+              "NormalizeText"
             ]
           },
           "nugetDependencies": {
@@ -114,7 +115,8 @@
                 "Microsoft.ML.Transforms.Image",
                 "Microsoft.ML.Trainers.FastTree",
                 "Microsoft.ML.TorchSharp",
-                "Microsoft.ML.Trainers.LightGbm"
+                "Microsoft.ML.Trainers.LightGbm",
+                "Microsoft.ML.Transforms.Text.TextNormalizingEstimator"
               ]
             }
           },
@@ -198,7 +200,11 @@
                     "scoreThreshold",
                     "steps",
                     "initLearningRate",
-                    "weightDecay"
+                    "weightDecay",
+                    "mode",
+                    "keepPunctuations",
+                    "keepDiacritics",
+                    "keepNumbers"
                   ]
                 },
                 "argumentType": {

src/Microsoft.ML.AutoML/CodeGen/normalize_text_search_space.json

@@ -0,0 +1,34 @@
+{
+  "$schema": "./search-space-schema.json#",
+  "name": "normalize_text_option",
+  "search_space": [
+    {
+      "name": "InputColumnName",
+      "type": "string"
+    },
+    {
+      "name": "OutputColumnName",
+      "type": "string"
+    },
+    {
+      "name": "Mode",
+      "type": "caseMode",
+      "default": "CaseMode.Lower"
+    },
+    {
+      "name": "KeepDiacritics",
+      "type": "boolean",
+      "default": false
+    },
+    {
+      "name": "KeepPunctuations",
+      "type": "boolean",
+      "default": true
+    },
+    {
+      "name": "KeepNumbers",
+      "type": "boolean",
+      "default": true
+    }
+  ]
+}

src/Microsoft.ML.AutoML/CodeGen/search-space-schema.json

@@ -66,6 +66,14 @@
         "DataKind.DateTimeOffset"
       ]
     },
+    "caseMode": {
+      "type": "string",
+      "enum": [
+        "CaseMode.Lower",
+        "CaseMode.Upper",
+        "CaseMode.None"
+      ]
+    },
     "bertArchitectureArray": {
       "type": "array",
       "items": {
@@ -90,7 +98,7 @@
           "$ref": "#/definitions/dnnModelFactoryArray"
         },
         {
-          "$ref":  "#/definitions/imageClassificationArchArray"
+          "$ref": "#/definitions/imageClassificationArchArray"
         },
         {
           "$ref": "#/definitions/boolArray"
@@ -168,7 +176,8 @@
         "sentence_similarity_option",
         "object_detection_option",
         "question_answering_option",
-        "named_entity_recognition_option"
+        "named_entity_recognition_option",
+        "normalize_text_option"
       ]
     },
     "option_name": {
@@ -241,7 +250,11 @@
         "TopKAnswers",
         "TargetType",
         "PredictionColumnName",
-        "KeyData"
+        "KeyData",
+        "Mode",
+        "KeepPunctuations",
+        "KeepDiacritics",
+        "KeepNumbers"
       ]
     },
     "option_type": {
@@ -261,7 +274,8 @@
         "bertArchitecture",
         "imageClassificationArchType",
         "dataKind",
-        "dataView"
+        "dataView",
+        "caseMode"
       ]
     }
   },

src/Microsoft.ML.AutoML/CodeGen/transformer-estimators.json

@@ -180,6 +180,39 @@
       "usingStatements": [ "Microsoft.ML", "Microsoft.ML.Data" ],
       "searchOption": "featurize_text_option"
     },
+    {
+      "functionName": "NormalizeText",
+      "estimatorTypes": [ "Text" ],
+      "arguments": [
+        {
+          "argumentName": "outputColumnName",
+          "argumentType": "string"
+        },
+        {
+          "argumentName": "inputColumnName",
+          "argumentType": "string"
+        },
+        {
+          "argumentName": "mode",
+          "argumentType": "caseMode"
+        },
+        {
+          "argumentName": "keepDiacritics",
+          "argumentType": "boolean"
+        },
+        {
+          "argumentName": "keepPunctuations",
+          "argumentType": "boolean"
+        },
+        {
+          "argumentName": "keepNumbers",
+          "argumentType": "boolean"
+        }
+      ],
+      "nugetDependencies": [ "Microsoft.ML" ],
+      "usingStatements": [ "Microsoft.ML", "Microsoft.ML.Data", "Microsoft.ML.Transforms.Text.TextNormalizingEstimator"],
+      "searchOption": "normalize_text_option"
+    },
     {
       "functionName": "ConvertType",
       "estimatorTypes": [ "Conversion" ],

src/Microsoft.ML.AutoML/CodeGen/type_converter_search_space.json

@@ -18,6 +18,10 @@
     {
       "name": "KeyData",
       "type": "dataView"
+    },
+    {
+      "name": "Mode",
+      "type": "caseMode"
     }
   ]
 }

src/Microsoft.ML.AutoML/Microsoft.ML.AutoML.csproj

@@ -69,6 +69,9 @@
     <AdditionalFiles Include="CodeGen\code_gen_flag.json" />
     <AdditionalFiles Include="CodeGen\*-estimators.json" />
   </ItemGroup>
+  <ItemGroup>
+    <None Remove="CodeGen\normalize_text_search_space.json" />
+  </ItemGroup>
 
   <ItemGroup>
     <EmbeddedResource Include="Tuner\Portfolios.json">
@@ -86,7 +89,7 @@
       <!--The path needed to be hardcoded for this to work on our publishing CI-->
       <BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\native\LdaNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x86\native\LdaNative.pdb" TargetPath="..\..\runtimes\win-x86\native" />
       <BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\native\LdaNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-x64\native\LdaNative.pdb" TargetPath="..\..\runtimes\win-x64\native" />
-      <BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-arm64\native\LdaNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-arm64\native\LdaNative.pdb" TargetPath="..\..\runtimes\win-arm64\native"/>
+      <BuildOutputInPackage Condition="Exists('$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-arm64\native\LdaNative.pdb')" Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\win-arm64\native\LdaNative.pdb" TargetPath="..\..\runtimes\win-arm64\native" />
     </ItemGroup>
   </Target>
 

src/Microsoft.ML.AutoML/SweepableEstimator/Estimators/NormalizeText.cs

@@ -0,0 +1,18 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Microsoft.ML.AutoML.CodeGen
+{
+    internal partial class NormalizeText
+    {
+        public override IEstimator<ITransformer> BuildFromOption(MLContext context, NormalizeTextOption param)
+        {
+            return context.Transforms.Text.NormalizeText(param.OutputColumnName, param.InputColumnName, param.Mode, param.KeepDiacritics, param.KeepPunctuations, param.KeepNumbers);
+        }
+    }
+}

tools-local/Microsoft.ML.AutoML.SourceGenerator/SearchSpaceGenerator.cs

@@ -58,6 +58,7 @@ public void Execute(GeneratorExecutionContext context)
                         "imageClassificationArchType" => "Microsoft.ML.Vision.ImageClassificationTrainer.Architecture",
                         "dataKind" => "Microsoft.ML.Data.DataKind",
                         "dataView" => "Microsoft.ML.IDataView",
+                        "caseMode" => "Microsoft.ML.Transforms.Text.TextNormalizingEstimator.CaseMode",
                         _ => throw new ArgumentException("unknown type"),
                     };
 
@@ -78,6 +79,7 @@ public void Execute(GeneratorExecutionContext context)
                         (_, "Microsoft.ML.Vision.ImageClassificationTrainer.Architecture") => defaultToken.GetValue<string>(),
                         (_, "Microsoft.ML.Data.DataKind") => defaultToken.GetValue<string>(),
                         (_, "Microsoft.ML.IDataView") => defaultToken.GetValue<string>(),
+                        (_, "Microsoft.ML.Transforms.Text.TextNormalizingEstimator.CaseMode") => defaultToken.GetValue<string>(),
                         (_, _) => throw new ArgumentException("unknown"),
                     };
 

tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SearchSpace.cs

@@ -35,6 +35,8 @@ public virtual string TransformText()
 using BertArchitecture = Microsoft.ML.TorchSharp.NasBert.BertArchitecture;
 using static Microsoft.ML.Vision.ImageClassificationTrainer.Architecture;
 using DataKind = Microsoft.ML.Data.DataKind;
+using CaseMode = Microsoft.ML.Transforms.Text.TextNormalizingEstimator.CaseMode;
+
 #nullable enable
 
 namespace ");

tools-local/Microsoft.ML.AutoML.SourceGenerator/Template/SearchSpace.tt

@@ -13,6 +13,8 @@ using Anchor = Microsoft.ML.Transforms.Image.ImageResizingEstimator.Anchor;
 using BertArchitecture = Microsoft.ML.TorchSharp.NasBert.BertArchitecture;
 using static Microsoft.ML.Vision.ImageClassificationTrainer.Architecture;
 using DataKind = Microsoft.ML.Data.DataKind;
+using CaseMode = Microsoft.ML.Transforms.Text.TextNormalizingEstimator.CaseMode;
+
 #nullable enable
 
 namespace <#=NameSpace#>