Unify cmdlets with parameter 'Encoding' to be of type System.Text.Enc…

…oding (PowerShell#5080)

This unifies file encoding across the inbox cmdlets to be UTF-8 without a BOM for all platforms. This is a breaking change as cmdlets on windows have a number of different encodings. This supports better interoperability with tradition Linux shells as we are using the same encoding.

Validate that files are created with UTF-8 encoding without BOM
Update tests to validate Encoding parameter to new type and create new tests for
parameter type validation.

[Breaking Change] The '-Encoding Byte' has been removed from the filesystem provider cmdlets. A new parameter '-AsByteStream' is now added to indicate that a byte stream is required as input, or output will be a stream of bytes.

src/Microsoft.PowerShell.Commands.Utility/commands/utility/CSVCommands.cs

@@ -212,8 +212,20 @@ public SwitchParameter NoClobber
         /// Encoding optional flag
         /// </summary>
         [Parameter()]
-        [ValidateSetAttribute(new string[] { "Unicode", "UTF7", "UTF8", "ASCII", "UTF32", "BigEndianUnicode", "Default", "OEM" })]
-        public string Encoding { get; set; }
+        [ArgumentToEncodingTransformationAttribute()]
+        [ArgumentCompletions(
+            EncodingConversion.Ascii,
+            EncodingConversion.BigEndianUnicode,
+            EncodingConversion.OEM,
+            EncodingConversion.Unicode,
+            EncodingConversion.Utf7,
+            EncodingConversion.Utf8,
+            EncodingConversion.Utf8Bom,
+            EncodingConversion.Utf8NoBom,
+            EncodingConversion.Utf32
+            )]
+        [ValidateNotNullOrEmpty]
+        public Encoding Encoding { get; set; } = ClrFacade.GetDefaultEncoding();
 
         /// <summary>
         /// Property that sets append parameter.
@@ -373,7 +385,7 @@ private void CreateFileStream()
                 PathUtils.MasterStreamOpen(
                     this,
                     this.Path,
-                    Encoding ?? "ASCII",
+                    Encoding,
                     false, // defaultEncoding
                     Append,
                     Force,
@@ -577,8 +589,20 @@ public SwitchParameter UseCulture
         /// Encoding optional flag
         /// </summary>
         [Parameter()]
-        [ValidateSetAttribute(new[] { "Unicode", "UTF7", "UTF8", "ASCII", "UTF32", "BigEndianUnicode", "Default", "OEM" })]
-        public string Encoding { get; set; }
+        [ArgumentToEncodingTransformationAttribute()]
+        [ArgumentCompletions(
+            EncodingConversion.Ascii,
+            EncodingConversion.BigEndianUnicode,
+            EncodingConversion.OEM,
+            EncodingConversion.Unicode,
+            EncodingConversion.Utf7,
+            EncodingConversion.Utf8,
+            EncodingConversion.Utf8Bom,
+            EncodingConversion.Utf8NoBom,
+            EncodingConversion.Utf32
+            )]
+        [ValidateNotNullOrEmpty]
+        public Encoding Encoding { get; set; } = ClrFacade.GetDefaultEncoding();
 
         /// <summary>
         /// Avoid writing out duplicate warning messages when there are

src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/format-hex/Format-Hex.cs

@@ -46,14 +46,20 @@ public sealed class FormatHex : PSCmdlet
         /// Type of character encoding for InputObject
         /// </summary>
         [Parameter(ParameterSetName = "ByInputObject")]
-        [ValidateSetAttribute(new string[] {
-            EncodingConversion.Unicode,
+        [ArgumentToEncodingTransformationAttribute()]
+        [ArgumentCompletions(
+            EncodingConversion.Ascii,
             EncodingConversion.BigEndianUnicode,
-            EncodingConversion.Utf8,
+            EncodingConversion.OEM,
+            EncodingConversion.Unicode,
             EncodingConversion.Utf7,
-            EncodingConversion.Utf32,
-            EncodingConversion.Ascii})]
-        public string Encoding { get; set; } = "Ascii";
+            EncodingConversion.Utf8,
+            EncodingConversion.Utf8Bom,
+            EncodingConversion.Utf8NoBom,
+            EncodingConversion.Utf32
+            )]
+        [ValidateNotNullOrEmpty]
+        public Encoding Encoding { get; set; } = ClrFacade.GetDefaultEncoding();
 
         /// <summary>
         /// This parameter is no-op
@@ -239,8 +245,7 @@ private void ProcessObjectContent(PSObject inputObject)
             else if (obj is string)
             {
                 string inputString = obj.ToString();
-                Encoding resolvedEncoding = EncodingConversion.Convert(this, Encoding);
-                inputBytes = resolvedEncoding.GetBytes(inputString);
+                inputBytes = Encoding.GetBytes(inputString);
             }
 
             else if (obj is byte)

src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/out-file/Out-File.cs

@@ -3,6 +3,7 @@
 --********************************************************************/
 
 using System;
+using System.Text;
 using System.Management.Automation;
 using System.Management.Automation.Internal;
 using System.Management.Automation.Host;
@@ -72,25 +73,20 @@ public string LiteralPath
         /// </summary>
         ///
         [Parameter(Position = 1)]
-        [ValidateNotNullOrEmpty]
-        [ValidateSetAttribute(new string[] {
-            EncodingConversion.Unknown,
-            EncodingConversion.String,
-            EncodingConversion.Unicode,
+        [ArgumentToEncodingTransformationAttribute()]
+        [ArgumentCompletions(
+            EncodingConversion.Ascii,
             EncodingConversion.BigEndianUnicode,
-            EncodingConversion.Utf8,
+            EncodingConversion.OEM,
+            EncodingConversion.Unicode,
             EncodingConversion.Utf7,
-            EncodingConversion.Utf32,
-            EncodingConversion.Ascii,
-            EncodingConversion.Default,
-            EncodingConversion.OEM })]
-        public string Encoding
-        {
-            get { return _encoding; }
-            set { _encoding = value; }
-        }
-
-        private string _encoding;
+            EncodingConversion.Utf8,
+            EncodingConversion.Utf8Bom,
+            EncodingConversion.Utf8NoBom,
+            EncodingConversion.Utf32
+            )]
+        [ValidateNotNullOrEmpty]
+        public Encoding Encoding { get; set; } = ClrFacade.GetDefaultEncoding();
 
         /// <summary>
         /// Property that sets append parameter.
@@ -196,7 +192,7 @@ private LineOutput InstantiateLineOutputInterface()
                 PathUtils.MasterStreamOpen(
                     this,
                     FilePath,
-                    _encoding,
+                    Encoding,
                     false, // defaultEncoding
                     Append,
                     Force,

src/Microsoft.PowerShell.Commands.Utility/commands/utility/ImplicitRemotingCommands.cs

@@ -77,19 +77,20 @@ public SwitchParameter Force
         /// Encoding optional flag
         /// </summary>
         [Parameter]
-        [ValidateSetAttribute(new string[] { "Unicode", "UTF7", "UTF8", "ASCII", "UTF32", "BigEndianUnicode", "Default", "OEM" })]
-        public string Encoding
-        {
-            get
-            {
-                return _encoding.GetType().Name;
-            }
-            set
-            {
-                _encoding = EncodingConversion.Convert(this, value);
-            }
-        }
-        private Encoding _encoding = System.Text.Encoding.UTF8;
+        [ArgumentToEncodingTransformationAttribute()]
+        [ArgumentCompletions(
+            EncodingConversion.Ascii,
+            EncodingConversion.BigEndianUnicode,
+            EncodingConversion.OEM,
+            EncodingConversion.Unicode,
+            EncodingConversion.Utf7,
+            EncodingConversion.Utf8,
+            EncodingConversion.Utf8Bom,
+            EncodingConversion.Utf8NoBom,
+            EncodingConversion.Utf32
+            )]
+        [ValidateNotNullOrEmpty]
+        public Encoding Encoding { get; set; } = ClrFacade.GetDefaultEncoding();
 
         #endregion Parameters
 
@@ -144,7 +145,7 @@ protected override void BeginProcessing()
             List<string> generatedFiles = GenerateProxyModule(
                 tempDirectory,
                 Path.GetFileName(directory.FullName),
-                _encoding,
+                Encoding,
                 _force,
                 listOfCommandMetadata,
                 alias2resolvedCommandName,

src/Microsoft.PowerShell.Commands.Utility/commands/utility/MatchString.cs

@@ -3,6 +3,7 @@
 --********************************************************************/
 
 using System;
+using System.Text;
 using System.Text.RegularExpressions;
 using System.IO;
 using System.Collections;
@@ -1201,19 +1202,20 @@ public SwitchParameter AllMatches
         /// The text encoding to process each file as.
         /// </summary>
         [Parameter]
-        [ValidateNotNullOrEmpty]
-        [ValidateSetAttribute(new string[] {
+        [ArgumentToEncodingTransformationAttribute()]
+        [ArgumentCompletions(
+            EncodingConversion.Ascii,
+            EncodingConversion.BigEndianUnicode,
+            EncodingConversion.OEM,
             EncodingConversion.Unicode,
             EncodingConversion.Utf7,
             EncodingConversion.Utf8,
-            EncodingConversion.Utf32,
-            EncodingConversion.Ascii,
-            EncodingConversion.BigEndianUnicode,
-            EncodingConversion.Default,
-            EncodingConversion.OEM })]
-        public string Encoding { get; set; }
-
-        private System.Text.Encoding _textEncoding;
+            EncodingConversion.Utf8Bom,
+            EncodingConversion.Utf8NoBom,
+            EncodingConversion.Utf32
+            )]
+        [ValidateNotNullOrEmpty]
+        public Encoding Encoding { get; set; } = ClrFacade.GetDefaultEncoding();
 
         /// <summary>
         /// The number of context lines to collect. If set to a
@@ -1282,16 +1284,6 @@ public SwitchParameter AllMatches
         /// </summary>
         protected override void BeginProcessing()
         {
-            // Process encoding switch.
-            if (Encoding != null)
-            {
-                _textEncoding = EncodingConversion.Convert(this, Encoding);
-            }
-            else
-            {
-                _textEncoding = new System.Text.UTF8Encoding();
-            }
-
             if (!_simpleMatch)
             {
                 RegexOptions regexOptions = (_caseSensitive) ? RegexOptions.None : RegexOptions.IgnoreCase;
@@ -1434,7 +1426,7 @@ private bool ProcessFile(string filename)
 
                 using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
                 {
-                    using (StreamReader sr = new StreamReader(fs, _textEncoding))
+                    using (StreamReader sr = new StreamReader(fs, Encoding))
                     {
                         String line;
                         int lineNo = 0;

src/Microsoft.PowerShell.Commands.Utility/commands/utility/Send-MailMessage.cs

@@ -89,20 +89,24 @@ public SwitchParameter BodyAsHtml
 
         /// <summary>
         /// Specifies the encoding used for the content of the body and also the subject.
+        /// This is set to ASCII to ensure there are no problems with any email server
         /// </summary>
         [Parameter()]
         [Alias("BE")]
         [ValidateNotNullOrEmpty]
-        [ArgumentToEncodingNameTransformationAttribute()]
-        public Encoding Encoding
-        {
-            get { return _encoding; }
-            set
-            {
-                _encoding = value;
-            }
-        }
-        private Encoding _encoding = new ASCIIEncoding();
+        [ArgumentCompletions(
+            EncodingConversion.Ascii,
+            EncodingConversion.BigEndianUnicode,
+            EncodingConversion.OEM,
+            EncodingConversion.Unicode,
+            EncodingConversion.Utf7,
+            EncodingConversion.Utf8,
+            EncodingConversion.Utf8Bom,
+            EncodingConversion.Utf8NoBom,
+            EncodingConversion.Utf32
+            )]
+        [ArgumentToEncodingTransformationAttribute()]
+        public Encoding Encoding { get; set; } = Encoding.ASCII;
 
         /// <summary>
         /// Specifies the address collection that contains the
@@ -369,8 +373,8 @@ protected override
             _mMailMessage.Body = _body;
 
             //set the subject and body encoding
-            _mMailMessage.SubjectEncoding = _encoding;
-            _mMailMessage.BodyEncoding = _encoding;
+            _mMailMessage.SubjectEncoding = Encoding;
+            _mMailMessage.BodyEncoding = Encoding;
 
             // Set the format of the mail message body as HTML
             _mMailMessage.IsBodyHtml = _bodyashtml;
@@ -490,37 +494,5 @@ protected override void EndProcessing()
         #endregion
     }
 
-    /// <summary>
-    /// To make it easier to specify -Encoding parameter, we add an ArgumentTransformationAttribute here.
-    /// When the input data is of type string and is valid to be converted to System.Text.Encoding, we do
-    /// the conversion and return the converted value. Otherwise, we just return the input data.
-    /// </summary>
-    internal sealed class ArgumentToEncodingNameTransformationAttribute : ArgumentTransformationAttribute
-    {
-        public override object Transform(EngineIntrinsics engineIntrinsics, object inputData)
-        {
-            string encodingName;
-            if (LanguagePrimitives.TryConvertTo<string>(inputData, out encodingName))
-            {
-                if (string.Equals(encodingName, EncodingConversion.Unknown, StringComparison.OrdinalIgnoreCase) ||
-                    string.Equals(encodingName, EncodingConversion.String, StringComparison.OrdinalIgnoreCase) ||
-                    string.Equals(encodingName, EncodingConversion.Unicode, StringComparison.OrdinalIgnoreCase) ||
-                    string.Equals(encodingName, EncodingConversion.BigEndianUnicode, StringComparison.OrdinalIgnoreCase) ||
-                    string.Equals(encodingName, EncodingConversion.Utf8, StringComparison.OrdinalIgnoreCase) ||
-                    string.Equals(encodingName, EncodingConversion.Utf7, StringComparison.OrdinalIgnoreCase) ||
-                    string.Equals(encodingName, EncodingConversion.Utf32, StringComparison.OrdinalIgnoreCase) ||
-                    string.Equals(encodingName, EncodingConversion.Ascii, StringComparison.OrdinalIgnoreCase) ||
-                    string.Equals(encodingName, EncodingConversion.Default, StringComparison.OrdinalIgnoreCase) ||
-                    string.Equals(encodingName, EncodingConversion.OEM, StringComparison.OrdinalIgnoreCase))
-                {
-                    // the encodingName is guaranteed to be valid, so it is safe to pass null to method
-                    // Convert(Cmdlet cmdlet, string encoding) as the value of 'cmdlet'.
-                    return EncodingConversion.Convert(null, encodingName);
-                }
-            }
-            return inputData;
-        }
-    }
-
     #endregion
-}
+}

src/Microsoft.PowerShell.Commands.Utility/commands/utility/XmlCommands.cs

@@ -108,8 +108,20 @@ public SwitchParameter NoClobber
         /// </summary>
         ///
         [Parameter]
-        [ValidateSetAttribute(new string[] { "Unicode", "UTF7", "UTF8", "ASCII", "UTF32", "BigEndianUnicode", "Default", "OEM" })]
-        public string Encoding { get; set; } = "Unicode";
+        [ArgumentToEncodingTransformationAttribute()]
+        [ArgumentCompletions(
+            EncodingConversion.Ascii,
+            EncodingConversion.BigEndianUnicode,
+            EncodingConversion.OEM,
+            EncodingConversion.Unicode,
+            EncodingConversion.Utf7,
+            EncodingConversion.Utf8,
+            EncodingConversion.Utf8Bom,
+            EncodingConversion.Utf8NoBom,
+            EncodingConversion.Utf32
+            )]
+        [ValidateNotNullOrEmpty]
+        public Encoding Encoding { get; set; } = ClrFacade.GetDefaultEncoding();
 
         #endregion Command Line Parameters