[Storage] Support excludePrefix, includeDeleted, and new schema fields in blob inventory

src/Storage/Storage.Management/ChangeLog.md

@@ -18,6 +18,8 @@
         - Additional information about change #1
 -->
 ## Upcoming Release
+* Supported excludePrefix, includeDeleted, and many new schema fields in Blob Inventory
+    - `New-AzStorageBlobInventoryPolicyRule`
 
 ## Version 5.1.0
 * Supported generate DataLakeGen2 Sas token with Encryption scope

src/Storage/Storage.Management/Models/PSBlobInventoryPolicy.cs

@@ -179,11 +179,16 @@ public BlobInventoryPolicyDefinition parseBlobInventoryPolicyDefinition()
         // Possible values include: 'Blob', 'Container'
         public string ObjectType { get; set; }
 
-        //     Valid values for this field for the blob object type include: Name, Creation-Time, Last-Modified, Content-Length,
-        //     Content-MD5, BlobType, AccessTier, AccessTierChangeTime, Expiry-Time, hdi_isfolder,
-        //     Owner, Group, Permissions, Acl, Snapshot, VersionId, IsCurrentVersion, Metadata, LastAccessTime, AccessTierInferred, Tags. 
-        //     Valid values for container object type include Name, Last-Modified,
-        //     Metadata, LeaseStatus, LeaseState, LeaseDuration, PublicAccess, HasImmutabilityPolicy, HasLegalHold.
+        // Valid values for this field for the blob object type include: Name, Creation-Time, Last-Modified, Content-Length,
+        // Content-MD5, BlobType, AccessTier, AccessTierChangeTime, Expiry-Time, hdi_isfolder,
+        // Owner, Group, Permissions, Acl, Snapshot, VersionId, IsCurrentVersion, Metadata, LastAccessTime, AccessTierInferred, Tags,
+        // Etag, Content-Type, Content-Encoding, Content-Language, Content-CRC64, Cache-Control, Content-Disposition,
+        // LeaseStatus, LeaseState, LeaseDuration, ServerEncrypted, Deleted, DeletionId, DeletedTime, RemainingRetentionDays,
+        // mmutabilityPolicyUntilDate, ImmutabilityPolicyMode, LegalHold,  CopyId, CopyStatus, CopySource, CopyProgress, CopyCompletionTime,
+        // CopyStatusDescription, CustomerProvidedKeySha256, RehydratePriority, ArchiveStatus, x-ms-blob-sequence-number, EncryptionScope, IncrementalCopy, TagCount
+        // Valid values for container object type include Name, Last-Modified,
+        // Metadata, LeaseStatus, LeaseState, LeaseDuration, PublicAccess, HasImmutabilityPolicy, HasLegalHold, Etag,
+        // DefaultEncryptionScope, DenyEncryptionScopeOverride, ImmutableStorageWithVersioningEnabled, Deleted, Version, DeletedTime, RemainingRetentionDays
         public string[] SchemaFields { get; set; }
 
         //private string[] BlobSchemaField = new string[] {"Name", "Creation-Time", "Last-Modified", "Content-Length", "Content-MD5", "BlobType", "AccessTier", "AccessTierChangeTime",
@@ -216,26 +221,32 @@ public PSBlobInventoryPolicyFilter() { }
         public PSBlobInventoryPolicyFilter(BlobInventoryPolicyFilter filters)
         {
             this.PrefixMatch = PSManagementPolicyRuleFilter.StringListToArray(filters.PrefixMatch);
+            this.ExcludePrefix = PSManagementPolicyRuleFilter.StringListToArray(filters.ExcludePrefix);
             this.BlobTypes = PSManagementPolicyRuleFilter.StringListToArray(filters.BlobTypes);
             this.IncludeBlobVersions = filters.IncludeBlobVersions;
             this.IncludeSnapshots = filters.IncludeSnapshots;
+            this.IncludeDeleted = filters.IncludeDeleted;
         }
 
         public BlobInventoryPolicyFilter ParseBlobInventoryPolicyFilter()
         {
             return new BlobInventoryPolicyFilter()
             {
                 PrefixMatch = PSManagementPolicyRuleFilter.StringArrayToList(this.PrefixMatch),
+                ExcludePrefix = PSManagementPolicyRuleFilter.StringArrayToList(this.ExcludePrefix),
                 BlobTypes = PSManagementPolicyRuleFilter.StringArrayToList(this.BlobTypes),
                 IncludeSnapshots = this.IncludeSnapshots,
-                IncludeBlobVersions = this.IncludeBlobVersions
+                IncludeBlobVersions = this.IncludeBlobVersions,
+                IncludeDeleted = this.IncludeDeleted
             };
         }
 
         public string[] PrefixMatch { get; set; }
+        public string[] ExcludePrefix { get; set; }
         public string[] BlobTypes { get; set; }
         public bool? IncludeBlobVersions { get; set; }
         public bool? IncludeSnapshots { get; set; }
+        public bool? IncludeDeleted { get; set; }
     }
 
     /// <summary>

src/Storage/Storage.Management/Storage.Management.format.ps1xml

@@ -947,6 +947,10 @@
             <Alignment>Left</Alignment>
             <Label>IncludeBlobVersions</Label>
           </TableColumnHeader>
+          <TableColumnHeader>
+              <Alignment>Left</Alignment>
+              <Label>IncludeDeleted</Label>
+          </TableColumnHeader>
           <TableColumnHeader>
             <Alignment>Left</Alignment>
             <Label>BlobTypes</Label>
@@ -955,6 +959,10 @@
             <Alignment>Left</Alignment>
             <Label>PrefixMatch</Label>
           </TableColumnHeader>
+          <TableColumnHeader>
+              <Alignment>Left</Alignment>
+              <Label>ExcludePrefix</Label>
+          </TableColumnHeader>
           <TableColumnHeader>
             <Alignment>Left</Alignment>
             <Label>SchemaFields</Label>
@@ -995,6 +1003,10 @@
                 <Alignment>Left</Alignment>
                 <ScriptBlock>$_.Definition.Filters.IncludeBlobVersions</ScriptBlock>
               </TableColumnItem>
+              <TableColumnItem>
+                  <Alignment>Left</Alignment>
+                  <ScriptBlock>$_.Definition.Filters.IncludeDeleted</ScriptBlock>
+              </TableColumnItem>  
               <TableColumnItem>
                 <Alignment>Left</Alignment>
                 <ScriptBlock>$_.Definition.Filters.BlobTypes</ScriptBlock>
@@ -1003,6 +1015,10 @@
                 <Alignment>Left</Alignment>
                 <ScriptBlock>$_.Definition.Filters.PrefixMatch</ScriptBlock>
               </TableColumnItem>
+              <TableColumnItem>
+                  <Alignment>Left</Alignment>
+                  <ScriptBlock>$_.Definition.Filters.ExcludePrefix</ScriptBlock>
+              </TableColumnItem>
               <TableColumnItem>
                 <Alignment>Left</Alignment>
                 <ScriptBlock>$_.Definition.SchemaFields</ScriptBlock>

src/Storage/Storage.Management/StorageAccount/NewAzureStorageBlobInventoryPolicyRule.cs

@@ -66,8 +66,13 @@ public class NewAzureStorageBlobInventoryPolicyRuleCommand : StorageAccountBaseC
             ParameterSetName = BlobRuleParameterSet,
             HelpMessage = "Specifies the fields and properties of the Blob object to be included in the inventory. Valid values include: " +
                             "Name, Creation-Time, Last-Modified, Content-Length, Content-MD5, BlobType, AccessTier, AccessTierChangeTime, Expiry-Time, hdi_isfolder, Owner, " +
-                            "Group, Permissions, Acl, Metadata, LastAccessTime, AccessTierInferred, Tags. 'Name' is a required schemafield. " + 
+                            "Group, Permissions, Acl, Metadata, LastAccessTime, AccessTierInferred, Tags, Etag, Content-Type, Content-Encoding, Content-Language, Content-CRC64, " +
+                            "Cache-Control, Content-Disposition, LeaseStatus, LeaseState, LeaseDuration, ServerEncrypted, Deleted, RemainingRetentionDays, ImmutabilityPolicyUntilDate" +
+                            "ImmutabilityPolicyMode, LegalHold, CopyId, CopyStatus, CopySource, CopyProgress, CopyCompletionTime, CopyStatusDescription, CustomerProvidedKeySha256 " +
+                            "RehydratePriority, ArchiveStatus, x-ms-blob-sequence-number, EncryptionScope, IncrementalCopy, DeletionId, DeletedTime, TagCount. " + 
+                            "'Name' is a required schemafield. " + 
                             "Schema field values 'Expiry-Time, hdi_isfolder, Owner, Group, Permissions, Acl' are valid only for Hns enabled accounts.'Tags' field is only valid for non Hns accounts." +
+                            "'Tags, TagCount' field is only valid for non Hns accounts. " + 
                             "If specify '-IncludeSnapshot', will include 'Snapshot'  in the inventory.  If specify '-IncludeBlobVersion', will include 'VersionId, 'IsCurrentVersion' in the inventory.")]
         [ValidateSet(BlobInventoryPolicyBlobSchemaField.Name,
             BlobInventoryPolicyBlobSchemaField.CreationTime,
@@ -87,15 +92,47 @@ public class NewAzureStorageBlobInventoryPolicyRuleCommand : StorageAccountBaseC
             BlobInventoryPolicyBlobSchemaField.LastAccessTime,
             BlobInventoryPolicyBlobSchemaField.AccessTierInferred,
             BlobInventoryPolicyBlobSchemaField.Tags,
+            BlobInventoryPolicyBlobSchemaField.Etag,
+            BlobInventoryPolicyBlobSchemaField.ContentType,
+            BlobInventoryPolicyBlobSchemaField.ContentEncoding,
+            BlobInventoryPolicyBlobSchemaField.ContentLanguage,
+            BlobInventoryPolicyBlobSchemaField.ContentCRC64,
+            BlobInventoryPolicyBlobSchemaField.CacheControl,
+            BlobInventoryPolicyBlobSchemaField.ContentDisposition,
+            BlobInventoryPolicyBlobSchemaField.LeaseStatus,
+            BlobInventoryPolicyBlobSchemaField.LeaseState,
+            BlobInventoryPolicyBlobSchemaField.LeaseDuration,
+            BlobInventoryPolicyBlobSchemaField.ServerEncrypted,
+            BlobInventoryPolicyBlobSchemaField.Deleted,
+            BlobInventoryPolicyBlobSchemaField.RemainingRetentionDays,
+            BlobInventoryPolicyBlobSchemaField.ImmutabilityPolicyUntilDate,
+            BlobInventoryPolicyBlobSchemaField.ImmutabilityPolicyMode,
+            BlobInventoryPolicyBlobSchemaField.LegalHold,
+            BlobInventoryPolicyBlobSchemaField.CopyId,
+            BlobInventoryPolicyBlobSchemaField.CopyStatus,
+            BlobInventoryPolicyBlobSchemaField.CopySource,
+            BlobInventoryPolicyBlobSchemaField.CopyProgress,
+            BlobInventoryPolicyBlobSchemaField.CopyCompletionTime,
+            BlobInventoryPolicyBlobSchemaField.CopyStatusDescription,
+            BlobInventoryPolicyBlobSchemaField.CustomerProvidedKeySha256,
+            BlobInventoryPolicyBlobSchemaField.RehydratePriority,
+            BlobInventoryPolicyBlobSchemaField.ArchiveStatus,
+            BlobInventoryPolicyBlobSchemaField.xmsblobsequencenumber,
+            BlobInventoryPolicyBlobSchemaField.EncryptionScope,
+            BlobInventoryPolicyBlobSchemaField.IncrementalCopy,
+            BlobInventoryPolicyBlobSchemaField.DeletionId,
+            BlobInventoryPolicyBlobSchemaField.DeletedTime,
+            BlobInventoryPolicyBlobSchemaField.TagCount,
             IgnoreCase = true)]
         [ValidateNotNullOrEmpty]
         public string[] BlobSchemaField { get; set; }
 
         [Parameter(Mandatory = true,
             ParameterSetName = ContainerRuleParameterSet,
             HelpMessage = "Specifies the fields and properties of the container object to be included in the inventory. Valid values include: " +
-                            "Name, Last-Modified, Metadata, LeaseStatus, LeaseState, LeaseDuration, PublicAccess, HasImmutabilityPolicy, HasLegalHold. " + 
-                            "'Name' is a required schemafield.")]
+                            "Name, Last-Modified, Metadata, LeaseStatus, LeaseState, LeaseDuration, PublicAccess, HasImmutabilityPolicy, HasLegalHold, " +
+                            "Etag, DefaultEncryptionScope, DenyEncryptionScopeOverride, ImmutableStorageWithVersioningEnabled, Deleted, Version, " +
+                            "DeletedTime, RemainingRetentionDays. 'Name' is a required schemafield.")]
         [ValidateSet(BlobInventoryPolicyContainerSchemaField.Name,
             BlobInventoryPolicyContainerSchemaField.LastModified,
             BlobInventoryPolicyContainerSchemaField.Metadata,
@@ -105,6 +142,14 @@ public class NewAzureStorageBlobInventoryPolicyRuleCommand : StorageAccountBaseC
             BlobInventoryPolicyContainerSchemaField.PublicAccess,
             BlobInventoryPolicyContainerSchemaField.HasImmutabilityPolicy,
             BlobInventoryPolicyContainerSchemaField.HasLegalHold,
+            BlobInventoryPolicyContainerSchemaField.Etag,
+            BlobInventoryPolicyContainerSchemaField.DefaultEncryptionScope,
+            BlobInventoryPolicyContainerSchemaField.DenyEncryptionScopeOverride,
+            BlobInventoryPolicyContainerSchemaField.ImmutableStorageWithVersioningEnabled,
+            BlobInventoryPolicyContainerSchemaField.Deleted,
+            BlobInventoryPolicyContainerSchemaField.Version,
+            BlobInventoryPolicyContainerSchemaField.DeletedTime,
+            BlobInventoryPolicyContainerSchemaField.RemainingRetentionDays,
             IgnoreCase = true)]
         [ValidateNotNullOrEmpty]
         public string[] ContainerSchemaField { get; set; }
@@ -120,31 +165,42 @@ public class NewAzureStorageBlobInventoryPolicyRuleCommand : StorageAccountBaseC
         public string[] BlobType { get; set; }
 
         [Parameter(Mandatory = false,
-            HelpMessage = "Sets an array of strings for blob prefixes to be matched..")]
+            HelpMessage = "Sets an array of strings for blob prefixes to be matched.")]
         [ValidateNotNullOrEmpty]
         public string[] PrefixMatch { get; set; }
 
+        [Parameter(Mandatory = false,
+            HelpMessage = "Sets an array of strings with maximum 10 blob prefixes to be excluded from the inventory.")]
+        [ValidateNotNullOrEmpty]
+        public string[] ExcludePrefix { get; set; }
+
         [Parameter(Mandatory = false,
             ParameterSetName = BlobRuleParameterSet,
-            HelpMessage = "The rule is disabled if set it.")]
+            HelpMessage = "Includes blob snapshots in blob inventory")]
         public SwitchParameter IncludeSnapshot { get; set; }
 
         [Parameter(Mandatory = false,
             ParameterSetName = BlobRuleParameterSet,
-            HelpMessage = "The rule is disabled if set it.")]
+            HelpMessage = "Includes blob versions in blob inventory.")]
         public SwitchParameter IncludeBlobVersion { get; set; }
 
+        [Parameter(Mandatory = false,
+            ParameterSetName = BlobRuleParameterSet,
+            HelpMessage = "Includes deleted blob in blob inventory. When include delete blob, for ContainerSchemaFields, must include 'Deleted, Version, DeletedTime and RemainingRetentionDays'. For BlobSchemaFields, on HNS enabled storage accounts, must include 'DeletionId, Deleted, DeletedTime and RemainingRetentionDays', and on Hns disabled accounts must include 'Deleted and RemainingRetentionDays', else they must be excluded.")]
+        public SwitchParameter IncludeDeleted { get; set; }
+
         public override void ExecuteCmdlet()
         {
             base.ExecuteCmdlet();
 
             PSBlobInventoryPolicyDefinition definition = new PSBlobInventoryPolicyDefinition();
-            if (this.BlobType != null || this.PrefixMatch != null || this.IncludeSnapshot.IsPresent || this.IncludeBlobVersion.IsPresent)
+            if (this.BlobType != null || this.PrefixMatch != null || this.ExcludePrefix != null || this.IncludeSnapshot.IsPresent || this.IncludeBlobVersion.IsPresent || this.IncludeDeleted.IsPresent)
             {
                 definition.Filters = new PSBlobInventoryPolicyFilter()
                 {
                     BlobTypes = NormalizeStringArray<AzureBlobType>(this.BlobType),
-                    PrefixMatch = this.PrefixMatch
+                    PrefixMatch = this.PrefixMatch,
+                    ExcludePrefix = this.ExcludePrefix
                 };
                 if (this.IncludeBlobVersion.IsPresent)
                 {
@@ -154,6 +210,10 @@ public override void ExecuteCmdlet()
                 {
                     definition.Filters.IncludeSnapshots = true;
                 }
+                if (this.IncludeDeleted.IsPresent)
+                {
+                    definition.Filters.IncludeDeleted = true;
+                }
             }
             definition.Format = NormalizeString<BlobInventoryPolicyRuleFormat>(this.Format);
             definition.Schedule = NormalizeString<BlobInventoryPolicyRuleSchedule>(this.Schedule);
@@ -218,6 +278,14 @@ protected struct BlobInventoryPolicyContainerSchemaField
             public const string PublicAccess = "PublicAccess";
             public const string HasImmutabilityPolicy = "HasImmutabilityPolicy";
             public const string HasLegalHold = "HasLegalHold";
+            public const string Etag = "Etag";
+            public const string DefaultEncryptionScope = "DefaultEncryptionScope";
+            public const string DenyEncryptionScopeOverride = "DenyEncryptionScopeOverride";
+            public const string ImmutableStorageWithVersioningEnabled = "ImmutableStorageWithVersioningEnabled";
+            public const string Deleted = "Deleted";
+            public const string Version = "Version";
+            public const string DeletedTime = "DeletedTime";
+            public const string RemainingRetentionDays = "RemainingRetentionDays";
         }
 
         protected struct BlobInventoryPolicyBlobSchemaField
@@ -243,6 +311,37 @@ protected struct BlobInventoryPolicyBlobSchemaField
             public const string LastAccessTime = "LastAccessTime";
             public const string AccessTierInferred = "AccessTierInferred";
             public const string Tags = "Tags";
+            public const string Etag = "Etag";
+            public const string ContentType = "Content-Type";
+            public const string ContentEncoding = "Content-Encoding";
+            public const string ContentLanguage = "Content-Language";
+            public const string ContentCRC64 = "Content-CRC64";
+            public const string CacheControl = "Cache-Control";
+            public const string ContentDisposition = "Content-Disposition";
+            public const string LeaseStatus = "LeaseStatus";
+            public const string LeaseState = "LeaseState";
+            public const string LeaseDuration = "LeaseDuration";
+            public const string ServerEncrypted = "ServerEncrypted";
+            public const string Deleted = "Deleted";
+            public const string RemainingRetentionDays = "RemainingRetentionDays";
+            public const string ImmutabilityPolicyUntilDate = "ImmutabilityPolicyUntilDate";
+            public const string ImmutabilityPolicyMode = "ImmutabilityPolicyMode";
+            public const string LegalHold = "LegalHold";
+            public const string CopyId = "CopyId";
+            public const string CopyStatus = "CopyStatus";
+            public const string CopySource = "CopySource";
+            public const string CopyProgress = "CopyProgress";
+            public const string CopyCompletionTime = "CopyCompletionTime";
+            public const string CopyStatusDescription = "CopyStatusDescription";
+            public const string CustomerProvidedKeySha256 = "CustomerProvidedKeySha256";
+            public const string RehydratePriority = "RehydratePriority";
+            public const string ArchiveStatus = "ArchiveStatus";
+            public const string xmsblobsequencenumber = "x-ms-blob-sequence-number";
+            public const string EncryptionScope = "EncryptionScope";
+            public const string IncrementalCopy = "IncrementalCopy";
+            public const string DeletionId = "DeletionId";
+            public const string DeletedTime = "DeletedTime";
+            public const string TagCount = "TagCount";
         }
 
         protected struct BlobInventoryPolicyRuleSchedule