[dataprotection] Support for PostgreSQLFlexibleServer and MySQL and c…

…ode cleanup (#7490) * Ignore intermediary jsons * Upgrade all modules to 2023-11-01 * all tests passing for updated API version * Add CRR group, validateCRR working, add ARG vault, modify ARG BI, cleanup BI init * fixed BI init, imports in helpers * Restore trigger now definitely works, about to start customizing * Validate CRR, BV list from resourcegraph complete Also code cleanup. * Recovery point list supports use-secondary-region now * Complete commands: restore trigger, job fetch. Added a secondary region mapping as fallback in case of poorly populated fields from service. * Hiding explicit CRR commands * backup vault creation/editing with CRR * Alias for --cross-region-restore-state * Backup vault ID option for list-from-resourcegraph * CRR Vault create, update tests updated * Backup vault list from RG test created * Cleaning up restore initialize * Running tests * Complete all tests for CRR refresh * fixed Update msi permissions * add tests, commented out * fix linter * version number update * single job show added, all tests except CRR recorded * committing CRR test, runs but not recorded * NON TESTED initial progress on pgflex * Everything complete, proceeding to review * Trying to pass the linter rules * Updating history * linter show_job rules * linter show_job rules * flake8 for test folder * flake8 more issues * show_command for linter * fix show command * All linters work * All linters work * Update MSI work for backup pgflex * Untested cleanup for update-msi, new workload should work. * Corrected role assignment for restore DSperms, minor refactoring * Fixed templatized help text for update perms for restore. Generic text for DS also added. * Made changes for update perms in MySQL and restore fixxes * Testing for new workloads * Version 1.1.0 and history * Style and linter changes * Linter long option name * Fix for linter exclusion * Remove experimental tag. --------- Co-authored-by: Zubair Abid <zubairabid@microsoft.com>
Azure · Apr 17, 2024 · 2f40c7f · 2f40c7f
1 parent cd3b6ff
commit 2f40c7f
Show file tree

Hide file tree

Showing 15 changed files with 24,443 additions and 171 deletions.
diff --git a/linter_exclusions.yml b/linter_exclusions.yml
@@ -536,6 +536,11 @@ dataprotection backup-instance restore initialize-for-data-recovery-as-files:
     target_blob_container_url:
       rule_exclusions:
       - option_length_too_long
+dataprotection backup-instance update-msi-permissions:
+  parameters:
+    target_storage_account_id:
+      rule_exclusions:
+      - option_length_too_long
 dataprotection backup-vault create:
   parameters:
     alerts_for_all_job_failures:

diff --git a/src/dataprotection/HISTORY.rst b/src/dataprotection/HISTORY.rst
@@ -3,6 +3,11 @@
 Release History
 ===============
 
+1.1.0
++++++
+* Added dataprotection support for PostgreSQLFlexibleServer and MySQL workloads: new manifests, code cleanup.
+* `az dataprotection backup-instance update-msi-permissions`: New parameter `--target-storage-account-id` for Restore, support Restore for new workloads, code cleanup.
+
 1.0.0
 ++++++
 * Added support for Cross Region Restore for Backup Vaults.

diff --git a/src/dataprotection/azext_dataprotection/azext_metadata.json b/src/dataprotection/azext_dataprotection/azext_metadata.json
@@ -1,4 +1,3 @@
 {
-    "azext.isExperimental": true,
     "azext.minCliCoreVersion": "2.51.0"
 }
diff --git a/src/dataprotection/azext_dataprotection/manual/Manifests/AzureDatabaseForMySQL.py b/src/dataprotection/azext_dataprotection/manual/Manifests/AzureDatabaseForMySQL.py
@@ -0,0 +1,98 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+manifest = '''
+{
+  "isProxyResource": false,
+  "enableDataSourceSetInfo": false,
+  "resourceType": "Microsoft.DBforMySQL/flexibleServers",
+  "parentResourceType": "Microsoft.DBforMySQL/flexibleServers",
+  "datasourceType": "Microsoft.DBforMySQL/flexibleServers",
+  "allowedRestoreModes": [ "RecoveryPointBased" ],
+  "allowedRestoreTargetTypes": [ "RestoreAsFiles" ],
+  "itemLevelRecoveyEnabled": false,
+  "addBackupDatasourceParametersList": false,
+  "addDataStoreParametersList": false,
+  "friendlyNameRequired": false,
+  "supportSecretStoreAuthentication": false,
+  "backupVaultPermissions": [
+    {
+      "roleDefinitionName": "Reader",
+      "type": "DataSourceRG"
+    },
+    {
+      "roleDefinitionName": "MySQL Backup And Export Operator",
+      "type": "DataSource"
+    }
+  ],
+  "backupVaultRestorePermissions": [
+    {
+      "roleDefinitionName": "Storage Blob Data Contributor",
+      "type": "TargetStorageAccount"
+    }
+  ],
+  "policySettings": {
+    "supportedRetentionTags": [ "Weekly", "Monthly", "Yearly" ],
+    "supportedDatastoreTypes": [ "VaultStore", "ArchiveStore" ],
+    "disableAddRetentionRule": false,
+    "disableCustomRetentionTag": false,
+    "backupScheduleSupported": true,
+    "supportedBackupFrequency": [ "Weekly" ],
+    "defaultPolicy": {
+    "policyRules": [
+      {
+        "name": "BackupWeekly",
+        "objectType": "AzureBackupRule",
+        "backupParameters": {
+          "backupType": "Full",
+          "objectType": "AzureBackupParams"
+        },
+        "dataStore": {
+          "dataStoreType": "VaultStore",
+          "objectType": "DataStoreInfoBase"
+        },
+        "trigger": {
+          "schedule": {
+            "timeZone": "UTC",
+            "repeatingTimeIntervals": [ "R/2021-08-15T06:30:00+00:00/P1W" ]
+          },
+          "taggingCriteria": [
+            {
+              "isDefault": true,
+              "taggingPriority": 99,
+              "tagInfo": {
+                "id": "Default_",
+                "tagName": "Default"
+              }
+            }
+          ],
+          "objectType": "ScheduleBasedTriggerContext"
+        }
+      },
+      {
+        "name": "Default",
+        "objectType": "AzureRetentionRule",
+        "isDefault": true,
+        "lifecycles": [
+          {
+            "deleteAfter": {
+              "duration": "P3M",
+              "objectType": "AbsoluteDeleteOption"
+            },
+            "sourceDataStore": {
+              "dataStoreType": "VaultStore",
+              "objectType": "DataStoreInfoBase"
+            },
+            "targetDataStoreCopySettings": []
+          }
+        ]
+      }
+    ],
+    "datasourceTypes": [ "Microsoft.DBforMySQL/flexibleServers" ],
+    "objectType": "BackupPolicy",
+    "name": "MySQLFlexiblePolicy1"
+    }
+  }
+}'''
diff --git a/...tection/azext_dataprotection/manual/Manifests/AzureDatabaseForPostgreSQLFlexibleServer.py b/...tection/azext_dataprotection/manual/Manifests/AzureDatabaseForPostgreSQLFlexibleServer.py
@@ -0,0 +1,98 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+manifest = '''
+{
+  "isProxyResource": false,
+  "enableDataSourceSetInfo": false,
+  "resourceType": "Microsoft.DBforPostgreSQL/flexibleServers",
+  "parentResourceType": "Microsoft.DBforPostgreSQL/flexibleServers",
+  "datasourceType": "Microsoft.DBforPostgreSQL/flexibleServers",
+  "allowedRestoreModes": [ "RecoveryPointBased" ],
+  "allowedRestoreTargetTypes": [ "RestoreAsFiles" ],
+  "itemLevelRecoveyEnabled": false,
+  "addBackupDatasourceParametersList": false,
+  "addDataStoreParametersList": false,
+  "friendlyNameRequired": false,
+  "supportSecretStoreAuthentication": false,
+  "backupVaultPermissions": [
+    {
+      "roleDefinitionName": "Reader",
+      "type": "DataSourceRG"
+    },
+    {
+      "roleDefinitionName": "PostgreSQL Flexible Server Long Term Retention Backup Role",
+      "type": "DataSource"
+    }
+  ],
+  "backupVaultRestorePermissions": [
+    {
+      "roleDefinitionName": "Storage Blob Data Contributor",
+      "type": "TargetStorageAccount"
+    }
+  ],
+  "policySettings": {
+    "supportedRetentionTags": [ "Weekly", "Monthly", "Yearly" ],
+    "supportedDatastoreTypes": [ "VaultStore", "ArchiveStore" ],
+    "disableAddRetentionRule": false,
+    "disableCustomRetentionTag": false,
+    "backupScheduleSupported": true,
+    "supportedBackupFrequency": [ "Weekly" ],
+    "defaultPolicy": {
+    "policyRules": [
+      {
+        "name": "BackupWeekly",
+        "objectType": "AzureBackupRule",
+        "backupParameters": {
+          "backupType": "Full",
+          "objectType": "AzureBackupParams"
+        },
+        "dataStore": {
+          "dataStoreType": "VaultStore",
+          "objectType": "DataStoreInfoBase"
+        },
+        "trigger": {
+          "schedule": {
+            "timeZone": "UTC",
+            "repeatingTimeIntervals": [ "R/2021-08-15T06:30:00+00:00/P1W" ]
+          },
+          "taggingCriteria": [
+            {
+              "isDefault": true,
+              "taggingPriority": 99,
+              "tagInfo": {
+                "id": "Default_",
+                "tagName": "Default"
+              }
+            }
+          ],
+          "objectType": "ScheduleBasedTriggerContext"
+        }
+      },
+      {
+        "name": "Default",
+        "objectType": "AzureRetentionRule",
+        "isDefault": true,
+        "lifecycles": [
+          {
+            "deleteAfter": {
+              "duration": "P3M",
+              "objectType": "AbsoluteDeleteOption"
+            },
+            "sourceDataStore": {
+              "dataStoreType": "VaultStore",
+              "objectType": "DataStoreInfoBase"
+            },
+            "targetDataStoreCopySettings": []
+          }
+        ]
+      }
+    ],
+    "datasourceTypes": [ "Microsoft.DBforPostgreSQL/flexibleServers" ],
+    "objectType": "BackupPolicy",
+    "name": "OssFlexiblePolicy1"
+    }
+  }
+}'''
diff --git a/src/dataprotection/azext_dataprotection/manual/Manifests/AzureKubernetesService.py b/src/dataprotection/azext_dataprotection/manual/Manifests/AzureKubernetesService.py
@@ -33,6 +33,22 @@
         "type": "SnapshotRG"
     }
   ],
+  "backupVaultRestorePermissions": [
+    {
+        "roleDefinitionName": "Reader",
+        "type": "DataSource"
+    },
+    {
+        "roleDefinitionName": "Reader",
+        "type": "SnapshotRG"
+    }
+  ],
+  "dataSourceRestorePermissions": [
+    {
+        "roleDefinitionName": "Contributor",
+        "type": "SnapshotRG"
+    }
+  ],
   "policySettings": {
     "supportedRetentionTags": [ "Daily", "Weekly" ],
     "supportedDatastoreTypes": [ "OperationalStore" ],

diff --git a/src/dataprotection/azext_dataprotection/manual/Manifests/config.py b/src/dataprotection/azext_dataprotection/manual/Manifests/config.py
@@ -3,4 +3,5 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
 
-supported_datasource_types = ["AzureDisk", "AzureBlob", "AzureDatabaseForPostgreSQL", "AzureKubernetesService"]
+supported_datasource_types = ["AzureDisk", "AzureBlob", "AzureDatabaseForPostgreSQL", "AzureKubernetesService",
+                              "AzureDatabaseForPostgreSQLFlexibleServer", "AzureDatabaseForMySQL"]
diff --git a/src/dataprotection/azext_dataprotection/manual/_params.py b/src/dataprotection/azext_dataprotection/manual/_params.py
@@ -137,6 +137,8 @@ def load_arguments(self, _):
         c.argument('yes', options_list=['--yes', '-y'], help='Do not prompt for confirmation.', action='store_true')
         c.argument('snapshot_resource_group_id', options_list=['--snapshot-resource-group-id', '--snapshot-rg-id'], type=str,
                    help='ARM id of the snapshot resource group. Required when assigning permissions over snapshot resource group and the --operation is Restore')
+        c.argument('target_storage_account_id', options_list=['--target-storage-account-id'], type=str,
+                   help='ARM id of the target storage account. Required when assigning permissions over target storage account and the --operation is Restore')
         c.argument('backup_instance', type=validate_file_or_dict, help='Request body for operation "Backup" Expected value: '
                    'json-string/@json-file. Required when --operation is Backup')
         c.argument('restore_request_object', type=validate_file_or_dict, help='Request body for operation "Restore" Expected value: '

diff --git a/src/dataprotection/azext_dataprotection/manual/commands.py b/src/dataprotection/azext_dataprotection/manual/commands.py
@@ -29,11 +29,13 @@ def load_command_table(self, _):
         g.custom_command('list-from-resourcegraph', 'dataprotection_backup_instance_list_from_resourcegraph', client_factory=cf_resource_graph_client)
         g.custom_command('update-msi-permissions', 'dataprotection_backup_instance_update_msi_permissions')
         g.custom_command('update-policy', "dataprotection_backup_instance_update_policy", supports_no_wait=True)
+        g.custom_command('validate-for-restore', 'dataprotection_backup_instance_validate_for_restore', supports_no_wait=True)
 
     with self.command_group('dataprotection backup-instance restore', exception_handler=exception_handler) as g:
         g.custom_command('initialize-for-data-recovery', 'restore_initialize_for_data_recovery')
         g.custom_command('initialize-for-data-recovery-as-files', 'restore_initialize_for_data_recovery_as_files')
         g.custom_command('initialize-for-item-recovery', 'restore_initialize_for_item_recovery')
+        g.custom_command('trigger', 'dataprotection_backup_instance_restore_trigger')
 
     with self.command_group('dataprotection backup-policy', exception_handler=exception_handler) as g:
         g.custom_command('get-default-policy-template', "dataprotection_backup_policy_get_default_policy_template")
@@ -55,6 +57,8 @@ def load_command_table(self, _):
 
     with self.command_group('dataprotection job') as g:
         g.custom_command('list-from-resourcegraph', "dataprotection_job_list_from_resourcegraph", client_factory=cf_resource_graph_client)
+        g.custom_command('list', "dataprotection_job_list", exception_handler=exception_handler)
+        g.custom_show_command('show', 'dataprotection_job_show', exception_handler=exception_handler)
 
     with self.command_group('dataprotection resource-guard', exception_handler=exception_handler) as g:
         g.custom_command('list-protected-operations', 'dataprotection_resource_guard_list_protected_operations')