Add Quantum CLI Extension Live E2E Client Validation Tests (#3943)

* Add Quantum CLI extension tests.live files

* Delete commented-out code

* Add azdev setup

* Fix azdev setup path params

* Add Set-Location in Run.ps1

* Omit --cli param for azdev setup

* Use environment variable for Subscription ID

* Update environment variable names

* Update 'Running locally' instructions in README.md

* Skip test if provider not in AZURE_QUANTUM_CAPABILITIES

* Convert capabilities string to lowercase

* Use --xml-path param to save test output with the recordings

* Handle multiple providers in capabilities logic

* Test re-run

* Test re-run, Take 2

* Remove test_results.xml

src/quantum/azext_quantum/tests/latest/test_quantum_workspace.py

@@ -10,7 +10,7 @@
 from azure.cli.testsdk import (ScenarioTest, ResourceGroupPreparer)
 from azure.cli.core.azclierror import RequiredArgumentMissingError, ResourceNotFoundError
 
-from .utils import get_test_resource_group, get_test_workspace, get_test_workspace_location, get_test_workspace_storage, get_test_workspace_random_name
+from .utils import get_test_resource_group, get_test_workspace, get_test_workspace_location, get_test_workspace_storage, get_test_workspace_random_name, get_test_capabilities, get_test_workspace_provider_sku_list, all_providers_are_in_capabilities
 from ..._version_check_helper import check_version
 from datetime import datetime
 from ...__init__ import CLI_REPORTED_VERSION 
@@ -56,18 +56,22 @@ def test_workspace_create_destroy(self):
         test_resource_group = get_test_resource_group()
         test_workspace_temp = get_test_workspace_random_name()
         test_storage_account = get_test_workspace_storage()
-
-        # create
-        self.cmd(f'az quantum workspace create -g {test_resource_group} -w {test_workspace_temp} -l {test_location} -a {test_storage_account} -r "Microsoft/Basic" -o json --skip-role-assignment', checks=[
-           self.check("name", test_workspace_temp),
-           self.check("provisioningState", "Accepted")  # Status is accepted since we're not linking the storage account.
-        ])
-
-        # delete
-        self.cmd(f'az quantum workspace delete -g {test_resource_group} -w {test_workspace_temp} -o json', checks=[
-           self.check("name", test_workspace_temp),
-           self.check("provisioningState", "Deleting")
-        ])
+        test_provider_sku_list = get_test_workspace_provider_sku_list()
+
+        if all_providers_are_in_capabilities(test_provider_sku_list, get_test_capabilities()):
+            # create
+            self.cmd(f'az quantum workspace create -g {test_resource_group} -w {test_workspace_temp} -l {test_location} -a {test_storage_account} -r {test_provider_sku_list} -o json --skip-role-assignment', checks=[
+            self.check("name", test_workspace_temp),
+            self.check("provisioningState", "Accepted")  # Status is accepted since we're not linking the storage account.
+            ])
+
+            # delete
+            self.cmd(f'az quantum workspace delete -g {test_resource_group} -w {test_workspace_temp} -o json', checks=[
+            self.check("name", test_workspace_temp),
+            self.check("provisioningState", "Deleting")
+            ])
+        else:
+            self.skipTest(f"Skipping test_workspace_create_destroy: One or more providers in '{test_provider_sku_list}' not found in AZURE_QUANTUM_CAPABILITIES")
 
     @live_only()
     def test_workspace_errors(self):

src/quantum/azext_quantum/tests/latest/utils.py

@@ -8,28 +8,41 @@
 TEST_WORKSPACE_DEFAULT = "e2e-qsharp-tests"
 TEST_WORKSPACE_DEFAULT_LOCATION = "westus2"
 TEST_WORKSPACE_DEFAULT_STORAGE = "/subscriptions/916dfd6d-030c-4bd9-b579-7bb6d1926e97/resourceGroups/e2e-scenarios/providers/Microsoft.Storage/storageAccounts/e2etests"
+TEST_WORKSPACE_DEFAULT_PROVIDER_SKU_LIST = "Microsoft/Basic"
+TEST_CAPABILITIES_DEFAULT = "new.microsoft;submit.microsoft" 
 
 def get_from_os_environment(env_name, default):
     import os
     return os.environ[env_name] if env_name in os.environ and os.environ[env_name] != "" else default
 
 def get_test_subscription_id():
-    return get_from_os_environment("AZUREQUANTUM_SUBSCRIPTION_ID", TEST_SUBS_DEFAULT)
+    return get_from_os_environment("AZURE_QUANTUM_SUBSCRIPTION_ID", TEST_SUBS_DEFAULT)
 
 def get_test_resource_group():
-    return get_from_os_environment("AZUREQUANTUM_WORKSPACE_RG", TEST_RG_DEFAULT)
+    return get_from_os_environment("AZURE_QUANTUM_WORKSPACE_RG", TEST_RG_DEFAULT)
 
 def get_test_workspace():
-    return get_from_os_environment("AZUREQUANTUM_WORKSPACE_NAME", TEST_WORKSPACE_DEFAULT)
+    return get_from_os_environment("AZURE_QUANTUM_WORKSPACE_NAME", TEST_WORKSPACE_DEFAULT)
 
 def get_test_workspace_location():
-    return get_from_os_environment("AZUREQUANTUM_WORKSPACE_LOCATION", TEST_WORKSPACE_DEFAULT_LOCATION)
+    return get_from_os_environment("AZURE_QUANTUM_WORKSPACE_LOCATION", TEST_WORKSPACE_DEFAULT_LOCATION)
 
 def get_test_workspace_storage():
-    return get_from_os_environment("AZUREQUANTUM_WORKSPACE_STORAGE", TEST_WORKSPACE_DEFAULT_STORAGE)
+    return get_from_os_environment("AZURE_QUANTUM_WORKSPACE_STORAGE", TEST_WORKSPACE_DEFAULT_STORAGE)
+
+def get_test_workspace_provider_sku_list():
+    return get_from_os_environment("AZURE_QUANTUM_WORKSPACE_PROVIDER_SKU_LIST", TEST_WORKSPACE_DEFAULT_PROVIDER_SKU_LIST)
+
+def get_test_capabilities():
+    return get_from_os_environment("AZURE_QUANTUM_CAPABILITIES", TEST_CAPABILITIES_DEFAULT).lower()
 
 def get_test_workspace_random_name():
     import random
     return "e2e-test-w" + str(random.randint(1000000, 9999999))
 
-
+def all_providers_are_in_capabilities(provider_sku_string, capabilities_string):
+    for provide_sku_pair in provider_sku_string.split(';'):
+        provider = "new." + provide_sku_pair.split('/')[0].lower()
+        if provider not in capabilities_string:
+            return False
+    return True

src/quantum/tests.live/README.md

@@ -0,0 +1,18 @@
+# Quantum CLI Extension Live tests
+
+This folder contains Run.ps1, a script the will run all of the live tests for the Quantum CLI Extension.  Live tests are end-to-end tests that require an actual connection with Azure Quantum
+to complete successfully.
+
+
+## Running locally
+
+1. Set the following environment variable to the Subscription ID of the "QuArC Internal Consumption" subscription:
+    * `$Env:AZURE_QUANTUM_SUBSCRIPTION_ID = "[Subscription ID]"`
+
+2. Login to Azure using either:
+    * the [Azure Account extension in VS Code](https://marketplace.visualstudio.com/items?itemName=ms-vscode.azure-account)
+    * `az login` from the [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/)
+
+3. Make this folder your current directory location: src\quantum\tests.live
+
+4. Use [`.\Run.ps1 -SkipInstall`](.\Run.ps1) to run all the tests.

src/quantum/tests.live/Run.ps1

@@ -0,0 +1,21 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+param (
+    [Parameter()]
+    [switch]
+    $SkipInstall=$False
+)
+
+# For debug, print all relevant environment variables:
+Get-ChildItem env:AZURE*, env:*VERSION, env:*OUTDIR | Format-Table | Out-String | Write-Host
+
+# Run the Quantum CLI Extension tests in an azdev environment
+Set-Location ../../..
+python -m venv env
+env\Scripts\activate.ps1
+python -m pip install -U pip
+pip install azdev
+azdev setup --repo .
+azdev extension add quantum
+az account set -s $Env:AZURE_QUANTUM_SUBSCRIPTION_ID
+azdev test quantum --live --verbose --xml-path src\quantum\azext_quantum\tests\latest\recordings