Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

{AKS} Update error handling when creating cluster #4283

Merged
merged 1 commit into from
Jan 6, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions src/aks-preview/azext_aks_preview/decorator.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
)
from azure.cli.core import AzCommandsLoader
from azure.cli.core.azclierror import (
AzCLIError,
CLIInternalError,
InvalidArgumentValueError,
MutuallyExclusiveArgumentError,
Expand All @@ -32,6 +33,7 @@
from azure.cli.core.commands import AzCliCommand
from azure.cli.core.profiles import ResourceType
from azure.cli.core.util import get_file_json
from azure.core.exceptions import HttpResponseError
from knack.log import get_logger
from knack.prompting import prompt_y_n
from msrestazure.azure_exceptions import CloudError
Expand Down Expand Up @@ -1835,7 +1837,7 @@ def create_mc_preview(self, mc: ManagedCluster) -> ManagedCluster:

# Due to SPN replication latency, we do a few retries here
max_retry = 30
retry_exception = Exception(None)
error_msg = ""
for _ in range(0, max_retry):
try:
if self.context.get_intermediate("monitoring") and self.context.get_enable_msi_auth_for_monitoring():
Expand All @@ -1853,13 +1855,15 @@ def create_mc_preview(self, mc: ManagedCluster) -> ManagedCluster:
create_dcra=True,
)
return created_cluster
except CloudError as ex:
retry_exception = ex
# CloudError was raised before, but since the adoption of track 2 SDK,
# HttpResponseError would be raised instead
except (CloudError, HttpResponseError) as ex:
error_msg = str(ex)
if 'not found in Active Directory tenant' in ex.message:
time.sleep(3)
else:
raise ex
raise retry_exception
raise AzCLIError("Maximum number of retries exceeded. " + error_msg)


class AKSPreviewUpdateDecorator(AKSUpdateDecorator):
Expand Down
47 changes: 36 additions & 11 deletions src/aks-preview/azext_aks_preview/tests/latest/test_decorator.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,13 +50,15 @@
DecoratorMode,
)
from azure.cli.core.azclierror import (
AzCLIError,
CLIInternalError,
InvalidArgumentValueError,
MutuallyExclusiveArgumentError,
RequiredArgumentMissingError,
UnknownError,
)
from knack.util import CLIError
from azure.core.exceptions import HttpResponseError
from msrestazure.azure_exceptions import CloudError


Expand Down Expand Up @@ -915,9 +917,7 @@ def test_get_enable_pod_identity(self):
self.models,
decorator_mode=DecoratorMode.UPDATE,
)
mc_4 = self.models.ManagedCluster(
location="test_location"
)
mc_4 = self.models.ManagedCluster(location="test_location")
ctx_4.attach_mc(mc_4)
# fail on managed identity not enabled
with self.assertRaises(RequiredArgumentMissingError):
Expand Down Expand Up @@ -2501,28 +2501,29 @@ def test_create_mc_preview(self):
"resource_group_name": "test_rg_name",
"name": "test_name",
"enable_managed_identity": True,
# "enable_msi_auth_for_monitoring": True,
"no_wait": False,
},
CUSTOM_MGMT_AKS_PREVIEW,
)

dec_1.context.attach_mc(mc_1)
dec_1.context.set_intermediate(
"monitoring", True, overwrite_exists=True
)
dec_1.context.set_intermediate(
"subscription_id", "test_subscription_id", overwrite_exists=True
)
resp = requests.Response()
resp.status_code = 500
err = CloudError(resp)
err.message = "not found in Active Directory tenant"
# fail on mock CloudError
with self.assertRaises(CloudError), patch("time.sleep"), patch(

# raise exception
err_1 = HttpResponseError(
message="not found in Active Directory tenant"
)
# fail on mock HttpResponseError, max retry exceeded
with self.assertRaises(AzCLIError), patch("time.sleep"), patch(
"azure.cli.command_modules.acs.decorator.AKSCreateDecorator.create_mc"
), patch(
"azext_aks_preview.decorator.ensure_container_insights_for_monitoring",
side_effect=err,
side_effect=err_1,
) as ensure_monitoring:
dec_1.create_mc_preview(mc_1)
ensure_monitoring.assert_called_with(
Expand All @@ -2538,6 +2539,30 @@ def test_create_mc_preview(self):
create_dcra=True,
)

# raise exception
resp = Mock(
reason="error reason",
status_code=500,
text=Mock(return_value="error text"),
)
err_2 = HttpResponseError(response=resp)
# fail on mock HttpResponseError
with self.assertRaises(HttpResponseError), patch("time.sleep",), patch(
"azure.cli.command_modules.acs.decorator.AKSCreateDecorator.create_mc"
), patch(
"azext_aks_preview.decorator.ensure_container_insights_for_monitoring",
side_effect=[err_1, err_2],
):
dec_1.create_mc_preview(mc_1)

# return mc
with patch(
"azure.cli.command_modules.acs.decorator.AKSCreateDecorator.create_mc", return_value=mc_1
), patch(
"azext_aks_preview.decorator.ensure_container_insights_for_monitoring",
):
self.assertEqual(dec_1.create_mc_preview(mc_1), mc_1)


class AKSPreviewUpdateDecoratorTestCase(unittest.TestCase):
def setUp(self):
Expand Down