feat: enable containermonitoring addon support for Azurestack (#2153)

* also update the agent to latest version
christinalau0 · Oct 23, 2019 · 103ffc9 · 103ffc9
1 parent 966baf0
commit 103ffc9
Show file tree

Hide file tree

Showing 16 changed files with 1,570 additions and 960 deletions.
diff --git a/cmd/deploy.go b/cmd/deploy.go
@@ -358,19 +358,11 @@ func autofillApimodel(dc *deployCmd) error {
 
 	if k8sConfig != nil && k8sConfig.Addons != nil && k8sConfig.IsContainerMonitoringAddonEnabled() {
 		log.Infoln("container monitoring addon enabled")
-		var workspaceDomain string
-		cloudspecConfig := dc.containerService.GetCloudSpecConfig()
-		switch cloudspecConfig.CloudName {
-		case "AzurePublicCloud":
-			workspaceDomain = "opinsights.azure.com"
-		case "AzureChinaCloud":
-			workspaceDomain = "opinsights.azure.cn"
-		case "AzureUSGovernmentCloud":
-			workspaceDomain = "opinsights.azure.us"
-		default:
-			return errors.Wrapf(err, "apimodel: container monitoring addon not supported in this cloud: %s", cloudspecConfig.CloudName)
+		cloudOrDependenciesLocation := dc.containerService.GetCloudSpecConfig().CloudName
+		if dc.containerService.Properties.IsAzureStackCloud() {
+			cloudOrDependenciesLocation = string(dc.containerService.Properties.CustomCloudProfile.DependenciesLocation)
 		}
-
+		workspaceDomain := helpers.GetLogAnalyticsWorkspaceDomain(cloudOrDependenciesLocation)
 		err := dc.configureContainerMonitoringAddon(ctx, k8sConfig, workspaceDomain)
 		if err != nil {
 			return errors.Wrap(err, "Failed to configure container monitoring addon")
@@ -472,6 +464,9 @@ func (dc *deployCmd) configureContainerMonitoringAddon(ctx context.Context, k8sC
 	var err error
 	addon := k8sConfig.GetAddonByName("container-monitoring")
 	if addon.Config == nil || len(addon.Config) == 0 || addon.Config["logAnalyticsWorkspaceResourceId"] != "" {
+		if dc.containerService.Properties.IsAzureStackCloud() {
+			return errors.New("This is not supported option for AzureStackCloud. Please provide config with workspaceGuid and workspaceKey")
+		}
 		workspaceResourceID = strings.TrimSpace(addon.Config["logAnalyticsWorkspaceResourceId"])
 		if workspaceResourceID != "" {
 			log.Infoln("using provided log analytics workspace resource id:", workspaceResourceID)

diff --git a/cmd/deploy_test.go b/cmd/deploy_test.go
@@ -667,7 +667,6 @@ func TestAPIModelWithContainerMonitoringAddonWithNoConfigInCmd(t *testing.T) {
 	}
 
 	apimodel := ExampleAPIModelWithContainerMonitoringAddonWithNoConfig
-
 	cs, ver, err := apiloader.DeserializeContainerService([]byte(apimodel), false, false, nil)
 	if err != nil {
 		t.Fatalf("unexpected error deserializing the example apimodel: %s", err)
@@ -851,55 +850,141 @@ func TestAPIModelWithContainerMonitoringAddonWithWorkspaceGuidAndKeyConfigInCmd(
 	}
 
 	apimodel := ExampleAPIModelWithContainerMonitoringAddonWithWorkspaceGUIDAndKeyConfig
-
 	cs, ver, err := apiloader.DeserializeContainerService([]byte(apimodel), false, false, nil)
 	if err != nil {
 		t.Fatalf("unexpected error deserializing the example apimodel: %s", err)
 	}
-	deployCmd := &deployCmd{
-		apimodelPath:     "./this/is/unused.json",
-		outputDirectory:  "_test_output",
-		forceOverwrite:   true,
-		location:         "westus",
-		containerService: cs,
-		apiVersion:       ver,
 
-		client: &armhelpers.MockAKSEngineClient{},
-		authProvider: &mockAuthProvider{
-			authArgs: &authArgs{},
-		},
-	}
-	err = autofillApimodel(deployCmd)
-	if err != nil {
-		t.Fatalf("unexpected error autofilling the example apimodel: %s", err)
+	type WorkspaceInfo struct {
+		WorkspaceGUID   string
+		WorkspaceKey    string
+		WorkspaceDomain string
 	}
 
-	defer os.RemoveAll(deployCmd.outputDirectory)
-
-	k8sConfig := deployCmd.containerService.Properties.OrchestratorProfile.KubernetesConfig
-
-	if k8sConfig == nil {
-		t.Fatalf("expected valid kubernetes config")
+	cases := []struct {
+		dc               *deployCmd
+		location         string
+		expectedResponse WorkspaceInfo
+	}{
+		{
+			dc: &deployCmd{
+				apimodelPath:     "./this/is/unused.json",
+				outputDirectory:  "_test_output",
+				forceOverwrite:   true,
+				location:         "westus",
+				containerService: cs,
+				apiVersion:       ver,
+
+				client: &armhelpers.MockAKSEngineClient{},
+				authProvider: &mockAuthProvider{
+					authArgs: &authArgs{},
+				},
+			},
+			location: "westus",
+			expectedResponse: WorkspaceInfo{
+				WorkspaceGUID:   "MDAwMDAwMDAtMDAwMC0wMDAwLTAwMDAtMDAwMDAwMDAwMDAw",
+				WorkspaceKey:    "NEQrdnlkNS9qU2NCbXNBd1pPRi8wR09CUTVrdUZRYzlKVmFXK0hsbko1OGN5ZVBKY3dUcGtzK3JWbXZnY1hHbW15dWpMRE5FVlBpVDhwQjI3NGE5WWc9PQ==",
+				WorkspaceDomain: "b3BpbnNpZ2h0cy5henVyZS5jb20=",
+			},
+		},
+		{
+			dc: &deployCmd{
+				apimodelPath:     "./this/is/unused.json",
+				outputDirectory:  "_test_output",
+				forceOverwrite:   true,
+				location:         "chinaeast2",
+				containerService: cs,
+				apiVersion:       ver,
+
+				client: &armhelpers.MockAKSEngineClient{},
+				authProvider: &mockAuthProvider{
+					authArgs: &authArgs{},
+				},
+			},
+			location: "chinaeast2",
+			expectedResponse: WorkspaceInfo{
+				WorkspaceGUID:   "MDAwMDAwMDAtMDAwMC0wMDAwLTAwMDAtMDAwMDAwMDAwMDAw",
+				WorkspaceKey:    "NEQrdnlkNS9qU2NCbXNBd1pPRi8wR09CUTVrdUZRYzlKVmFXK0hsbko1OGN5ZVBKY3dUcGtzK3JWbXZnY1hHbW15dWpMRE5FVlBpVDhwQjI3NGE5WWc9PQ==",
+				WorkspaceDomain: "b3BpbnNpZ2h0cy5henVyZS5jbg==",
+			},
+		},
+		{
+			dc: &deployCmd{
+				apimodelPath:     "./this/is/unused.json",
+				outputDirectory:  "_test_output",
+				forceOverwrite:   true,
+				location:         "usgovvirginia",
+				containerService: cs,
+				apiVersion:       ver,
+
+				client: &armhelpers.MockAKSEngineClient{},
+				authProvider: &mockAuthProvider{
+					authArgs: &authArgs{},
+				},
+			},
+			location: "usgovvirginia",
+			expectedResponse: WorkspaceInfo{
+				WorkspaceGUID:   "MDAwMDAwMDAtMDAwMC0wMDAwLTAwMDAtMDAwMDAwMDAwMDAw",
+				WorkspaceKey:    "NEQrdnlkNS9qU2NCbXNBd1pPRi8wR09CUTVrdUZRYzlKVmFXK0hsbko1OGN5ZVBKY3dUcGtzK3JWbXZnY1hHbW15dWpMRE5FVlBpVDhwQjI3NGE5WWc9PQ==",
+				WorkspaceDomain: "b3BpbnNpZ2h0cy5henVyZS51cw==",
+			},
+		},
+		{
+			dc: &deployCmd{
+				apimodelPath:     "./this/is/unused.json",
+				outputDirectory:  "_test_output",
+				forceOverwrite:   true,
+				location:         "germanynortheast",
+				containerService: cs,
+				apiVersion:       ver,
+
+				client: &armhelpers.MockAKSEngineClient{},
+				authProvider: &mockAuthProvider{
+					authArgs: &authArgs{},
+				},
+			},
+			location: "germanynortheast",
+			expectedResponse: WorkspaceInfo{
+				WorkspaceGUID:   "MDAwMDAwMDAtMDAwMC0wMDAwLTAwMDAtMDAwMDAwMDAwMDAw",
+				WorkspaceKey:    "NEQrdnlkNS9qU2NCbXNBd1pPRi8wR09CUTVrdUZRYzlKVmFXK0hsbko1OGN5ZVBKY3dUcGtzK3JWbXZnY1hHbW15dWpMRE5FVlBpVDhwQjI3NGE5WWc9PQ==",
+				WorkspaceDomain: "b3BpbnNpZ2h0cy5henVyZS5kZQ==",
+			},
+		},
 	}
 
-	if len(k8sConfig.Addons) != 1 {
-		t.Fatalf("expected one addon")
-	}
+	for _, c := range cases {
+		c.dc.containerService.Location = c.location
+		err = autofillApimodel(c.dc)
+		if err != nil {
+			t.Fatalf("unexpected error autofilling the example apimodel: %s", err)
+		}
 
-	addon := k8sConfig.Addons[0]
+		defer os.RemoveAll(c.dc.outputDirectory)
 
-	if addon.Name != "container-monitoring" {
-		t.Fatalf("unexpected addon found : %s", addon.Name)
-	}
+		k8sConfig := c.dc.containerService.Properties.OrchestratorProfile.KubernetesConfig
+		if k8sConfig == nil {
+			t.Fatalf("expected valid kubernetes config")
+		}
+		if len(k8sConfig.Addons) != 1 {
+			t.Fatalf("expected one addon")
+		}
+		addon := k8sConfig.Addons[0]
+		if addon.Name != "container-monitoring" {
+			t.Fatalf("unexpected addon found : %s", addon.Name)
+		}
 
-	expectedWorkspaceGUIDInBase64 := "MDAwMDAwMDAtMDAwMC0wMDAwLTAwMDAtMDAwMDAwMDAwMDAw"
-	if addon.Config["workspaceGuid"] != expectedWorkspaceGUIDInBase64 {
-		t.Fatalf("expected workspaceGuid : %s but got : %s", expectedWorkspaceGUIDInBase64, addon.Config["workspaceGuid"])
-	}
+		if addon.Config["workspaceGuid"] != c.expectedResponse.WorkspaceGUID {
+			t.Fatalf("expected workspaceGuid : %s but got : %s", c.expectedResponse.WorkspaceGUID, addon.Config["workspaceGuid"])
+		}
 
-	expectedWorkspaceKeyInBase64 := "NEQrdnlkNS9qU2NCbXNBd1pPRi8wR09CUTVrdUZRYzlKVmFXK0hsbko1OGN5ZVBKY3dUcGtzK3JWbXZnY1hHbW15dWpMRE5FVlBpVDhwQjI3NGE5WWc9PQ=="
-	if addon.Config["workspaceKey"] != expectedWorkspaceKeyInBase64 {
-		t.Fatalf("unexpected workspaceKey : %s", addon.Config["workspaceKey"])
-		t.Fatalf("expected workspaceKey : %s but got : %s", expectedWorkspaceKeyInBase64, addon.Config["workspaceKey"])
+		if addon.Config["workspaceKey"] != c.expectedResponse.WorkspaceKey {
+			t.Fatalf("unexpected workspaceKey : %s", addon.Config["workspaceKey"])
+			t.Fatalf("expected workspaceKey : %s but got : %s", c.expectedResponse.WorkspaceKey, addon.Config["workspaceKey"])
+		}
+
+		if addon.Config["workspaceDomain"] != c.expectedResponse.WorkspaceDomain {
+			t.Fatalf("unexpected workspaceDomain : %s", addon.Config["workspaceDomain"])
+			t.Fatalf("expected workspaceDomain : %s but got : %s", c.expectedResponse.WorkspaceDomain, addon.Config["workspaceDomain"])
+		}
 	}
 }
diff --git a/docs/topics/azure-stack.md b/docs/topics/azure-stack.md
@@ -12,6 +12,7 @@
     * [agentPoolProfiles](#agentPoolProfiles)
 * [Azure Stack Instances Registered with Azure's China cloud](#azure-stack-instances-registered-with-azures-china-cloud)
 * [Disconnected Azure Stack Instances](#disconnected-azure-stack-instances)
+* [Azure Monitor for containers](#azure-Monitor-for-containers)
 * [Unsupported Addons](#unsupported-addons)
 * [Known Issues and Limitations](#known-issues-and-limitations)
 * [Frequently Asked Questions](#frequently-asked-questions)
@@ -135,6 +136,11 @@ The `AKS Base Image` marketplace item has to be available in your Azure Stack's
 
 Each AKS Engine release is validated and tied to a specific version of the AKS Base Image. Therefore, you need to take note of the base image version required by the AKS Engine release that you plan to use, and then download exactly that base image version. New builds of the `AKS Base Image` are frequently released to ensure that your disconnected cluster can be upgraded to the latest supported version of each component.
 
+## Azure Monitor for containers
+
+Azure Monitor for containers supports the monitoring of AKS-Engine clusters hosted in Azure Stack Cloud Environment(s).
+Refer to [Azure Monitor for containers](../topics/monitoring.md#azure-monitor-for-containers) for more details how to onboard and monitor the cluster, nodes, pods and containers inventory, performance metrics, health and logs etc.
+
 ## Unsupported Addons
 
 AKS Engine includes a number of optional [addons](../topics/clusterdefinitions.md#addons) that can be deployed as part of the cluster provisioning process.
@@ -145,7 +151,6 @@ The list below includes the addons currently unsupported on Azure Stack:
 * ACI Connector
 * Blobfuse Flex Volume
 * Cluster Autoscaler
-* Container Monitoring
 * KeyVault Flex Volume
 * NVIDIA Device Plugin
 * Rescheduler

diff --git a/docs/topics/monitoring.md b/docs/topics/monitoring.md
@@ -65,8 +65,6 @@ Azure Monitor for containers for AKS Engine cluster(s) can be configured through
 1. Helm chart [azuremonitor-containers](https://github.com/helm/charts/tree/master/incubator/azuremonitor-containers)
 2. [Container-monitoring add-on](../../examples/addons/container-monitoring/README.md)
 
-> Note: If more than one AKS Engine cluster planned to configure to the same Azure Log Analytics Workspace then recommend option is to use Helm chart (i.e. option #1 above)
-
 Navigate to [azmon-containers](https://aka.ms/azmon-containers) to view the health, metrics and logs of AKS-engine cluster(s).
 
 For more details on how to use the product, see [Azure Monitor for containers](https://docs.microsoft.com/en-us/azure/azure-monitor/insights/container-insights-overview)

diff --git a/docs/tutorials/containermonitoringaddon.md b/docs/tutorials/containermonitoringaddon.md
@@ -22,6 +22,7 @@ Container Monitoring can be onboarded either through the monitoring add-on or th
 Following are supported options to enable container-monitoring add-on during the cluster creation or post cluster creation.
 
 > Note: option 1) and 2) are supported only through `aks-engine deploy` command.
+> Note: For Azure stack cloud environments, only option 3) and 4) are supported.
 
 ### 1. Using Default Log Analytics Workspace
 
@@ -49,7 +50,7 @@ Azure Log analytics workspace can be in any Azure subscription in which you have
              }
      }
 
-Refer to [Sample Kubernetes definition file with monitoringa addon using existing log analytics workspace](../../examples/addons/container-monitoring/kubernetes-container-monitoring_existing_log_analytics_workspace.json)
+Refer to [Sample Kubernetes definition file with monitoring addon using existing log analytics workspace](../../examples/addons/container-monitoring/kubernetes-container-monitoring_existing_log_analytics_workspace.json)
 
 ### 3. Using Workspace GUID or Key
 
@@ -64,7 +65,7 @@ You can also configure with workspace GUID and Key of the existing Log analytics
             }
           }
 
-Refer to [Sample Kubernetes definition file with monitoringa addon using workspace GUID and key of the existing log analytics workspace](../../examples/addons/container-monitoring/kubernetes-container-monitoring_existing_workspace_id_and_key.json)
+Refer to [Sample Kubernetes definition file with monitoring addon using workspace GUID and key of the existing log analytics workspace](../../examples/addons/container-monitoring/kubernetes-container-monitoring_existing_workspace_id_and_key.json)
 
 ### 4. Using Azure Monitor for containers Helm chart
 
@@ -78,11 +79,36 @@ After successful onboarding, navigating to [Azure Monitor for containers](https:
 
 ## Required Roles and Permissions
 
-- User requires the reader role permission on the Azure Log Analytics workspace and AKS Engine cluster resource group to view and monitor, and analyze health of your onboarded AKS Engine cluster, pods and containers etc.
 - For onboarding monitoring addon
      -  If the existing Azure Log Analytics workspace is used, then the Log Analytics Contributor role on existing Azure Log Analytics is required
      -  For the new Azure Log Analytics workspace, user requires the contributor role on the Subscription or the Resource group where the AKS Engine cluster resources will be deployed
 
+  - User requires the reader role permission on the Azure Log Analytics workspace
+  - For Azure AKS-Engine clusters, user requires reader role permission on cluster resource group and resources under that
+
+## Supported Azure Cloud Environment(s)
+
+ -  Azure Public Cloud
+ -  Azure China Cloud
+ -  Azure US Government Cloud
+ -  Azure Stack Cloud
+
+### Disable Monitoring
+
+After you enable monitoring of your AKS Engine cluster, you can stop container monitoring on the cluster if you decide you no longer want to monitor it.
+
+- If you have onboarded the monitoring using the HELM chart, then you can disable monitoring by uninstalling the chart. Refer Uninstalling the Chart section in [azuremonitor-containers](https://github.com/helm/charts/tree/master/incubator/azuremonitor-containers)
+
+- If you have onboarded using the Container Monitoring addon, then you can remove monitoring addon with below steps
+
+      1. ssh to master node of your AKS Engine cluster master node and navigate to /etc/kubernetes/addons directory
+      2. delete all the resources related to container monitoring addon with `kubectl delete -f omsagent-daemonset.yaml` command against your AKS Engine cluster
+      3. delete the container monitoring addon manifest file omsagent-daemonset.yaml  under /etc/kubernetes/addons
+
+### Upgrade Container Monitoring Addon
+
+For upgrading the container monitoring addon, you can disable the monitoring addon as described in Disable Monitoring section and use the HELM chart to install and upgrade.
+
 ## Contact
 
 If you have any questions or feedback regarding the container monitoring addon, please reach us out through [this](mailto:askcoin@microsoft.com) email.