Automate splunk monitoring console app for indexer clustering

pkg/apis/enterprise/v1alpha3/common_types.go

@@ -66,6 +66,9 @@ type CommonSplunkSpec struct {
 
 	// ClusterMasterRef refers to a Splunk Enterprise indexer cluster managed by the operator within Kubernetes
 	ClusterMasterRef corev1.ObjectReference `json:"clusterMasterRef"`
+
+	// Mock to differentiate between UTs and actual reconcile
+	Mock bool `json:"Mock"`
 }
 
 // SmartStoreSpec defines Splunk indexes and remote storage volume configuration

pkg/splunk/client/enterprise.go

@@ -21,6 +21,8 @@ import (
 	"io/ioutil"
 	"net/http"
 	"regexp"
+	"strconv"
+	"strings"
 	"time"
 
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
@@ -66,15 +68,15 @@ func NewSplunkClient(managementURI, username, password string) *SplunkClient {
 }
 
 // Do processes a Splunk REST API request and unmarshals response into obj, if not nil.
-func (c *SplunkClient) Do(request *http.Request, expectedStatus int, obj interface{}) error {
+func (c *SplunkClient) Do(request *http.Request, expectedStatus string, obj interface{}) error {
 	// send HTTP response and check status
 	request.SetBasicAuth(c.Username, c.Password)
 	response, err := c.Client.Do(request)
 	if err != nil {
 		return err
 	}
-	if response.StatusCode != expectedStatus {
-		return fmt.Errorf("Response code=%d from %s; want %d", response.StatusCode, request.URL, expectedStatus)
+	if !strings.Contains(expectedStatus, strconv.Itoa(response.StatusCode)) {
+		return fmt.Errorf("Response code=%d from %s; want %s", response.StatusCode, request.URL, expectedStatus)
 	}
 	if obj == nil {
 		return nil
@@ -95,7 +97,7 @@ func (c *SplunkClient) Get(path string, obj interface{}) error {
 	if err != nil {
 		return err
 	}
-	return c.Do(request, 200, obj)
+	return c.Do(request, "200", obj)
 }
 
 // SearchHeadCaptainInfo represents the status of the search head cluster.
@@ -291,7 +293,7 @@ func (c *SplunkClient) SetSearchHeadDetention(detain bool) error {
 	if err != nil {
 		return err
 	}
-	return c.Do(request, 200, nil)
+	return c.Do(request, "200", nil)
 }
 
 // RemoveSearchHeadClusterMember removes a search head cluster member.
@@ -595,7 +597,7 @@ func (c *SplunkClient) RemoveIndexerClusterPeer(id string) error {
 	if err != nil {
 		return err
 	}
-	return c.Do(request, 200, nil)
+	return c.Do(request, "200", nil)
 }
 
 // DecommissionIndexerClusterPeer takes an indexer cluster peer offline using the decommission endpoint.
@@ -611,7 +613,291 @@ func (c *SplunkClient) DecommissionIndexerClusterPeer(enforceCounts bool) error
 	if err != nil {
 		return err
 	}
-	return c.Do(request, 200, nil)
+	return c.Do(request, "200", nil)
+}
+
+//MCServerRolesInfo is the struct for the server roles of the localhost, in this case SplunkMonitoringConsole
+type MCServerRolesInfo struct {
+	ServerRoles []string `json:"server_roles"`
+}
+
+//MCDistributedPeers is the struct for information about distributed peers of the monitoring console
+type MCDistributedPeers struct {
+	ClusterLabel []string `json:"cluster_label"`
+	ServerRoles  []string `json:"server_roles"`
+}
+
+//AutomateMCApplyChanges change the state of new indexers from "New" to "Configured" and add them in monitoring console asset table
+func (c *SplunkClient) AutomateMCApplyChanges(mock bool) error {
+	if mock {
+		return nil
+	}
+	var configuredPeers, indexerMemberList, licenseMasterMemberList string
+	apiResponseServerRoles, err := c.GetMonitoringconsoleServerRoles()
+	if err != nil {
+		return err
+	}
+
+	apiResponseMCDistributedPeers := struct {
+		Entry []struct {
+			Name    string             `json:"name"`
+			Content MCDistributedPeers `json:"content"`
+		} `json:"entry"`
+	}{}
+	path := "/services/search/distributed/peers"
+	err = c.Get(path, &apiResponseMCDistributedPeers)
+	if err != nil {
+		return err
+	}
+
+	for _, e := range apiResponseMCDistributedPeers.Entry {
+		if configuredPeers == "" {
+			configuredPeers = e.Name
+		} else {
+			str := []string{configuredPeers, e.Name}
+			configuredPeers = strings.Join(str, ",")
+		}
+		for _, s := range e.Content.ServerRoles {
+			if s == "indexer" {
+				indexerMemberList = indexerMemberList + "&member=" + e.Name
+			}
+			if s == "license_master" {
+				licenseMasterMemberList = licenseMasterMemberList + "&member=" + e.Name
+			}
+		}
+	}
+
+	for _, e := range apiResponseServerRoles.ServerRoles {
+		if e == "indexer" {
+			indexerMemberList = "&member=localhost:localhost" + indexerMemberList
+		}
+		if e == "license_master" {
+			licenseMasterMemberList = licenseMasterMemberList + "&member=localhost:localhost"
+		}
+	}
+	reqBodyIndexer := indexerMemberList + "&default=true"
+	reqBodyLicenseMaster := licenseMasterMemberList + "&default=false"
+	err = c.UpdateDMCGroups("dmc_group_indexer", reqBodyIndexer)
+	if err != nil {
+		return err
+	}
+	err = c.UpdateDMCGroups("dmc_group_license_master", reqBodyLicenseMaster)
+	if err != nil {
+		return err
+	}
+
+	clusterRoleDict := make(map[string][]string)
+	//map of Name to Roles
+	for _, e := range apiResponseMCDistributedPeers.Entry {
+		for _, s := range e.Content.ClusterLabel {
+			clusterRoleDict[e.Name] = append(clusterRoleDict[e.Name], s)
+		}
+	}
+	//TODO: check different labels here
+	clusterRoleDictToDict := make(map[string][]string)
+	for key, value := range clusterRoleDict {
+		for _, val := range value {
+			clusterRoleDictToDict[val] = append(clusterRoleDictToDict[val], key)
+		}
+	}
+
+	clusterRoleDictToDictString := make(map[string]string)
+	for key, value := range clusterRoleDictToDict {
+		for _, val := range value {
+			clusterRoleDictToDictString[key] = clusterRoleDictToDictString[key] + "&member=" + val
+		}
+	}
+
+	for key, value := range clusterRoleDictToDictString {
+		if key == "" {
+			break
+		} else {
+			err = c.UpdateDMCClusteringLabelGroup(key, value)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	apiResponseMCAssetTableBuild, err := c.GetMonitoringconsoleAssetTable()
+	if err != nil {
+		return err
+	}
+	err = c.PostMonitoringConsoleAssetTable(apiResponseMCAssetTableBuild)
+	if err != nil {
+		return err
+	}
+	UISettingsObject, err := c.GetMonitoringConsoleUISettings()
+	if err != nil {
+		return err
+	}
+	err = c.UpdateLookupUISettings(configuredPeers, UISettingsObject)
+	if err != nil {
+		return err
+	}
+	err = c.UpdateMonitoringConsoleApp()
+	if err != nil {
+		return err
+	}
+	return err
+}
+
+//GetMonitoringconsoleServerRoles to retrive server roles of the local host or SplunkMonitoringConsole
+func (c *SplunkClient) GetMonitoringconsoleServerRoles() (*MCServerRolesInfo, error) {
+	apiResponseServerRoles := struct {
+		Entry []struct {
+			Content MCServerRolesInfo `json:"content"`
+		} `json:"entry"`
+	}{}
+	path := "/services/server/info/server-info"
+	err := c.Get(path, &apiResponseServerRoles)
+	if err != nil {
+		return nil, err
+	}
+	if len(apiResponseServerRoles.Entry) < 1 {
+		return nil, err
+	}
+	return &apiResponseServerRoles.Entry[0].Content, nil
+}
+
+//UpdateDMCGroups dmc* groups with new members
+func (c *SplunkClient) UpdateDMCGroups(dmcGroupName string, groupMembers string) error {
+	endpoint := fmt.Sprintf("%s/services/search/distributed/groups/%s/edit", c.ManagementURI, dmcGroupName)
+	request, err := http.NewRequest("POST", endpoint, strings.NewReader(groupMembers))
+	err = c.Do(request, "200, 201, 409", nil)
+	if err != nil {
+		return err
+	}
+	return err
+}
+
+//UpdateDMCClusteringLabelGroup update respective clustering group
+func (c *SplunkClient) UpdateDMCClusteringLabelGroup(groupName string, groupMembers string) error {
+	endpoint := fmt.Sprintf("%s/services/search/distributed/groups/dmc_indexerclustergroup_%s/edit", c.ManagementURI, groupName)
+	reqBodyClusterGroup := groupMembers + "&default=false"
+	request, err := http.NewRequest("POST", endpoint, strings.NewReader(reqBodyClusterGroup))
+	err = c.Do(request, "200, 201, 409", nil)
+	if err != nil {
+		return err
+	}
+	return err
+}
+
+//MCAssetBuildTable is the struct for information about asset table
+type MCAssetBuildTable struct {
+	DispatchAutoCancel string `json:"dispatch.auto_cancel"`
+	DispatchBuckets    int64  `json:"dispatch.buckets"`
+}
+
+//GetMonitoringconsoleAssetTable to GET monitoring console asset table data.
+func (c *SplunkClient) GetMonitoringconsoleAssetTable() (*MCAssetBuildTable, error) {
+	apiResponseMCAssetTableBuild := struct {
+		Entry []struct {
+			Content MCAssetBuildTable `json:"content"`
+		} `json:"entry"`
+	}{}
+	path := "/servicesNS/nobody/splunk_monitoring_console/saved/searches/DMC%20Asset%20-%20Build%20Full"
+	err := c.Get(path, &apiResponseMCAssetTableBuild)
+	if err != nil {
+		return nil, err
+	}
+	if len(apiResponseMCAssetTableBuild.Entry) < 1 {
+		return nil, err
+	}
+	return &apiResponseMCAssetTableBuild.Entry[0].Content, nil
+}
+
+//PostMonitoringConsoleAssetTable to build monitoring console asset table. Kicks off the search [Build Asset Table full]
+func (c *SplunkClient) PostMonitoringConsoleAssetTable(apiResponseMCAssetTableBuild *MCAssetBuildTable) error {
+	reqBodyAssetTable := "&trigger_actions=true&dispatch.auto_cancel=" + apiResponseMCAssetTableBuild.DispatchAutoCancel + "&dispatch.buckets=" + strconv.FormatInt(apiResponseMCAssetTableBuild.DispatchBuckets, 10) + "&dispatch.enablePreview=true"
+	endpoint := fmt.Sprintf("%s", c.ManagementURI) + "/servicesNS/nobody/splunk_monitoring_console/saved/searches/DMC%20Asset%20-%20Build%20Full/dispatch"
+	request, err := http.NewRequest("POST", endpoint, strings.NewReader(reqBodyAssetTable))
+	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	err = c.Do(request, "201", nil)
+	if err != nil {
+		return err
+	}
+	return err
+}
+
+//UISettings is the struct for storing monitoring console app UI settings
+type UISettings struct {
+	EaiData     string `json:"eai:data"`
+	Disabled    bool   `json:"disabled"`
+	EaiACL      string `json:"eai:acl"`
+	EaiAppName  string `json:"eai:appName"`
+	EaiUserName string `json:"eai:userName"`
+}
+
+//GetMonitoringConsoleUISettings do a Get for app UI settings
+func (c *SplunkClient) GetMonitoringConsoleUISettings() (*UISettings, error) {
+	apiResponseUISettings := struct {
+		Entry []struct {
+			Content UISettings `json:"content"`
+		} `json:"entry"`
+	}{}
+	path := "/servicesNS/nobody/splunk_monitoring_console/data/ui/nav/default.distributed"
+	err := c.Get(path, &apiResponseUISettings)
+	if err != nil {
+		return nil, err
+	}
+	if len(apiResponseUISettings.Entry) < 1 {
+		return nil, err
+	}
+	return &apiResponseUISettings.Entry[0].Content, nil
+}
+
+//UpdateLookupUISettings updates assets.csv
+func (c *SplunkClient) UpdateLookupUISettings(configuredPeers string, apiResponseUISettings *UISettings) error {
+	reqBodyMCLookups := "configuredPeers=" + configuredPeers + "&eai:appName=" + apiResponseUISettings.EaiAppName + "&eai:acl=" + apiResponseUISettings.EaiACL + "&eai:userName=" + apiResponseUISettings.EaiUserName + "&disabled=" + strconv.FormatBool(apiResponseUISettings.Disabled)
+	endpoint := fmt.Sprintf("%s/servicesNS/nobody/splunk_monitoring_console/configs/conf-splunk_monitoring_console_assets/settings", c.ManagementURI)
+	request, err := http.NewRequest("POST", endpoint, strings.NewReader(reqBodyMCLookups))
+	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	err = c.Do(request, "200", nil)
+	if err != nil {
+		return err
+	}
+	return err
+}
+
+//UpdateMonitoringConsoleApp updates the monitoring console app
+func (c *SplunkClient) UpdateMonitoringConsoleApp() error {
+	endpoint := fmt.Sprintf("%s/servicesNS/nobody/system/apps/local/splunk_monitoring_console", c.ManagementURI)
+	request, err := http.NewRequest("POST", endpoint, nil)
+	if err != nil {
+		return err
+	}
+	err = c.Do(request, "200", nil)
+	if err != nil {
+		return err
+	}
+	return err
+}
+
+//ClusterInfo is the struct for checking ClusterInfo
+type ClusterInfo struct {
+	MultiSite string `json:"multisite"`
+}
+
+// GetClusterInfo queries the cluster about multi-site or single-site.
+//See https://docs.splunk.com/Documentation/Splunk/8.0.6/RESTREF/RESTcluster#cluster.2Fconfig
+func (c *SplunkClient) GetClusterInfo(mockCall bool) (*ClusterInfo, error) {
+	if mockCall {
+		return nil, nil
+	}
+	apiResponse := struct {
+		Entry []struct {
+			Content ClusterInfo `json:"content"`
+		} `json:"entry"`
+	}{}
+	path := "/services/cluster/config"
+	err := c.Get(path, &apiResponse)
+	if err != nil {
+		return nil, err
+	}
+	if len(apiResponse.Entry) < 1 {
+		return nil, fmt.Errorf("Invalid response from %s%s", c.ManagementURI, path)
+	}
+	return &apiResponse.Entry[0].Content, nil
 }
 
 // SetIdxcSecret sets idxc_secret for a Splunk Instance
@@ -625,7 +911,7 @@ func (c *SplunkClient) SetIdxcSecret(idxcSecret string) error {
 	}
 	request.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 
-	return c.Do(request, 200, nil)
+	return c.Do(request, "200", nil)
 }
 
 // RestartSplunk restarts specific Splunk instance
@@ -637,5 +923,5 @@ func (c *SplunkClient) RestartSplunk() error {
 	if err != nil {
 		return err
 	}
-	return c.Do(request, 200, nil)
+	return c.Do(request, "200", nil)
 }