diff --git a/build/Dockerfile b/build/Dockerfile index 38f42143c..c96f5163d 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -16,10 +16,10 @@ LABEL name="splunk" \ COPY build/_output/bin/splunk-operator ${OPERATOR} COPY build/bin /usr/local/bin -RUN mkdir /licenses \ - && curl -o /licenses/apache-2.0.txt https://www.apache.org/licenses/LICENSE-2.0.txt \ - && curl -o /licenses/EULA_Red_Hat_Universal_Base_Image_English_20190422.pdf https://www.redhat.com/licenses/EULA_Red_Hat_Universal_Base_Image_English_20190422.pdf \ - && /usr/local/bin/user_setup +RUN mkdir /licenses && /usr/local/bin/user_setup + +COPY build/EULA_Red_Hat_Universal_Base_Image_English_20190422.pdf /licenses +COPY LICENSE /licenses/LICENSE-2.0.txt ENTRYPOINT ["/usr/local/bin/entrypoint"] diff --git a/build/EULA_Red_Hat_Universal_Base_Image_English_20190422.pdf b/build/EULA_Red_Hat_Universal_Base_Image_English_20190422.pdf new file mode 100644 index 000000000..3a32abd75 Binary files /dev/null and b/build/EULA_Red_Hat_Universal_Base_Image_English_20190422.pdf differ diff --git a/deploy/all-in-one-cluster.yaml b/deploy/all-in-one-cluster.yaml index d9243af45..f3b7587c3 100644 --- a/deploy/all-in-one-cluster.yaml +++ b/deploy/all-in-one-cluster.yaml @@ -6705,18 +6705,25 @@ spec: description: SearchHeadClusterMemberStatus is used to track the status of each search head cluster member properties: - activeSearches: - description: total number of active historical + realtime searches + active_historical_search_count: + description: Number of currently running historical searches. type: integer + active_realtime_search_count: + description: Number of currently running realtime searches. + type: integer + adhoc_searchhead: + description: Flag that indicates if this member can run scheduled + searches. + type: boolean + is_registered: + description: Indicates if this member is registered with the searchhead + cluster captain. + type: boolean name: description: Name of the search head cluster member type: string - registered: - description: true if this member is registered with the search - head captain - type: boolean status: - description: Status of the search head cluster member + description: Indicates the status of the member. type: string type: object type: array @@ -8956,6 +8963,45 @@ spec: - Terminating - Error type: string + indexing_ready_flag: + description: Indicates if the cluster is ready for indexing. + type: boolean + initialized_flag: + description: Indicates if the cluster is initialized. + type: boolean + maintenance_mode: + description: Indicates if the cluster is in maintenance mode. + type: boolean + peers: + description: status of each indexer cluster peer + items: + description: IndexerClusterMemberStatus is used to track the status + of each indexer cluster peer. + properties: + active_bundle_id: + description: The ID of the configuration bundle currently being + used by the master. + type: string + bucket_count: + description: Count of the number of buckets on this peer, across + all indexes. + format: int64 + type: integer + guid: + description: Unique identifier or GUID for the peer + type: string + is_searchable: + description: Flag indicating if this peer belongs to the current + committed generation and is searchable. + type: boolean + name: + description: Name of the indexer cluster peer + type: string + status: + description: Status of the indexer cluster peer + type: string + type: object + type: array phase: description: current phase of the indexer cluster enum: @@ -8978,6 +9024,10 @@ spec: selector: description: selector for pods, used by HorizontalPodAutoscaler type: string + service_ready_flag: + description: Indicates whether the master is ready to begin servicing, + based on whether it is initialized. + type: boolean type: object type: object version: v1alpha2 diff --git a/deploy/all-in-one-scoped.yaml b/deploy/all-in-one-scoped.yaml index 6766feb43..9dbe0c264 100644 --- a/deploy/all-in-one-scoped.yaml +++ b/deploy/all-in-one-scoped.yaml @@ -6705,18 +6705,25 @@ spec: description: SearchHeadClusterMemberStatus is used to track the status of each search head cluster member properties: - activeSearches: - description: total number of active historical + realtime searches + active_historical_search_count: + description: Number of currently running historical searches. type: integer + active_realtime_search_count: + description: Number of currently running realtime searches. + type: integer + adhoc_searchhead: + description: Flag that indicates if this member can run scheduled + searches. + type: boolean + is_registered: + description: Indicates if this member is registered with the searchhead + cluster captain. + type: boolean name: description: Name of the search head cluster member type: string - registered: - description: true if this member is registered with the search - head captain - type: boolean status: - description: Status of the search head cluster member + description: Indicates the status of the member. type: string type: object type: array @@ -8956,6 +8963,45 @@ spec: - Terminating - Error type: string + indexing_ready_flag: + description: Indicates if the cluster is ready for indexing. + type: boolean + initialized_flag: + description: Indicates if the cluster is initialized. + type: boolean + maintenance_mode: + description: Indicates if the cluster is in maintenance mode. + type: boolean + peers: + description: status of each indexer cluster peer + items: + description: IndexerClusterMemberStatus is used to track the status + of each indexer cluster peer. + properties: + active_bundle_id: + description: The ID of the configuration bundle currently being + used by the master. + type: string + bucket_count: + description: Count of the number of buckets on this peer, across + all indexes. + format: int64 + type: integer + guid: + description: Unique identifier or GUID for the peer + type: string + is_searchable: + description: Flag indicating if this peer belongs to the current + committed generation and is searchable. + type: boolean + name: + description: Name of the indexer cluster peer + type: string + status: + description: Status of the indexer cluster peer + type: string + type: object + type: array phase: description: current phase of the indexer cluster enum: @@ -8978,6 +9024,10 @@ spec: selector: description: selector for pods, used by HorizontalPodAutoscaler type: string + service_ready_flag: + description: Indicates whether the master is ready to begin servicing, + based on whether it is initialized. + type: boolean type: object type: object version: v1alpha2 diff --git a/deploy/crds/combined.yaml b/deploy/crds/combined.yaml index 0c3660f49..5e59c5eb0 100644 --- a/deploy/crds/combined.yaml +++ b/deploy/crds/combined.yaml @@ -6705,18 +6705,25 @@ spec: description: SearchHeadClusterMemberStatus is used to track the status of each search head cluster member properties: - activeSearches: - description: total number of active historical + realtime searches + active_historical_search_count: + description: Number of currently running historical searches. type: integer + active_realtime_search_count: + description: Number of currently running realtime searches. + type: integer + adhoc_searchhead: + description: Flag that indicates if this member can run scheduled + searches. + type: boolean + is_registered: + description: Indicates if this member is registered with the searchhead + cluster captain. + type: boolean name: description: Name of the search head cluster member type: string - registered: - description: true if this member is registered with the search - head captain - type: boolean status: - description: Status of the search head cluster member + description: Indicates the status of the member. type: string type: object type: array @@ -8956,6 +8963,45 @@ spec: - Terminating - Error type: string + indexing_ready_flag: + description: Indicates if the cluster is ready for indexing. + type: boolean + initialized_flag: + description: Indicates if the cluster is initialized. + type: boolean + maintenance_mode: + description: Indicates if the cluster is in maintenance mode. + type: boolean + peers: + description: status of each indexer cluster peer + items: + description: IndexerClusterMemberStatus is used to track the status + of each indexer cluster peer. + properties: + active_bundle_id: + description: The ID of the configuration bundle currently being + used by the master. + type: string + bucket_count: + description: Count of the number of buckets on this peer, across + all indexes. + format: int64 + type: integer + guid: + description: Unique identifier or GUID for the peer + type: string + is_searchable: + description: Flag indicating if this peer belongs to the current + committed generation and is searchable. + type: boolean + name: + description: Name of the indexer cluster peer + type: string + status: + description: Status of the indexer cluster peer + type: string + type: object + type: array phase: description: current phase of the indexer cluster enum: @@ -8978,6 +9024,10 @@ spec: selector: description: selector for pods, used by HorizontalPodAutoscaler type: string + service_ready_flag: + description: Indicates whether the master is ready to begin servicing, + based on whether it is initialized. + type: boolean type: object type: object version: v1alpha2 diff --git a/deploy/crds/enterprise.splunk.com_indexerclusters_crd.yaml b/deploy/crds/enterprise.splunk.com_indexerclusters_crd.yaml index 06571cbdc..d0c6cb144 100644 --- a/deploy/crds/enterprise.splunk.com_indexerclusters_crd.yaml +++ b/deploy/crds/enterprise.splunk.com_indexerclusters_crd.yaml @@ -2200,6 +2200,45 @@ spec: - Terminating - Error type: string + indexing_ready_flag: + description: Indicates if the cluster is ready for indexing. + type: boolean + initialized_flag: + description: Indicates if the cluster is initialized. + type: boolean + maintenance_mode: + description: Indicates if the cluster is in maintenance mode. + type: boolean + peers: + description: status of each indexer cluster peer + items: + description: IndexerClusterMemberStatus is used to track the status + of each indexer cluster peer. + properties: + active_bundle_id: + description: The ID of the configuration bundle currently being + used by the master. + type: string + bucket_count: + description: Count of the number of buckets on this peer, across + all indexes. + format: int64 + type: integer + guid: + description: Unique identifier or GUID for the peer + type: string + is_searchable: + description: Flag indicating if this peer belongs to the current + committed generation and is searchable. + type: boolean + name: + description: Name of the indexer cluster peer + type: string + status: + description: Status of the indexer cluster peer + type: string + type: object + type: array phase: description: current phase of the indexer cluster enum: @@ -2222,6 +2261,10 @@ spec: selector: description: selector for pods, used by HorizontalPodAutoscaler type: string + service_ready_flag: + description: Indicates whether the master is ready to begin servicing, + based on whether it is initialized. + type: boolean type: object type: object version: v1alpha2 diff --git a/deploy/crds/enterprise.splunk.com_searchheadclusters_crd.yaml b/deploy/crds/enterprise.splunk.com_searchheadclusters_crd.yaml index 3e68f21cc..b1f7c6e3e 100644 --- a/deploy/crds/enterprise.splunk.com_searchheadclusters_crd.yaml +++ b/deploy/crds/enterprise.splunk.com_searchheadclusters_crd.yaml @@ -2261,18 +2261,25 @@ spec: description: SearchHeadClusterMemberStatus is used to track the status of each search head cluster member properties: - activeSearches: - description: total number of active historical + realtime searches + active_historical_search_count: + description: Number of currently running historical searches. type: integer + active_realtime_search_count: + description: Number of currently running realtime searches. + type: integer + adhoc_searchhead: + description: Flag that indicates if this member can run scheduled + searches. + type: boolean + is_registered: + description: Indicates if this member is registered with the searchhead + cluster captain. + type: boolean name: description: Name of the search head cluster member type: string - registered: - description: true if this member is registered with the search - head captain - type: boolean status: - description: Status of the search head cluster member + description: Indicates the status of the member. type: string type: object type: array diff --git a/pkg/apis/enterprise/v1alpha2/indexercluster_types.go b/pkg/apis/enterprise/v1alpha2/indexercluster_types.go index c9b5a97f3..514120f25 100644 --- a/pkg/apis/enterprise/v1alpha2/indexercluster_types.go +++ b/pkg/apis/enterprise/v1alpha2/indexercluster_types.go @@ -34,6 +34,27 @@ type IndexerClusterSpec struct { Replicas int32 `json:"replicas"` } +// IndexerClusterMemberStatus is used to track the status of each indexer cluster peer. +type IndexerClusterMemberStatus struct { + // Unique identifier or GUID for the peer + ID string `json:"guid"` + + // Name of the indexer cluster peer + Name string `json:"name"` + + // Status of the indexer cluster peer + Status string `json:"status"` + + // The ID of the configuration bundle currently being used by the master. + ActiveBundleID string `json:"active_bundle_id"` + + // Count of the number of buckets on this peer, across all indexes. + BucketCount int64 `json:"bucket_count"` + + // Flag indicating if this peer belongs to the current committed generation and is searchable. + Searchable bool `json:"is_searchable"` +} + // IndexerClusterStatus defines the observed state of a Splunk Enterprise indexer cluster type IndexerClusterStatus struct { // current phase of the indexer cluster @@ -50,6 +71,21 @@ type IndexerClusterStatus struct { // selector for pods, used by HorizontalPodAutoscaler Selector string `json:"selector"` + + // Indicates if the cluster is initialized. + Initialized bool `json:"initialized_flag"` + + // Indicates if the cluster is ready for indexing. + IndexingReady bool `json:"indexing_ready_flag"` + + // Indicates whether the master is ready to begin servicing, based on whether it is initialized. + ServiceReady bool `json:"service_ready_flag"` + + // Indicates if the cluster is in maintenance mode. + MaintenanceMode bool `json:"maintenance_mode"` + + // status of each indexer cluster peer + Peers []IndexerClusterMemberStatus `json:"peers"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/apis/enterprise/v1alpha2/searchheadcluster_types.go b/pkg/apis/enterprise/v1alpha2/searchheadcluster_types.go index e67df0584..8e71a150c 100644 --- a/pkg/apis/enterprise/v1alpha2/searchheadcluster_types.go +++ b/pkg/apis/enterprise/v1alpha2/searchheadcluster_types.go @@ -47,14 +47,20 @@ type SearchHeadClusterMemberStatus struct { // Name of the search head cluster member Name string `json:"name"` - // Status of the search head cluster member + // Indicates the status of the member. Status string `json:"status"` - // true if this member is registered with the search head captain - Registered bool `json:"registered"` + // Flag that indicates if this member can run scheduled searches. + Adhoc bool `json:"adhoc_searchhead"` - // total number of active historical + realtime searches - ActiveSearches int `json:"activeSearches"` + // Indicates if this member is registered with the searchhead cluster captain. + Registered bool `json:"is_registered"` + + // Number of currently running historical searches. + ActiveHistoricalSearchCount int `json:"active_historical_search_count"` + + // Number of currently running realtime searches. + ActiveRealtimeSearchCount int `json:"active_realtime_search_count"` } // SearchHeadClusterStatus defines the observed state of a Splunk Enterprise search head cluster diff --git a/pkg/apis/enterprise/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/enterprise/v1alpha2/zz_generated.deepcopy.go index 3660d2a24..5f89f9b5f 100644 --- a/pkg/apis/enterprise/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/enterprise/v1alpha2/zz_generated.deepcopy.go @@ -60,7 +60,7 @@ func (in *IndexerCluster) DeepCopyInto(out *IndexerCluster) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) - out.Status = in.Status + in.Status.DeepCopyInto(&out.Status) return } @@ -115,6 +115,22 @@ func (in *IndexerClusterList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *IndexerClusterMemberStatus) DeepCopyInto(out *IndexerClusterMemberStatus) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IndexerClusterMemberStatus. +func (in *IndexerClusterMemberStatus) DeepCopy() *IndexerClusterMemberStatus { + if in == nil { + return nil + } + out := new(IndexerClusterMemberStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IndexerClusterSpec) DeepCopyInto(out *IndexerClusterSpec) { *out = *in @@ -135,6 +151,11 @@ func (in *IndexerClusterSpec) DeepCopy() *IndexerClusterSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IndexerClusterStatus) DeepCopyInto(out *IndexerClusterStatus) { *out = *in + if in.Peers != nil { + in, out := &in.Peers, &out.Peers + *out = make([]IndexerClusterMemberStatus, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/controller/indexercluster/indexercluster_controller.go b/pkg/controller/indexercluster/indexercluster_controller.go index 935066884..f0b66ba6a 100644 --- a/pkg/controller/indexercluster/indexercluster_controller.go +++ b/pkg/controller/indexercluster/indexercluster_controller.go @@ -115,7 +115,7 @@ func (r *ReconcileIndexerCluster) Reconcile(request reconcile.Request) (reconcil instance.TypeMeta.APIVersion = "enterprise.splunk.com/v1alpha2" instance.TypeMeta.Kind = "IndexerCluster" - result, err := splunkreconcile.ReconcileIndexerCluster(r.client, instance) + result, err := splunkreconcile.ApplyIndexerCluster(r.client, instance) if err != nil { reqLogger.Error(err, "IndexerCluster reconciliation requeued", "RequeueAfter", result.RequeueAfter) return result, nil diff --git a/pkg/controller/licensemaster/licensemaster_controller.go b/pkg/controller/licensemaster/licensemaster_controller.go index 30894c176..9b640cd45 100644 --- a/pkg/controller/licensemaster/licensemaster_controller.go +++ b/pkg/controller/licensemaster/licensemaster_controller.go @@ -115,7 +115,7 @@ func (r *ReconcileLicenseMaster) Reconcile(request reconcile.Request) (reconcile instance.TypeMeta.APIVersion = "enterprise.splunk.com/v1alpha2" instance.TypeMeta.Kind = "LicenseMaster" - result, err := splunkreconcile.ReconcileLicenseMaster(r.client, instance) + result, err := splunkreconcile.ApplyLicenseMaster(r.client, instance) if err != nil { reqLogger.Error(err, "LicenseMaster reconciliation requeued", "RequeueAfter", result.RequeueAfter) return result, nil diff --git a/pkg/controller/searchheadcluster/searchheadcluster_controller.go b/pkg/controller/searchheadcluster/searchheadcluster_controller.go index b5768d446..09f0738ff 100644 --- a/pkg/controller/searchheadcluster/searchheadcluster_controller.go +++ b/pkg/controller/searchheadcluster/searchheadcluster_controller.go @@ -115,7 +115,7 @@ func (r *ReconcileSearchHeadCluster) Reconcile(request reconcile.Request) (recon instance.TypeMeta.APIVersion = "enterprise.splunk.com/v1alpha2" instance.TypeMeta.Kind = "SearchHeadCluster" - result, err := splunkreconcile.ReconcileSearchHeadCluster(r.client, instance) + result, err := splunkreconcile.ApplySearchHeadCluster(r.client, instance) if err != nil { reqLogger.Error(err, "SearchHeadCluster reconciliation requeued", "RequeueAfter", result.RequeueAfter) return result, nil diff --git a/pkg/controller/spark/spark_controller.go b/pkg/controller/spark/spark_controller.go index 8f682b142..daecd624e 100644 --- a/pkg/controller/spark/spark_controller.go +++ b/pkg/controller/spark/spark_controller.go @@ -115,7 +115,7 @@ func (r *ReconcileSpark) Reconcile(request reconcile.Request) (reconcile.Result, instance.TypeMeta.APIVersion = "enterprise.splunk.com/v1alpha2" instance.TypeMeta.Kind = "Spark" - result, err := splunkreconcile.ReconcileSpark(r.client, instance) + result, err := splunkreconcile.ApplySpark(r.client, instance) if err != nil { reqLogger.Error(err, "Spark reconciliation requeued", "RequeueAfter", result.RequeueAfter) return result, nil diff --git a/pkg/controller/standalone/standalone_controller.go b/pkg/controller/standalone/standalone_controller.go index 33e3d0460..f5bd11f86 100644 --- a/pkg/controller/standalone/standalone_controller.go +++ b/pkg/controller/standalone/standalone_controller.go @@ -115,7 +115,7 @@ func (r *ReconcileStandalone) Reconcile(request reconcile.Request) (reconcile.Re instance.TypeMeta.APIVersion = "enterprise.splunk.com/v1alpha2" instance.TypeMeta.Kind = "Standalone" - result, err := splunkreconcile.ReconcileStandalone(r.client, instance) + result, err := splunkreconcile.ApplyStandalone(r.client, instance) if err != nil { reqLogger.Error(err, "Standalone reconciliation requeued", "RequeueAfter", result.RequeueAfter) return result, nil diff --git a/pkg/splunk/enterprise/restapi.go b/pkg/splunk/enterprise/restapi.go index 9135baa2d..c1ce40f60 100644 --- a/pkg/splunk/enterprise/restapi.go +++ b/pkg/splunk/enterprise/restapi.go @@ -45,7 +45,7 @@ type SplunkClient struct { client SplunkHTTPClient } -// NewSplunkClient returns a new SplunkClient object initialized with a username and password +// NewSplunkClient returns a new SplunkClient object initialized with a username and password. func NewSplunkClient(managementURI, username, password string) *SplunkClient { return &SplunkClient{ managementURI: managementURI, @@ -60,7 +60,7 @@ func NewSplunkClient(managementURI, username, password string) *SplunkClient { } } -// Do processes a Splunk REST API request and unmarshals response into obj, if not nil +// Do processes a Splunk REST API request and unmarshals response into obj, if not nil. func (c *SplunkClient) Do(request *http.Request, expectedStatus int, obj interface{}) error { // send HTTP response and check status request.SetBasicAuth(c.username, c.password) @@ -83,7 +83,7 @@ func (c *SplunkClient) Do(request *http.Request, expectedStatus int, obj interfa return json.Unmarshal(data, obj) } -// Get sends a REST API request and unmarshals response into obj, if not nil +// Get sends a REST API request and unmarshals response into obj, if not nil. func (c *SplunkClient) Get(path string, obj interface{}) error { endpoint := fmt.Sprintf("%s%s?count=0&output_mode=json", c.managementURI, path) request, err := http.NewRequest("GET", endpoint, nil) @@ -93,7 +93,7 @@ func (c *SplunkClient) Get(path string, obj interface{}) error { return c.Do(request, 200, obj) } -// SearchHeadCaptainInfo represents the status of the search head cluster +// SearchHeadCaptainInfo represents the status of the search head cluster. // See https://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTcluster#shcluster.2Fcaptain.2Finfo type SearchHeadCaptainInfo struct { // Id of this SH cluster. This is used as the unique identifier for the Search Head Cluster in bundle replication and acceleration summary management. @@ -103,7 +103,7 @@ type SearchHeadCaptainInfo struct { ElectedCaptain int64 `json:"elected_captain"` // Indicates if the searchhead cluster is initialized. - InitializedFlag bool `json:"initialized_flag"` + Initialized bool `json:"initialized_flag"` // The name for the captain. Displayed on the Splunk Web manager page. Label string `json:"label"` @@ -112,22 +112,23 @@ type SearchHeadCaptainInfo struct { MaintenanceMode bool `json:"maintenance_mode"` // Flag to indicate if more then replication_factor peers have joined the cluster. - MinPeersJoinedFlag bool `json:"min_peers_joined_flag"` + MinPeersJoined bool `json:"min_peers_joined_flag"` // URI of the current captain. PeerSchemeHostPort string `json:"peer_scheme_host_port"` // Indicates whether the captain is restarting the members in a searchhead cluster. - RollingRestartFlag bool `json:"rolling_restart_flag"` + RollingRestart bool `json:"rolling_restart_flag"` // Indicates whether the captain is ready to begin servicing, based on whether it is initialized. - ServiceReadyFlag bool `json:"service_ready_flag"` + ServiceReady bool `json:"service_ready_flag"` // Timestamp corresponding to the creation of the captain. StartTime int64 `json:"start_time"` } -// GetSearchHeadCaptainInfo queries the captain for info about the search head cluster +// GetSearchHeadCaptainInfo queries the captain for info about the search head cluster. +// You can use this on any member of a search head cluster. // See https://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTcluster#shcluster.2Fcaptain.2Finfo func (c *SplunkClient) GetSearchHeadCaptainInfo() (*SearchHeadCaptainInfo, error) { apiResponse := struct { @@ -146,7 +147,7 @@ func (c *SplunkClient) GetSearchHeadCaptainInfo() (*SearchHeadCaptainInfo, error return &apiResponse.Entry[0].Content, nil } -// SearchHeadCaptainMemberInfo represents the status of a search head cluster member (captain endpoint) +// SearchHeadCaptainMemberInfo represents the status of a search head cluster member (captain endpoint). // See https://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTcluster#shcluster.2Fcaptain.2Fmembers type SearchHeadCaptainMemberInfo struct { // Flag that indicates if this member can run scheduled searches. @@ -195,10 +196,10 @@ type SearchHeadCaptainMemberInfo struct { Status string `json:"status"` } -// GetSearchHeadCaptainMembers queries the search head captain for info about cluster members +// GetSearchHeadCaptainMembers queries the search head captain for info about cluster members. +// You can only use this on a search head cluster captain. +// See https://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTcluster#shcluster.2Fcaptain.2Fmembers func (c *SplunkClient) GetSearchHeadCaptainMembers() (map[string]SearchHeadCaptainMemberInfo, error) { - // query and parse - // See https://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTcluster#shcluster.2Fcaptain.2Fmembers apiResponse := struct { Entry []struct { Content SearchHeadCaptainMemberInfo `json:"content"` @@ -218,8 +219,8 @@ func (c *SplunkClient) GetSearchHeadCaptainMembers() (map[string]SearchHeadCapta return members, nil } -// SearchHeadClusterMemberInfo represents the status of a search head cluster member -// and https://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTcluster#shcluster.2Fmember.2Finfo +// SearchHeadClusterMemberInfo represents the status of a search head cluster member. +// See https://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTcluster#shcluster.2Fmember.2Finfo type SearchHeadClusterMemberInfo struct { // Number of currently running historical searches. ActiveHistoricalSearchCount int `json:"active_historical_search_count"` @@ -252,7 +253,8 @@ type SearchHeadClusterMemberInfo struct { Status string `json:"status"` } -// GetSearchHeadClusterMemberInfo queries info from a search head cluster member using /shcluster/member/info +// GetSearchHeadClusterMemberInfo queries info from a search head cluster member. +// You can use this on any member of a search head cluster. // See https://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTcluster#shcluster.2Fmember.2Finfo func (c *SplunkClient) GetSearchHeadClusterMemberInfo() (*SearchHeadClusterMemberInfo, error) { apiResponse := struct { @@ -271,7 +273,8 @@ func (c *SplunkClient) GetSearchHeadClusterMemberInfo() (*SearchHeadClusterMembe return &apiResponse.Entry[0].Content, nil } -// SetSearchHeadDetention enables or disables detention of a search head cluster member +// SetSearchHeadDetention enables or disables detention of a search head cluster member. +// You can use this on any member of a search head cluster. // See https://docs.splunk.com/Documentation/Splunk/latest/DistSearch/SHdetention func (c *SplunkClient) SetSearchHeadDetention(detain bool) error { mode := "off" @@ -286,7 +289,8 @@ func (c *SplunkClient) SetSearchHeadDetention(detain bool) error { return c.Do(request, 200, nil) } -// RemoveSearchHeadClusterMember removes a search head cluster member +// RemoveSearchHeadClusterMember removes a search head cluster member. +// You can use this on any member of a search head cluster. // See https://docs.splunk.com/Documentation/Splunk/latest/DistSearch/Removeaclustermember func (c *SplunkClient) RemoveSearchHeadClusterMember() error { // sent request to remove from search head cluster consensus @@ -337,3 +341,270 @@ func (c *SplunkClient) RemoveSearchHeadClusterMember() error { return fmt.Errorf("Received unrecognized 503 response from %s", request.URL) } + +// ClusterBundleInfo represents the status of a configuration bundle. +type ClusterBundleInfo struct { + // BundlePath is filesystem path to the file represending the bundle + BundlePath string `json:"bundle_path"` + + // Checksum used to verify bundle integrity + Checksum string `json:"checksum"` + + // Timestamp of the bundle + Timestamp int64 `json:"timestamp"` +} + +// ClusterMasterInfo represents the status of the indexer cluster master. +// See https://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTcluster#cluster.2Fmaster.2Finfo +type ClusterMasterInfo struct { + // Indicates if the cluster is initialized. + Initialized bool `json:"initialized_flag"` + + // Indicates if the cluster is ready for indexing. + IndexingReady bool `json:"indexing_ready_flag"` + + // Indicates whether the master is ready to begin servicing, based on whether it is initialized. + ServiceReady bool `json:"service_ready_flag"` + + // Indicates if the cluster is in maintenance mode. + MaintenanceMode bool `json:"maintenance_mode"` + + // Indicates whether the master is restarting the peers in a cluster. + RollingRestart bool `json:"rolling_restart_flag"` + + // The name for the master. Displayed in the Splunk Web manager page. + Label string `json:"label"` + + // Provides information about the active bundle for this master. + ActiveBundle ClusterBundleInfo `json:"active_bundle"` + + // The most recent information reflecting any changes made to the master-apps configuration bundle. + // In steady state, this is equal to active_bundle. If it is not equal, then pushing the latest bundle to all peers is in process (or needs to be started). + LatestBundle ClusterBundleInfo `json:"latest_bundle"` + + // Timestamp corresponding to the creation of the master. + StartTime int64 `json:"start_time"` +} + +// GetClusterMasterInfo queries the cluster master for info about the indexer cluster. +// You can only use this on a cluster master. +// See https://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTcluster#cluster.2Fmaster.2Finfo +func (c *SplunkClient) GetClusterMasterInfo() (*ClusterMasterInfo, error) { + apiResponse := struct { + Entry []struct { + Content ClusterMasterInfo `json:"content"` + } `json:"entry"` + }{} + path := "/services/cluster/master/info" + err := c.Get(path, &apiResponse) + if err != nil { + return nil, err + } + if len(apiResponse.Entry) < 1 { + return nil, fmt.Errorf("Invalid response from %s%s", c.managementURI, path) + } + return &apiResponse.Entry[0].Content, nil +} + +// IndexerClusterPeerInfo represents the status of a indexer cluster peer. +// See https://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTcluster#cluster.2Fslave.2Finfo +type IndexerClusterPeerInfo struct { + // Current bundle being used by this peer. + ActiveBundle ClusterBundleInfo `json:"active_bundle"` + + // Lists information about the most recent bundle downloaded from the master. + LatestBundle ClusterBundleInfo `json:"latest_bundle"` + + // The initial bundle generation ID recognized by this peer. Any searches from previous generations fail. + // The initial bundle generation ID is created when a peer first comes online, restarts, or recontacts the master. + // Note that this is reported as a very large number (18446744073709552000) that breaks Go's JSON library, while the peer is being decommissioned. + //BaseGenerationID uint64 `json:"base_generation_id"` + + // Indicates if this peer is registered with the master in the cluster. + Registered bool `json:"is_registered"` + + // Timestamp for the last attempt to contact the master. + LastHeartbeatAttempt int64 `json:"last_heartbeat_attempt"` + + // Indicates whether the peer needs to be restarted to enable its cluster configuration. + RestartState string `json:"restart_state"` + + // Indicates the status of the peer. + Status string `json:"status"` +} + +// GetIndexerClusterPeerInfo queries info from a indexer cluster peer. +// You can use this on any peer in an indexer cluster. +// See https://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTcluster#cluster.2Fslave.2Finfo +func (c *SplunkClient) GetIndexerClusterPeerInfo() (*IndexerClusterPeerInfo, error) { + apiResponse := struct { + Entry []struct { + Content IndexerClusterPeerInfo `json:"content"` + } `json:"entry"` + }{} + path := "/services/cluster/slave/info" + err := c.Get(path, &apiResponse) + if err != nil { + return nil, err + } + if len(apiResponse.Entry) < 1 { + return nil, fmt.Errorf("Invalid response from %s%s", c.managementURI, path) + } + return &apiResponse.Entry[0].Content, nil +} + +// ClusterMasterPeerInfo represents the status of a indexer cluster peer (cluster master endpoint). +// See https://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTcluster#cluster.2Fmaster.2Fpeers +type ClusterMasterPeerInfo struct { + // Unique identifier or GUID for the peer + ID string `json:"guid"` + + // The name for the peer. Displayed on the manager page. + Label string `json:"label"` + + // The ID of the configuration bundle currently being used by the master. + ActiveBundleID string `json:"active_bundle_id"` + + // The initial bundle generation ID recognized by this peer. Any searches from previous generations fail. + // The initial bundle generation ID is created when a peer first comes online, restarts, or recontacts the master. + // Note that this is reported as a very large number (18446744073709552000) that breaks Go's JSON library, while the peer is being decommissioned. + //BaseGenerationID uint64 `json:"base_generation_id"` + + // Count of the number of buckets on this peer, across all indexes. + BucketCount int64 `json:"bucket_count"` + + // Count of the number of buckets by index on this peer. + BucketCountByIndex map[string]int64 `json:"bucket_count_by_index"` + + // Flag indicating if this peer has started heartbeating. + HeartbeatStarted bool `json:"heartbeat_started"` + + // The host and port advertised to peers for the data replication channel. + // Can be either of the form IP:port or hostname:port. + HostPortPair string `json:"host_port_pair"` + + // Flag indicating if this peer belongs to the current committed generation and is searchable. + Searchable bool `json:"is_searchable"` + + // Timestamp for last heartbeat recieved from the peer. + LastHeartbeat int64 `json:"last_heartbeat"` + + // The ID of the configuration bundle this peer is using. + LatestBundleID string `json:"latest_bundle_id"` + + // Used by the master to keep track of pending jobs requested by the master to this peer. + PendingJobCount int `json:"pending_job_count"` + + // Number of buckets for which the peer is primary in its local site, or the number of buckets that return search results from same site as the peer. + PrimaryCount int64 `json:"primary_count"` + + // Number of buckets for which the peer is primary that are not in its local site. + PrimaryCountRemote int64 `json:"primary_count_remote"` + + // Number of replications this peer is part of, as either source or target. + ReplicationCount int `json:"replication_count"` + + // TCP port to listen for replicated data from another cluster member. + ReplicationPort int `json:"replication_port"` + + // Indicates whether to use SSL when sending replication data. + ReplicationUseSSL bool `json:"replication_use_ssl"` + + // To which site the peer belongs. + Site string `json:"site"` + + // Indicates the status of the peer. + Status string `json:"status"` + + // Lists the number of buckets on the peer for each search state for the bucket. + SearchStateCounter struct { + Searchable int64 `json:"Searchable"` + Unsearchable int64 `json:"Unsearchable"` + PendingSearchable int64 `json:"PendingSearchable"` + SearchablePendingMask int64 `json:"SearchablePendingMask"` + } `json:"search_state_counter"` + + // Lists the number of buckets on the peer for each bucket status. + StatusCounter struct { + // complete (warm/cold) bucket + Complete int64 `json:"Complete"` + + // target of replication for already completed (warm/cold) bucket + NonStreamingTarget int64 `json:"NonStreamingTarget"` + + // bucket pending truncation + PendingTruncate int64 `json:"PendingTruncate"` + + // bucket pending discard + PendingDiscard int64 `json:"PendingDiscard"` + + // bucket that is not replicated + Standalone int64 `json:"Standalone"` + + // copy of streaming bucket where some error was encountered + StreamingError int64 `json:"StreamingError"` + + // streaming hot bucket on source side + StreamingSource int64 `json:"StreamingSource"` + + // streaming hot bucket copy on target side + StreamingTarget int64 `json:"StreamingTarget"` + + // uninitialized + Unset int64 `json:"Unset"` + } `json:"status_counter"` +} + +// GetClusterMasterPeers queries the cluster master for info about indexer cluster peers. +// You can only use this on a cluster master. +// See https://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTcluster#cluster.2Fmaster.2Fpeers +func (c *SplunkClient) GetClusterMasterPeers() (map[string]ClusterMasterPeerInfo, error) { + apiResponse := struct { + Entry []struct { + Name string `json:"name"` + Content ClusterMasterPeerInfo `json:"content"` + } `json:"entry"` + }{} + path := "/services/cluster/master/peers" + err := c.Get(path, &apiResponse) + if err != nil { + return nil, err + } + + peers := make(map[string]ClusterMasterPeerInfo) + for _, e := range apiResponse.Entry { + e.Content.ID = e.Name + peers[e.Content.Label] = e.Content + } + + return peers, nil +} + +// RemoveIndexerClusterPeer removes peer from an indexer cluster, where id=unique GUID for the peer. +// You can only use this on a cluster master. +// See https://docs.splunk.com/Documentation/Splunk/8.0.2/Indexer/Removepeerfrommasterlist +func (c *SplunkClient) RemoveIndexerClusterPeer(id string) error { + // sent request to remove from search head cluster consensus + endpoint := fmt.Sprintf("%s/services/cluster/master/control/control/remove_peers?peers=%s", c.managementURI, id) + request, err := http.NewRequest("POST", endpoint, nil) + if err != nil { + return err + } + return c.Do(request, 200, nil) +} + +// DecommissionIndexerClusterPeer takes an indexer cluster peer offline using the decommission endpoint. +// You can use this on any peer in an indexer cluster. +// See https://docs.splunk.com/Documentation/Splunk/latest/Indexer/Takeapeeroffline +func (c *SplunkClient) DecommissionIndexerClusterPeer(enforceCounts bool) error { + enforceCountsAsInt := 0 + if enforceCounts { + enforceCountsAsInt = 1 + } + endpoint := fmt.Sprintf("%s/services/cluster/slave/control/control/decommission?enforce_counts=%d", c.managementURI, enforceCountsAsInt) + request, err := http.NewRequest("POST", endpoint, nil) + if err != nil { + return err + } + return c.Do(request, 200, nil) +} diff --git a/pkg/splunk/enterprise/restapi_test.go b/pkg/splunk/enterprise/restapi_test.go index 816d4602f..fd2dde457 100644 --- a/pkg/splunk/enterprise/restapi_test.go +++ b/pkg/splunk/enterprise/restapi_test.go @@ -231,3 +231,152 @@ func TestRemoveSearchHeadClusterMember(t *testing.T) { // test bad response code splunkClientTester(t, "TestRemoveSearchHeadClusterMember", 404, "", want, test) } + +func TestGetClusterMasterInfo(t *testing.T) { + wantRequest, _ := http.NewRequest("GET", "https://localhost:8089/services/cluster/master/info?count=0&output_mode=json", nil) + want := []http.Request{*wantRequest} + wantInfo := ClusterMasterInfo{ + Initialized: true, + IndexingReady: true, + ServiceReady: true, + MaintenanceMode: false, + RollingRestart: false, + Label: "splunk-s1-cluster-master-0", + ActiveBundle: ClusterBundleInfo{ + BundlePath: "/opt/splunk/var/run/splunk/cluster/remote-bundle/506c58d5aeda1dd6017889e3186e7337-1583870198.bundle", + Checksum: "14310A4AABD23E85BBD4559C4A3B59F8", + Timestamp: 1583870198, + }, + LatestBundle: ClusterBundleInfo{ + BundlePath: "/opt/splunk/var/run/splunk/cluster/remote-bundle/506c58d5aeda1dd6017889e3186e7337-1583870198.bundle", + Checksum: "14310A4AABD23E85BBD4559C4A3B59F8", + Timestamp: 1583870198, + }, + StartTime: 1583948636, + } + test := func(c SplunkClient) error { + gotInfo, err := c.GetClusterMasterInfo() + if err != nil { + return err + } + if *gotInfo != wantInfo { + t.Errorf("info.Status=%v; want %v", *gotInfo, wantInfo) + } + return nil + } + body := `{"links":{},"origin":"https://localhost:8089/services/cluster/master/info","updated":"2020-03-18T01:04:53+00:00","generator":{"build":"a7f645ddaf91","version":"8.0.2"},"entry":[{"name":"master","id":"https://localhost:8089/services/cluster/master/info/master","updated":"1970-01-01T00:00:00+00:00","links":{"alternate":"/services/cluster/master/info/master","list":"/services/cluster/master/info/master"},"author":"system","acl":{"app":"","can_list":true,"can_write":true,"modifiable":false,"owner":"system","perms":{"read":["admin","splunk-system-role"],"write":["admin","splunk-system-role"]},"removable":false,"sharing":"system"},"content":{"active_bundle":{"bundle_path":"/opt/splunk/var/run/splunk/cluster/remote-bundle/506c58d5aeda1dd6017889e3186e7337-1583870198.bundle","checksum":"14310A4AABD23E85BBD4559C4A3B59F8","timestamp":1583870198},"apply_bundle_status":{"invalid_bundle":{"bundle_path":"","bundle_validation_errors_on_master":[],"checksum":"","timestamp":0},"reload_bundle_issued":false,"status":"None"},"backup_and_restore_primaries":false,"controlled_rolling_restart_flag":false,"eai:acl":null,"indexing_ready_flag":true,"initialized_flag":true,"label":"splunk-s1-cluster-master-0","last_check_restart_bundle_result":false,"last_dry_run_bundle":{"bundle_path":"","checksum":"","timestamp":0},"last_validated_bundle":{"bundle_path":"/opt/splunk/var/run/splunk/cluster/remote-bundle/0af7c0e95f313f7be3b0cb1d878df9a1-1583948640.bundle","checksum":"14310A4AABD23E85BBD4559C4A3B59F8","is_valid_bundle":true,"timestamp":1583948640},"latest_bundle":{"bundle_path":"/opt/splunk/var/run/splunk/cluster/remote-bundle/506c58d5aeda1dd6017889e3186e7337-1583870198.bundle","checksum":"14310A4AABD23E85BBD4559C4A3B59F8","timestamp":1583870198},"maintenance_mode":false,"multisite":false,"previous_active_bundle":{"bundle_path":"","checksum":"","timestamp":0},"primaries_backup_status":"No on-going (or) completed primaries backup yet. Check back again in few minutes if you expect a backup.","quiet_period_flag":false,"rolling_restart_flag":false,"rolling_restart_or_upgrade":false,"service_ready_flag":true,"start_time":1583948636,"summary_replication":"false"}}],"paging":{"total":1,"perPage":30,"offset":0},"messages":[]}` + splunkClientTester(t, "TestGetClusterMasterInfo", 200, body, want, test) + + // test body with no entries + test = func(c SplunkClient) error { + _, err := c.GetClusterMasterInfo() + if err == nil { + t.Errorf("GetClusterMasterInfo returned nil; want error") + } + return nil + } + body = `{"links":{},"origin":"https://localhost:8089/services/cluster/master/info","updated":"2020-03-18T01:04:53+00:00","generator":{"build":"a7f645ddaf91","version":"8.0.2"},"entry":[],"paging":{"total":1,"perPage":30,"offset":0},"messages":[]}` + splunkClientTester(t, "TestGetClusterMasterInfo", 200, body, want, test) + + // test error code + splunkClientTester(t, "TestGetClusterMasterInfo", 500, "", want, test) +} + +func TestGetIndexerClusterPeerInfo(t *testing.T) { + wantRequest, _ := http.NewRequest("GET", "https://localhost:8089/services/cluster/slave/info?count=0&output_mode=json", nil) + want := []http.Request{*wantRequest} + wantMemberStatus := "Up" + test := func(c SplunkClient) error { + info, err := c.GetIndexerClusterPeerInfo() + if err != nil { + return err + } + if info.Status != wantMemberStatus { + t.Errorf("info.Status=%s; want %s", info.Status, wantMemberStatus) + } + return nil + } + body := `{"links":{},"origin":"https://localhost:8089/services/cluster/slave/info","updated":"2020-03-18T01:28:18+00:00","generator":{"build":"a7f645ddaf91","version":"8.0.2"},"entry":[{"name":"slave","id":"https://localhost:8089/services/cluster/slave/info/slave","updated":"1970-01-01T00:00:00+00:00","links":{"alternate":"/services/cluster/slave/info/slave","list":"/services/cluster/slave/info/slave"},"author":"system","acl":{"app":"","can_list":true,"can_write":true,"modifiable":false,"owner":"system","perms":{"read":["admin","splunk-system-role"],"write":["admin","splunk-system-role"]},"removable":false,"sharing":"system"},"content":{"active_bundle":{"bundle_path":"/opt/splunk/var/run/splunk/cluster/remote-bundle/87c8c24e7fabc3ff9683c26652cb5890-1583870244.bundle","checksum":"14310A4AABD23E85BBD4559C4A3B59F8","timestamp":1583870244},"base_generation_id":26,"eai:acl":null,"is_registered":true,"last_dry_run_bundle":{"bundle_path":"","checksum":"","timestamp":0},"last_heartbeat_attempt":0,"latest_bundle":{"bundle_path":"/opt/splunk/var/run/splunk/cluster/remote-bundle/87c8c24e7fabc3ff9683c26652cb5890-1583870244.bundle","checksum":"14310A4AABD23E85BBD4559C4A3B59F8","timestamp":1583870244},"maintenance_mode":false,"registered_summary_state":3,"restart_state":"NoRestart","site":"default","status":"Up"}}],"paging":{"total":1,"perPage":30,"offset":0},"messages":[]}` + splunkClientTester(t, "TestGetIndexerClusterPeerInfo", 200, body, want, test) + + // test body with no entries + test = func(c SplunkClient) error { + _, err := c.GetIndexerClusterPeerInfo() + if err == nil { + t.Errorf("GetIndexerClusterPeerInfo returned nil; want error") + } + return nil + } + body = `{"links":{},"origin":"https://localhost:8089/services/cluster/slave/info","updated":"2020-03-18T01:28:18+00:00","generator":{"build":"a7f645ddaf91","version":"8.0.2"},"entry":[],"paging":{"total":1,"perPage":30,"offset":0},"messages":[]}` + splunkClientTester(t, "TestGetIndexerClusterPeerInfo", 200, body, want, test) + + // test error code + splunkClientTester(t, "TestGetIndexerClusterPeerInfo", 500, "", want, test) +} + +func TestGetClusterMasterPeers(t *testing.T) { + wantRequest, _ := http.NewRequest("GET", "https://localhost:8089/services/cluster/master/peers?count=0&output_mode=json", nil) + want := []http.Request{*wantRequest} + var wantPeers = []struct { + ID string + Label string + Status string + }{ + {ID: "D39B1729-E2C5-4273-B9B2-534DA7C2F866", Label: "splunk-s1-indexer-0", Status: "Up"}, + } + test := func(c SplunkClient) error { + peers, err := c.GetClusterMasterPeers() + if err != nil { + return err + } + if len(peers) != len(wantPeers) { + t.Errorf("len(peers)=%d; want %d", len(peers), len(wantPeers)) + } + for n := range wantPeers { + p, ok := peers[wantPeers[n].Label] + if !ok { + t.Errorf("wanted peer not found: %s", wantPeers[n].Label) + } + if p.ID != wantPeers[n].ID { + t.Errorf("peer %s want ID=%s: got %s", wantPeers[n].Label, p.ID, wantPeers[n].ID) + } + if p.Label != wantPeers[n].Label { + t.Errorf("peer %s want Label=%s: got %s", wantPeers[n].Label, p.Label, wantPeers[n].Label) + } + if p.Status != wantPeers[n].Status { + t.Errorf("peer %s want Status=%s: got %s", wantPeers[n].Label, p.Status, wantPeers[n].Status) + } + } + return nil + } + body := `{"links":{"create":"/services/cluster/master/peers/_new"},"origin":"https://localhost:8089/services/cluster/master/peers","updated":"2020-03-18T01:08:53+00:00","generator":{"build":"a7f645ddaf91","version":"8.0.2"},"entry":[{"name":"D39B1729-E2C5-4273-B9B2-534DA7C2F866","id":"https://localhost:8089/services/cluster/master/peers/D39B1729-E2C5-4273-B9B2-534DA7C2F866","updated":"1970-01-01T00:00:00+00:00","links":{"alternate":"/services/cluster/master/peers/D39B1729-E2C5-4273-B9B2-534DA7C2F866","list":"/services/cluster/master/peers/D39B1729-E2C5-4273-B9B2-534DA7C2F866","edit":"/services/cluster/master/peers/D39B1729-E2C5-4273-B9B2-534DA7C2F866"},"author":"system","acl":{"app":"","can_list":true,"can_write":true,"modifiable":false,"owner":"system","perms":{"read":["admin","splunk-system-role"],"write":["admin","splunk-system-role"]},"removable":false,"sharing":"system"},"content":{"active_bundle_id":"14310A4AABD23E85BBD4559C4A3B59F8","apply_bundle_status":{"invalid_bundle":{"bundle_validation_errors":[],"invalid_bundle_id":""},"reasons_for_restart":[],"restart_required_for_apply_bundle":false,"status":"None"},"base_generation_id":26,"bucket_count":73,"bucket_count_by_index":{"_audit":24,"_internal":45,"_telemetry":4},"buckets_rf_by_origin_site":{"default":73},"buckets_sf_by_origin_site":{"default":73},"delayed_buckets_to_discard":[],"eai:acl":null,"fixup_set":[],"heartbeat_started":true,"host_port_pair":"10.36.0.6:8089","indexing_disk_space":210707374080,"is_searchable":true,"is_valid_bundle":true,"label":"splunk-s1-indexer-0","last_dry_run_bundle":"","last_heartbeat":1584493732,"last_validated_bundle":"14310A4AABD23E85BBD4559C4A3B59F8","latest_bundle_id":"14310A4AABD23E85BBD4559C4A3B59F8","peer_registered_summaries":true,"pending_builds":[],"pending_job_count":0,"primary_count":73,"primary_count_remote":0,"register_search_address":"10.36.0.6:8089","replication_count":0,"replication_port":9887,"replication_use_ssl":false,"restart_required_for_applying_dry_run_bundle":false,"search_state_counter":{"PendingSearchable":0,"Searchable":73,"SearchablePendingMask":0,"Unsearchable":0},"site":"default","splunk_version":"8.0.2","status":"Up","status_counter":{"Complete":69,"NonStreamingTarget":0,"StreamingSource":4,"StreamingTarget":0},"summary_replication_count":0}}],"paging":{"total":1,"perPage":30,"offset":0},"messages":[]}` + splunkClientTester(t, "TestGetClusterMasterPeers", 200, body, want, test) + + // test error response + test = func(c SplunkClient) error { + _, err := c.GetClusterMasterPeers() + if err == nil { + t.Errorf("GetClusterMasterPeers returned nil; want error") + } + return nil + } + splunkClientTester(t, "TestGetClusterMasterPeers", 503, "", want, test) +} + +func TestRemoveIndexerClusterPeer(t *testing.T) { + wantRequest, _ := http.NewRequest("POST", "https://localhost:8089/services/cluster/master/control/control/remove_peers?peers=D39B1729-E2C5-4273-B9B2-534DA7C2F866", nil) + want := []http.Request{*wantRequest} + test := func(c SplunkClient) error { + return c.RemoveIndexerClusterPeer("D39B1729-E2C5-4273-B9B2-534DA7C2F866") + } + splunkClientTester(t, "TestRemoveIndexerClusterPeer", 200, "", want, test) +} + +func TestDecommissionIndexerClusterPeer(t *testing.T) { + wantRequest, _ := http.NewRequest("POST", "https://localhost:8089/services/cluster/slave/control/control/decommission?enforce_counts=1", nil) + want := []http.Request{*wantRequest} + test := func(c SplunkClient) error { + return c.DecommissionIndexerClusterPeer(true) + } + splunkClientTester(t, "TestDecommissionIndexerClusterPeer", 200, "", want, test) +} diff --git a/pkg/splunk/reconcile/config.go b/pkg/splunk/reconcile/config.go index dd31a1ee0..3fcb75dc7 100644 --- a/pkg/splunk/reconcile/config.go +++ b/pkg/splunk/reconcile/config.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package deploy +package reconcile import ( "context" @@ -27,8 +27,8 @@ import ( "github.com/splunk/splunk-operator/pkg/splunk/resources" ) -// ReconcileSplunkConfig reconciles the state of Kubernetes Secrets, ConfigMaps and other general settings for Splunk Enterprise instances. -func ReconcileSplunkConfig(client ControllerClient, cr enterprisev1.MetaObject, spec enterprisev1.CommonSplunkSpec, instanceType enterprise.InstanceType) (*corev1.Secret, error) { +// ApplySplunkConfig reconciles the state of Kubernetes Secrets, ConfigMaps and other general settings for Splunk Enterprise instances. +func ApplySplunkConfig(client ControllerClient, cr enterprisev1.MetaObject, spec enterprisev1.CommonSplunkSpec, instanceType enterprise.InstanceType) (*corev1.Secret, error) { var err error // if reference to indexer cluster, extract and re-use idxc.secret diff --git a/pkg/splunk/reconcile/config_test.go b/pkg/splunk/reconcile/config_test.go index 57da3088b..154010450 100644 --- a/pkg/splunk/reconcile/config_test.go +++ b/pkg/splunk/reconcile/config_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package deploy +package reconcile import ( "testing" @@ -23,7 +23,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func TestReconcileSplunkConfig(t *testing.T) { +func TestApplySplunkConfig(t *testing.T) { funcCalls := []mockFuncCall{ {metaName: "*v1.Secret-test-splunk-stack1-search-head-secrets"}, {metaName: "*v1.ConfigMap-test-splunk-stack1-search-head-defaults"}, @@ -44,10 +44,10 @@ func TestReconcileSplunkConfig(t *testing.T) { searchHeadRevised.Spec.Image = "splunk/test" reconcile := func(c *mockClient, cr interface{}) error { obj := cr.(*enterprisev1.SearchHeadCluster) - _, err := ReconcileSplunkConfig(c, obj, obj.Spec.CommonSplunkSpec, enterprise.SplunkSearchHead) + _, err := ApplySplunkConfig(c, obj, obj.Spec.CommonSplunkSpec, enterprise.SplunkSearchHead) return err } - reconcileTester(t, "TestReconcileSplunkConfig", &searchHeadCR, searchHeadRevised, createCalls, updateCalls, reconcile) + reconcileTester(t, "TestApplySplunkConfig", &searchHeadCR, searchHeadRevised, createCalls, updateCalls, reconcile) // test search head with indexer reference secret := corev1.Secret{ @@ -65,7 +65,7 @@ func TestReconcileSplunkConfig(t *testing.T) { {metaName: "*v1.Secret-test-splunk-stack1-search-head-secrets"}, {metaName: "*v1.ConfigMap-test-splunk-stack1-search-head-defaults"}, } - reconcileTester(t, "TestReconcileSplunkConfig", &searchHeadCR, searchHeadRevised, createCalls, updateCalls, reconcile, &secret) + reconcileTester(t, "TestApplySplunkConfig", &searchHeadCR, searchHeadRevised, createCalls, updateCalls, reconcile, &secret) // test indexer with license master indexerCR := enterprisev1.IndexerCluster{ @@ -92,7 +92,7 @@ func TestReconcileSplunkConfig(t *testing.T) { indexerRevised.Spec.LicenseMasterRef.Name = "stack2" reconcile = func(c *mockClient, cr interface{}) error { obj := cr.(*enterprisev1.IndexerCluster) - _, err := ReconcileSplunkConfig(c, obj, obj.Spec.CommonSplunkSpec, enterprise.SplunkIndexer) + _, err := ApplySplunkConfig(c, obj, obj.Spec.CommonSplunkSpec, enterprise.SplunkIndexer) return err } funcCalls = []mockFuncCall{ @@ -102,7 +102,7 @@ func TestReconcileSplunkConfig(t *testing.T) { } createCalls = map[string][]mockFuncCall{"Get": {funcCalls[2]}, "Create": {funcCalls[2]}} updateCalls = map[string][]mockFuncCall{"Get": funcCalls} - reconcileTester(t, "TestReconcileSplunkConfig", &indexerCR, indexerRevised, createCalls, updateCalls, reconcile, &secret) + reconcileTester(t, "TestApplySplunkConfig", &indexerCR, indexerRevised, createCalls, updateCalls, reconcile, &secret) } func TestApplyConfigMap(t *testing.T) { diff --git a/pkg/splunk/reconcile/deployment.go b/pkg/splunk/reconcile/deployment.go index afff7541b..0569d9e1a 100644 --- a/pkg/splunk/reconcile/deployment.go +++ b/pkg/splunk/reconcile/deployment.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package deploy +package reconcile import ( "context" diff --git a/pkg/splunk/reconcile/deployment_test.go b/pkg/splunk/reconcile/deployment_test.go index 456912a2e..eec86a39d 100644 --- a/pkg/splunk/reconcile/deployment_test.go +++ b/pkg/splunk/reconcile/deployment_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package deploy +package reconcile import ( "testing" diff --git a/pkg/splunk/reconcile/doc.go b/pkg/splunk/reconcile/doc.go index f1a89b599..db7d4b988 100644 --- a/pkg/splunk/reconcile/doc.go +++ b/pkg/splunk/reconcile/doc.go @@ -13,8 +13,8 @@ // limitations under the License. /* -Package deploy is used to manipulate Kubernetes resources using its REST API. +Package reconcile is used to manipulate Kubernetes resources using its REST API. Methods within this package are likely to change state and/or mutate data. -This package has dependencies on enterprise, spark and resources. +This package has dependencies on the enterprise, spark and resource modules. */ -package deploy +package reconcile diff --git a/pkg/splunk/reconcile/finalizers.go b/pkg/splunk/reconcile/finalizers.go index e4cc313a2..c3961714e 100644 --- a/pkg/splunk/reconcile/finalizers.go +++ b/pkg/splunk/reconcile/finalizers.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package deploy +package reconcile import ( "context" diff --git a/pkg/splunk/reconcile/finalizers_test.go b/pkg/splunk/reconcile/finalizers_test.go index a7796c846..969ef1551 100644 --- a/pkg/splunk/reconcile/finalizers_test.go +++ b/pkg/splunk/reconcile/finalizers_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package deploy +package reconcile import ( "fmt" diff --git a/pkg/splunk/reconcile/indexercluster.go b/pkg/splunk/reconcile/indexercluster.go index a1a78bef7..318884a90 100644 --- a/pkg/splunk/reconcile/indexercluster.go +++ b/pkg/splunk/reconcile/indexercluster.go @@ -12,27 +12,32 @@ // See the License for the specific language governing permissions and // limitations under the License. -package deploy +package reconcile import ( "context" "fmt" "time" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "github.com/go-logr/logr" enterprisev1 "github.com/splunk/splunk-operator/pkg/apis/enterprise/v1alpha2" "github.com/splunk/splunk-operator/pkg/splunk/enterprise" + "github.com/splunk/splunk-operator/pkg/splunk/resources" ) -// ReconcileIndexerCluster reconciles the state of a Splunk Enterprise indexer cluster. -func ReconcileIndexerCluster(client ControllerClient, cr *enterprisev1.IndexerCluster) (reconcile.Result, error) { +// ApplyIndexerCluster reconciles the state of a Splunk Enterprise indexer cluster. +func ApplyIndexerCluster(client ControllerClient, cr *enterprisev1.IndexerCluster) (reconcile.Result, error) { // unless modified, reconcile for this object will be requeued after 5 seconds result := reconcile.Result{ Requeue: true, RequeueAfter: time.Second * 5, } + scopedLog := log.WithName("ApplyIndexerCluster").WithValues("name", cr.GetIdentifier(), "namespace", cr.GetNamespace()) // validate and updates defaults for CR err := enterprise.ValidateIndexerClusterSpec(&cr.Spec) @@ -61,7 +66,7 @@ func ReconcileIndexerCluster(client ControllerClient, cr *enterprisev1.IndexerCl } // create or update general config resources - _, err = ReconcileSplunkConfig(client, cr, cr.Spec.CommonSplunkSpec, enterprise.SplunkIndexer) + secrets, err := ApplySplunkConfig(client, cr, cr.Spec.CommonSplunkSpec, enterprise.SplunkIndexer) if err != nil { return result, err } @@ -92,7 +97,7 @@ func ReconcileIndexerCluster(client ControllerClient, cr *enterprisev1.IndexerCl cr.Status.ClusterMasterPhase, err = ApplyStatefulSet(client, statefulSet) if err == nil && cr.Status.Phase == enterprisev1.PhaseReady { mgr := DefaultStatefulSetPodManager{} - cr.Status.ClusterMasterPhase, err = ReconcileStatefulSetPods(client, statefulSet, &mgr, 1) + cr.Status.ClusterMasterPhase, err = UpdateStatefulSetPods(client, statefulSet, &mgr, 1) } if err != nil { cr.Status.ClusterMasterPhase = enterprisev1.PhaseError @@ -105,15 +110,162 @@ func ReconcileIndexerCluster(client ControllerClient, cr *enterprisev1.IndexerCl return result, err } cr.Status.Phase, err = ApplyStatefulSet(client, statefulSet) - cr.Status.ReadyReplicas = statefulSet.Status.ReadyReplicas - if err == nil && cr.Status.Phase == enterprisev1.PhaseReady { - mgr := DefaultStatefulSetPodManager{} - cr.Status.Phase, err = ReconcileStatefulSetPods(client, statefulSet, &mgr, cr.Spec.Replicas) + if err != nil { + return result, err } + + // update CR status with SHC information + mgr := IndexerClusterPodManager{client: client, log: scopedLog, cr: cr, secrets: secrets} + err = mgr.updateStatus(statefulSet) + if err != nil || cr.Status.ReadyReplicas == 0 || !cr.Status.Initialized || !cr.Status.IndexingReady || !cr.Status.ServiceReady { + scopedLog.Error(err, "Indexer cluster is not ready") + cr.Status.Phase = enterprisev1.PhasePending + return result, nil + } + + // manage scaling and updates + cr.Status.Phase, err = UpdateStatefulSetPods(client, statefulSet, &mgr, cr.Spec.Replicas) if err != nil { cr.Status.Phase = enterprisev1.PhaseError - } else if cr.Status.Phase == enterprisev1.PhaseReady { + return result, err + } + + // no need to requeue if everything is ready + if cr.Status.Phase == enterprisev1.PhaseReady { result.Requeue = false } - return result, err + return result, nil +} + +// IndexerClusterPodManager is used to manage the pods within a search head cluster +type IndexerClusterPodManager struct { + client ControllerClient + log logr.Logger + cr *enterprisev1.IndexerCluster + secrets *corev1.Secret +} + +// PrepareScaleDown for IndexerClusterPodManager prepares indexer pod to be removed via scale down event; it returns true when ready +func (mgr *IndexerClusterPodManager) PrepareScaleDown(n int32) (bool, error) { + // first, decommission indexer peer with enforceCounts=true; this will rebalance buckets across other peers + complete, err := mgr.decommission(n, true) + if err != nil { + return false, err + } + if !complete { + return false, nil + } + + // next, remove the peer + c := mgr.getClusterMasterClient() + return true, c.RemoveIndexerClusterPeer(mgr.cr.Status.Peers[n].ID) +} + +// PrepareRecycle for IndexerClusterPodManager prepares indexer pod to be recycled for updates; it returns true when ready +func (mgr *IndexerClusterPodManager) PrepareRecycle(n int32) (bool, error) { + return mgr.decommission(n, false) +} + +// FinishRecycle for IndexerClusterPodManager completes recycle event for indexer pod; it returns true when complete +func (mgr *IndexerClusterPodManager) FinishRecycle(n int32) (bool, error) { + return mgr.cr.Status.Peers[n].Status == "Up", nil +} + +// decommission for IndexerClusterPodManager decommissions an indexer pod; it returns true when ready +func (mgr *IndexerClusterPodManager) decommission(n int32, enforceCounts bool) (bool, error) { + peerName := enterprise.GetSplunkStatefulsetPodName(enterprise.SplunkIndexer, mgr.cr.GetIdentifier(), n) + + switch mgr.cr.Status.Peers[n].Status { + case "Up": + mgr.log.Info("Decommissioning indexer cluster peer", "peerName", peerName, "enforceCounts", enforceCounts) + c := mgr.getClient(n) + return false, c.DecommissionIndexerClusterPeer(enforceCounts) + + case "Decommissioning": + mgr.log.Info("Waiting for decommission to complete", "peerName", peerName) + return false, nil + + case "ReassigningPrimaries": + mgr.log.Info("Waiting for decommission to complete", "peerName", peerName) + return false, nil + + case "GracefulShutdown": + mgr.log.Info("Decommission complete", "peerName", peerName, "Status", mgr.cr.Status.Peers[n].Status) + return true, nil + + case "Down": + mgr.log.Info("Decommission complete", "peerName", peerName, "Status", mgr.cr.Status.Peers[n].Status) + return true, nil + + case "": // this can happen after the peer has been removed from the indexer cluster + mgr.log.Info("Peer has empty ID", "peerName", peerName) + return false, nil + } + + // unhandled status + return false, fmt.Errorf("Status=%s", mgr.cr.Status.Peers[n].Status) +} + +// getClient for IndexerClusterPodManager returns a SplunkClient for the member n +func (mgr *IndexerClusterPodManager) getClient(n int32) *enterprise.SplunkClient { + memberName := enterprise.GetSplunkStatefulsetPodName(enterprise.SplunkIndexer, mgr.cr.GetIdentifier(), n) + fqdnName := resources.GetServiceFQDN(mgr.cr.GetNamespace(), + fmt.Sprintf("%s.%s", memberName, enterprise.GetSplunkServiceName(enterprise.SplunkIndexer, mgr.cr.GetIdentifier(), true))) + return enterprise.NewSplunkClient(fmt.Sprintf("https://%s:8089", fqdnName), "admin", string(mgr.secrets.Data["password"])) +} + +// getClusterMasterClient for IndexerClusterPodManager returns a SplunkClient for cluster master +func (mgr *IndexerClusterPodManager) getClusterMasterClient() *enterprise.SplunkClient { + fqdnName := resources.GetServiceFQDN(mgr.cr.GetNamespace(), enterprise.GetSplunkServiceName(enterprise.SplunkClusterMaster, mgr.cr.GetIdentifier(), false)) + return enterprise.NewSplunkClient(fmt.Sprintf("https://%s:8089", fqdnName), "admin", string(mgr.secrets.Data["password"])) +} + +// updateStatus for IndexerClusterPodManager uses the REST API to update the status for a SearcHead custom resource +func (mgr *IndexerClusterPodManager) updateStatus(statefulSet *appsv1.StatefulSet) error { + mgr.cr.Status.ReadyReplicas = statefulSet.Status.ReadyReplicas + mgr.cr.Status.Peers = []enterprisev1.IndexerClusterMemberStatus{} + + if mgr.cr.Status.ClusterMasterPhase != enterprisev1.PhaseReady { + mgr.cr.Status.Phase = enterprisev1.PhasePending + mgr.cr.Status.Initialized = false + mgr.cr.Status.IndexingReady = false + mgr.cr.Status.ServiceReady = false + mgr.cr.Status.MaintenanceMode = false + mgr.log.Info("Waiting for cluster master to become ready") + return nil + } + + // get indexer cluster info from cluster master if it's ready + c := mgr.getClusterMasterClient() + clusterInfo, err := c.GetClusterMasterInfo() + if err != nil { + return err + } + mgr.cr.Status.Initialized = clusterInfo.Initialized + mgr.cr.Status.IndexingReady = clusterInfo.IndexingReady + mgr.cr.Status.ServiceReady = clusterInfo.ServiceReady + mgr.cr.Status.MaintenanceMode = clusterInfo.MaintenanceMode + + // get peer information from cluster master + peers, err := c.GetClusterMasterPeers() + if err != nil { + return err + } + for n := int32(0); n < statefulSet.Status.Replicas; n++ { + peerName := enterprise.GetSplunkStatefulsetPodName(enterprise.SplunkIndexer, mgr.cr.GetIdentifier(), n) + peerStatus := enterprisev1.IndexerClusterMemberStatus{Name: peerName} + peerInfo, ok := peers[peerName] + if ok { + peerStatus.ID = peerInfo.ID + peerStatus.Status = peerInfo.Status + peerStatus.ActiveBundleID = peerInfo.ActiveBundleID + peerStatus.BucketCount = peerInfo.BucketCount + peerStatus.Searchable = peerInfo.Searchable + } else { + mgr.log.Info("Peer is not known by cluster master", "peerName", peerName) + } + mgr.cr.Status.Peers = append(mgr.cr.Status.Peers, peerStatus) + } + + return nil } diff --git a/pkg/splunk/reconcile/indexercluster_test.go b/pkg/splunk/reconcile/indexercluster_test.go index 06d1f729c..6838e2e60 100644 --- a/pkg/splunk/reconcile/indexercluster_test.go +++ b/pkg/splunk/reconcile/indexercluster_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package deploy +package reconcile import ( "testing" @@ -23,7 +23,7 @@ import ( enterprisev1 "github.com/splunk/splunk-operator/pkg/apis/enterprise/v1alpha2" ) -func TestReconcileIndexerCluster(t *testing.T) { +func TestApplyIndexerCluster(t *testing.T) { funcCalls := []mockFuncCall{ {metaName: "*v1.Secret-test-splunk-stack1-indexer-secrets"}, {metaName: "*v1.Service-test-splunk-stack1-indexer-headless"}, @@ -47,17 +47,17 @@ func TestReconcileIndexerCluster(t *testing.T) { revised := current.DeepCopy() revised.Spec.Image = "splunk/test" reconcile := func(c *mockClient, cr interface{}) error { - _, err := ReconcileIndexerCluster(c, cr.(*enterprisev1.IndexerCluster)) + _, err := ApplyIndexerCluster(c, cr.(*enterprisev1.IndexerCluster)) return err } - reconcileTester(t, "TestReconcileIndexerCluster", ¤t, revised, createCalls, updateCalls, reconcile) + reconcileTester(t, "TestApplyIndexerCluster", ¤t, revised, createCalls, updateCalls, reconcile) // test deletion currentTime := metav1.NewTime(time.Now()) revised.ObjectMeta.DeletionTimestamp = ¤tTime revised.ObjectMeta.Finalizers = []string{"enterprise.splunk.com/delete-pvc"} deleteFunc := func(cr enterprisev1.MetaObject, c ControllerClient) (bool, error) { - _, err := ReconcileIndexerCluster(c, cr.(*enterprisev1.IndexerCluster)) + _, err := ApplyIndexerCluster(c, cr.(*enterprisev1.IndexerCluster)) return true, err } splunkDeletionTester(t, revised, deleteFunc) diff --git a/pkg/splunk/reconcile/licensemaster.go b/pkg/splunk/reconcile/licensemaster.go index a99739090..af51c7ae3 100644 --- a/pkg/splunk/reconcile/licensemaster.go +++ b/pkg/splunk/reconcile/licensemaster.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package deploy +package reconcile import ( "context" @@ -24,8 +24,8 @@ import ( "github.com/splunk/splunk-operator/pkg/splunk/enterprise" ) -// ReconcileLicenseMaster reconciles the state for the Splunk Enterprise license master. -func ReconcileLicenseMaster(client ControllerClient, cr *enterprisev1.LicenseMaster) (reconcile.Result, error) { +// ApplyLicenseMaster reconciles the state for the Splunk Enterprise license master. +func ApplyLicenseMaster(client ControllerClient, cr *enterprisev1.LicenseMaster) (reconcile.Result, error) { // unless modified, reconcile for this object will be requeued after 5 seconds result := reconcile.Result{ @@ -57,7 +57,7 @@ func ReconcileLicenseMaster(client ControllerClient, cr *enterprisev1.LicenseMas } // create or update general config resources - _, err = ReconcileSplunkConfig(client, cr, cr.Spec.CommonSplunkSpec, enterprise.SplunkLicenseMaster) + _, err = ApplySplunkConfig(client, cr, cr.Spec.CommonSplunkSpec, enterprise.SplunkLicenseMaster) if err != nil { return result, err } @@ -76,7 +76,7 @@ func ReconcileLicenseMaster(client ControllerClient, cr *enterprisev1.LicenseMas cr.Status.Phase, err = ApplyStatefulSet(client, statefulSet) if err == nil && cr.Status.Phase == enterprisev1.PhaseReady { mgr := DefaultStatefulSetPodManager{} - cr.Status.Phase, err = ReconcileStatefulSetPods(client, statefulSet, &mgr, 1) + cr.Status.Phase, err = UpdateStatefulSetPods(client, statefulSet, &mgr, 1) } if err != nil { cr.Status.Phase = enterprisev1.PhaseError diff --git a/pkg/splunk/reconcile/licensemaster_test.go b/pkg/splunk/reconcile/licensemaster_test.go index 70d03d7a0..7d5f67fad 100644 --- a/pkg/splunk/reconcile/licensemaster_test.go +++ b/pkg/splunk/reconcile/licensemaster_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package deploy +package reconcile import ( "testing" @@ -23,7 +23,7 @@ import ( enterprisev1 "github.com/splunk/splunk-operator/pkg/apis/enterprise/v1alpha2" ) -func TestReconcileLicenseMaster(t *testing.T) { +func TestApplyLicenseMaster(t *testing.T) { funcCalls := []mockFuncCall{ {metaName: "*v1.Secret-test-splunk-stack1-license-master-secrets"}, {metaName: "*v1.Service-test-splunk-stack1-license-master-service"}, @@ -43,17 +43,17 @@ func TestReconcileLicenseMaster(t *testing.T) { revised := current.DeepCopy() revised.Spec.Image = "splunk/test" reconcile := func(c *mockClient, cr interface{}) error { - _, err := ReconcileLicenseMaster(c, cr.(*enterprisev1.LicenseMaster)) + _, err := ApplyLicenseMaster(c, cr.(*enterprisev1.LicenseMaster)) return err } - reconcileTester(t, "TestReconcileLicenseMaster", ¤t, revised, createCalls, updateCalls, reconcile) + reconcileTester(t, "TestApplyLicenseMaster", ¤t, revised, createCalls, updateCalls, reconcile) // test deletion currentTime := metav1.NewTime(time.Now()) revised.ObjectMeta.DeletionTimestamp = ¤tTime revised.ObjectMeta.Finalizers = []string{"enterprise.splunk.com/delete-pvc"} deleteFunc := func(cr enterprisev1.MetaObject, c ControllerClient) (bool, error) { - _, err := ReconcileLicenseMaster(c, cr.(*enterprisev1.LicenseMaster)) + _, err := ApplyLicenseMaster(c, cr.(*enterprisev1.LicenseMaster)) return true, err } splunkDeletionTester(t, revised, deleteFunc) diff --git a/pkg/splunk/reconcile/searchheadcluster.go b/pkg/splunk/reconcile/searchheadcluster.go index 9fff44755..db893a593 100644 --- a/pkg/splunk/reconcile/searchheadcluster.go +++ b/pkg/splunk/reconcile/searchheadcluster.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package deploy +package reconcile import ( "context" @@ -30,14 +30,14 @@ import ( "github.com/splunk/splunk-operator/pkg/splunk/resources" ) -// ReconcileSearchHeadCluster reconciles the state for a Splunk Enterprise search head cluster. -func ReconcileSearchHeadCluster(client ControllerClient, cr *enterprisev1.SearchHeadCluster) (reconcile.Result, error) { +// ApplySearchHeadCluster reconciles the state for a Splunk Enterprise search head cluster. +func ApplySearchHeadCluster(client ControllerClient, cr *enterprisev1.SearchHeadCluster) (reconcile.Result, error) { // unless modified, reconcile for this object will be requeued after 5 seconds result := reconcile.Result{ Requeue: true, RequeueAfter: time.Second * 5, } - scopedLog := log.WithName("ReconcileSearchHeadCluster").WithValues("name", cr.GetIdentifier(), "namespace", cr.GetNamespace()) + scopedLog := log.WithName("ApplySearchHeadCluster").WithValues("name", cr.GetIdentifier(), "namespace", cr.GetNamespace()) // validate and updates defaults for CR err := enterprise.ValidateSearchHeadClusterSpec(&cr.Spec) @@ -66,7 +66,7 @@ func ReconcileSearchHeadCluster(client ControllerClient, cr *enterprisev1.Search } // create or update general config resources - secrets, err := ReconcileSplunkConfig(client, cr, cr.Spec.CommonSplunkSpec, enterprise.SplunkSearchHead) + secrets, err := ApplySplunkConfig(client, cr, cr.Spec.CommonSplunkSpec, enterprise.SplunkSearchHead) if err != nil { return result, err } @@ -97,7 +97,7 @@ func ReconcileSearchHeadCluster(client ControllerClient, cr *enterprisev1.Search cr.Status.DeployerPhase, err = ApplyStatefulSet(client, statefulSet) if err == nil && cr.Status.DeployerPhase == enterprisev1.PhaseReady { mgr := DefaultStatefulSetPodManager{} - cr.Status.DeployerPhase, err = ReconcileStatefulSetPods(client, statefulSet, &mgr, 1) + cr.Status.DeployerPhase, err = UpdateStatefulSetPods(client, statefulSet, &mgr, 1) } if err != nil { cr.Status.DeployerPhase = enterprisev1.PhaseError @@ -124,7 +124,7 @@ func ReconcileSearchHeadCluster(client ControllerClient, cr *enterprisev1.Search } // manage scaling and updates - cr.Status.Phase, err = ReconcileStatefulSetPods(client, statefulSet, &mgr, cr.Spec.Replicas) + cr.Status.Phase, err = UpdateStatefulSetPods(client, statefulSet, &mgr, cr.Spec.Replicas) if err != nil { cr.Status.Phase = enterprisev1.PhaseError return result, err @@ -145,10 +145,10 @@ type SearchHeadClusterPodManager struct { secrets *corev1.Secret } -// Decommission for SearchHeadClusterPodManager decommissions search head cluster member; it returns true when complete -func (mgr *SearchHeadClusterPodManager) Decommission(n int32) (bool, error) { +// PrepareScaleDown for SearchHeadClusterPodManager prepares search head pod to be removed via scale down event; it returns true when ready +func (mgr *SearchHeadClusterPodManager) PrepareScaleDown(n int32) (bool, error) { // start by quarantining the pod - result, err := mgr.Quarantine(n) + result, err := mgr.PrepareRecycle(n) if err != nil || !result { return result, err } @@ -185,8 +185,8 @@ func (mgr *SearchHeadClusterPodManager) Decommission(n int32) (bool, error) { return true, nil } -// Quarantine for SearchHeadClusterPodManager quarantines a search head cluster member; it returns true when complete -func (mgr *SearchHeadClusterPodManager) Quarantine(n int32) (bool, error) { +// PrepareRecycle for SearchHeadClusterPodManager prepares search head pod to be recycled for updates; it returns true when ready +func (mgr *SearchHeadClusterPodManager) PrepareRecycle(n int32) (bool, error) { memberName := enterprise.GetSplunkStatefulsetPodName(enterprise.SplunkSearchHead, mgr.cr.GetIdentifier(), n) switch mgr.cr.Status.Members[n].Status { @@ -198,7 +198,7 @@ func (mgr *SearchHeadClusterPodManager) Quarantine(n int32) (bool, error) { case "ManualDetention": // Wait until active searches have drained - searchesComplete := mgr.cr.Status.Members[n].ActiveSearches == 0 + searchesComplete := mgr.cr.Status.Members[n].ActiveHistoricalSearchCount+mgr.cr.Status.Members[n].ActiveRealtimeSearchCount == 0 if searchesComplete { mgr.log.Info("Detention complete", "memberName", memberName) } else { @@ -211,20 +211,20 @@ func (mgr *SearchHeadClusterPodManager) Quarantine(n int32) (bool, error) { return false, fmt.Errorf("Status=%s", mgr.cr.Status.Members[n].Status) } -// ReleaseQuarantine for SearchHeadClusterPodManager releases quarantine and returns true, or returns false if not quarantined -func (mgr *SearchHeadClusterPodManager) ReleaseQuarantine(n int32) (bool, error) { +// FinishRecycle for SearchHeadClusterPodManager completes recycle event for search head pod; it returns true when complete +func (mgr *SearchHeadClusterPodManager) FinishRecycle(n int32) (bool, error) { memberName := enterprise.GetSplunkStatefulsetPodName(enterprise.SplunkSearchHead, mgr.cr.GetIdentifier(), n) switch mgr.cr.Status.Members[n].Status { case "Up": // not in detention - return false, nil + return true, nil case "ManualDetention": // release from detention mgr.log.Info("Releasing search head cluster member from detention", "memberName", memberName) c := mgr.getClient(n) - return true, c.SetSearchHeadDetention(false) + return false, c.SetSearchHeadDetention(false) } // unhandled status @@ -246,6 +246,7 @@ func (mgr *SearchHeadClusterPodManager) updateStatus(statefulSet *appsv1.Statefu if mgr.cr.Status.ReadyReplicas == 0 { return nil } + gotCaptainInfo := false mgr.cr.Status.Members = []enterprisev1.SearchHeadClusterMemberStatus{} for n := int32(0); n < mgr.cr.Status.ReadyReplicas; n++ { c := mgr.getClient(n) @@ -254,8 +255,22 @@ func (mgr *SearchHeadClusterPodManager) updateStatus(statefulSet *appsv1.Statefu memberInfo, err := c.GetSearchHeadClusterMemberInfo() if err == nil { memberStatus.Status = memberInfo.Status + memberStatus.Adhoc = memberInfo.Adhoc memberStatus.Registered = memberInfo.Registered - memberStatus.ActiveSearches = memberInfo.ActiveHistoricalSearchCount + memberInfo.ActiveRealtimeSearchCount + memberStatus.ActiveHistoricalSearchCount = memberInfo.ActiveHistoricalSearchCount + memberStatus.ActiveRealtimeSearchCount = memberInfo.ActiveRealtimeSearchCount + if !gotCaptainInfo { + // try querying captain api; note that this should work on any node + captainInfo, err := c.GetSearchHeadCaptainInfo() + if err == nil { + mgr.cr.Status.Captain = captainInfo.Label + mgr.cr.Status.CaptainReady = captainInfo.ServiceReady + mgr.cr.Status.Initialized = captainInfo.Initialized + mgr.cr.Status.MinPeersJoined = captainInfo.MinPeersJoined + mgr.cr.Status.MaintenanceMode = captainInfo.MaintenanceMode + gotCaptainInfo = true + } + } } else if n < statefulSet.Status.Replicas { // ignore error if pod was just terminated for scale down event (n >= Replicas) mgr.log.Error(err, "Unable to retrieve search head cluster member info", "memberName", memberName) @@ -264,18 +279,5 @@ func (mgr *SearchHeadClusterPodManager) updateStatus(statefulSet *appsv1.Statefu mgr.cr.Status.Members = append(mgr.cr.Status.Members, memberStatus) } - // get search head cluster info from captain - fqdnName := resources.GetServiceFQDN(mgr.cr.GetNamespace(), enterprise.GetSplunkServiceName(enterprise.SplunkSearchHead, mgr.cr.GetIdentifier(), false)) - c := enterprise.NewSplunkClient(fmt.Sprintf("https://%s:8089", fqdnName), "admin", string(mgr.secrets.Data["password"])) - captainInfo, err := c.GetSearchHeadCaptainInfo() - if err != nil { - return err - } - mgr.cr.Status.Captain = captainInfo.Label - mgr.cr.Status.CaptainReady = captainInfo.ServiceReadyFlag - mgr.cr.Status.Initialized = captainInfo.InitializedFlag - mgr.cr.Status.MinPeersJoined = captainInfo.MinPeersJoinedFlag - mgr.cr.Status.MaintenanceMode = captainInfo.MaintenanceMode - return nil } diff --git a/pkg/splunk/reconcile/searchheadcluster_test.go b/pkg/splunk/reconcile/searchheadcluster_test.go index 723aca723..37aa7fb6e 100644 --- a/pkg/splunk/reconcile/searchheadcluster_test.go +++ b/pkg/splunk/reconcile/searchheadcluster_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package deploy +package reconcile import ( "testing" @@ -23,7 +23,7 @@ import ( enterprisev1 "github.com/splunk/splunk-operator/pkg/apis/enterprise/v1alpha2" ) -func TestReconcileSearchHeadCluster(t *testing.T) { +func TestApplySearchHeadCluster(t *testing.T) { funcCalls := []mockFuncCall{ {metaName: "*v1.Secret-test-splunk-stack1-search-head-secrets"}, {metaName: "*v1.Service-test-splunk-stack1-search-head-headless"}, @@ -46,17 +46,17 @@ func TestReconcileSearchHeadCluster(t *testing.T) { revised := current.DeepCopy() revised.Spec.Image = "splunk/test" reconcile := func(c *mockClient, cr interface{}) error { - _, err := ReconcileSearchHeadCluster(c, cr.(*enterprisev1.SearchHeadCluster)) + _, err := ApplySearchHeadCluster(c, cr.(*enterprisev1.SearchHeadCluster)) return err } - reconcileTester(t, "TestReconcileSearchHeadCluster", ¤t, revised, createCalls, updateCalls, reconcile) + reconcileTester(t, "TestApplySearchHeadCluster", ¤t, revised, createCalls, updateCalls, reconcile) // test deletion currentTime := metav1.NewTime(time.Now()) revised.ObjectMeta.DeletionTimestamp = ¤tTime revised.ObjectMeta.Finalizers = []string{"enterprise.splunk.com/delete-pvc"} deleteFunc := func(cr enterprisev1.MetaObject, c ControllerClient) (bool, error) { - _, err := ReconcileSearchHeadCluster(c, cr.(*enterprisev1.SearchHeadCluster)) + _, err := ApplySearchHeadCluster(c, cr.(*enterprisev1.SearchHeadCluster)) return true, err } splunkDeletionTester(t, revised, deleteFunc) diff --git a/pkg/splunk/reconcile/service.go b/pkg/splunk/reconcile/service.go index 0bf52137c..ba42571a0 100644 --- a/pkg/splunk/reconcile/service.go +++ b/pkg/splunk/reconcile/service.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package deploy +package reconcile import ( "context" diff --git a/pkg/splunk/reconcile/service_test.go b/pkg/splunk/reconcile/service_test.go index ec0d4cb42..f2e5898cd 100644 --- a/pkg/splunk/reconcile/service_test.go +++ b/pkg/splunk/reconcile/service_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package deploy +package reconcile import ( "testing" diff --git a/pkg/splunk/reconcile/spark.go b/pkg/splunk/reconcile/spark.go index 8cb4422c7..c8a31bf7f 100644 --- a/pkg/splunk/reconcile/spark.go +++ b/pkg/splunk/reconcile/spark.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package deploy +package reconcile import ( "context" @@ -25,8 +25,8 @@ import ( "github.com/splunk/splunk-operator/pkg/splunk/spark" ) -// ReconcileSpark reconciles the Deployments and Services for a Spark cluster. -func ReconcileSpark(client ControllerClient, cr *enterprisev1.Spark) (reconcile.Result, error) { +// ApplySpark reconciles the Deployments and Services for a Spark cluster. +func ApplySpark(client ControllerClient, cr *enterprisev1.Spark) (reconcile.Result, error) { // unless modified, reconcile for this object will be requeued after 5 seconds result := reconcile.Result{ diff --git a/pkg/splunk/reconcile/spark_test.go b/pkg/splunk/reconcile/spark_test.go index e1a13bb35..1f6f72dd0 100644 --- a/pkg/splunk/reconcile/spark_test.go +++ b/pkg/splunk/reconcile/spark_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package deploy +package reconcile import ( "testing" @@ -23,7 +23,7 @@ import ( enterprisev1 "github.com/splunk/splunk-operator/pkg/apis/enterprise/v1alpha2" ) -func TestReconcileSpark(t *testing.T) { +func TestApplySpark(t *testing.T) { funcCalls := []mockFuncCall{ {metaName: "*v1.Service-test-splunk-stack1-spark-master-service"}, {metaName: "*v1.Service-test-splunk-stack1-spark-worker-headless"}, @@ -44,17 +44,17 @@ func TestReconcileSpark(t *testing.T) { revised := current.DeepCopy() revised.Spec.Image = "splunk/test" reconcile := func(c *mockClient, cr interface{}) error { - _, err := ReconcileSpark(c, cr.(*enterprisev1.Spark)) + _, err := ApplySpark(c, cr.(*enterprisev1.Spark)) return err } - reconcileTester(t, "TestReconcileSpark", ¤t, revised, createCalls, updateCalls, reconcile) + reconcileTester(t, "TestApplySpark", ¤t, revised, createCalls, updateCalls, reconcile) // test deletion currentTime := metav1.NewTime(time.Now()) revised.ObjectMeta.DeletionTimestamp = ¤tTime revised.ObjectMeta.Finalizers = []string{"enterprise.splunk.com/delete-pvc"} deleteFunc := func(cr enterprisev1.MetaObject, c ControllerClient) (bool, error) { - _, err := ReconcileSpark(c, cr.(*enterprisev1.Spark)) + _, err := ApplySpark(c, cr.(*enterprisev1.Spark)) return true, err } splunkDeletionTester(t, revised, deleteFunc) diff --git a/pkg/splunk/reconcile/standalone.go b/pkg/splunk/reconcile/standalone.go index 129c00da1..ed0cdac94 100644 --- a/pkg/splunk/reconcile/standalone.go +++ b/pkg/splunk/reconcile/standalone.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package deploy +package reconcile import ( "context" @@ -25,8 +25,8 @@ import ( "github.com/splunk/splunk-operator/pkg/splunk/enterprise" ) -// ReconcileStandalone reconciles the StatefulSet for N standalone instances of Splunk Enterprise. -func ReconcileStandalone(client ControllerClient, cr *enterprisev1.Standalone) (reconcile.Result, error) { +// ApplyStandalone reconciles the StatefulSet for N standalone instances of Splunk Enterprise. +func ApplyStandalone(client ControllerClient, cr *enterprisev1.Standalone) (reconcile.Result, error) { // unless modified, reconcile for this object will be requeued after 5 seconds result := reconcile.Result{ @@ -60,7 +60,7 @@ func ReconcileStandalone(client ControllerClient, cr *enterprisev1.Standalone) ( } // create or update general config resources - _, err = ReconcileSplunkConfig(client, cr, cr.Spec.CommonSplunkSpec, enterprise.SplunkStandalone) + _, err = ApplySplunkConfig(client, cr, cr.Spec.CommonSplunkSpec, enterprise.SplunkStandalone) if err != nil { return result, err } @@ -80,7 +80,7 @@ func ReconcileStandalone(client ControllerClient, cr *enterprisev1.Standalone) ( cr.Status.ReadyReplicas = statefulSet.Status.ReadyReplicas if err == nil && cr.Status.Phase == enterprisev1.PhaseReady { mgr := DefaultStatefulSetPodManager{} - cr.Status.Phase, err = ReconcileStatefulSetPods(client, statefulSet, &mgr, cr.Spec.Replicas) + cr.Status.Phase, err = UpdateStatefulSetPods(client, statefulSet, &mgr, cr.Spec.Replicas) } if err != nil { cr.Status.Phase = enterprisev1.PhaseError diff --git a/pkg/splunk/reconcile/standalone_test.go b/pkg/splunk/reconcile/standalone_test.go index 8b44b94ac..a4627d590 100644 --- a/pkg/splunk/reconcile/standalone_test.go +++ b/pkg/splunk/reconcile/standalone_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package deploy +package reconcile import ( "testing" @@ -23,7 +23,7 @@ import ( enterprisev1 "github.com/splunk/splunk-operator/pkg/apis/enterprise/v1alpha2" ) -func TestReconcileStandalone(t *testing.T) { +func TestApplyStandalone(t *testing.T) { funcCalls := []mockFuncCall{ {metaName: "*v1.Secret-test-splunk-stack1-standalone-secrets"}, {metaName: "*v1.Service-test-splunk-stack1-standalone-headless"}, @@ -43,17 +43,17 @@ func TestReconcileStandalone(t *testing.T) { revised := current.DeepCopy() revised.Spec.Image = "splunk/test" reconcile := func(c *mockClient, cr interface{}) error { - _, err := ReconcileStandalone(c, cr.(*enterprisev1.Standalone)) + _, err := ApplyStandalone(c, cr.(*enterprisev1.Standalone)) return err } - reconcileTester(t, "TestReconcileStandalone", ¤t, revised, createCalls, updateCalls, reconcile) + reconcileTester(t, "TestApplyStandalone", ¤t, revised, createCalls, updateCalls, reconcile) // test deletion currentTime := metav1.NewTime(time.Now()) revised.ObjectMeta.DeletionTimestamp = ¤tTime revised.ObjectMeta.Finalizers = []string{"enterprise.splunk.com/delete-pvc"} deleteFunc := func(cr enterprisev1.MetaObject, c ControllerClient) (bool, error) { - _, err := ReconcileStandalone(c, cr.(*enterprisev1.Standalone)) + _, err := ApplyStandalone(c, cr.(*enterprisev1.Standalone)) return true, err } splunkDeletionTester(t, revised, deleteFunc) diff --git a/pkg/splunk/reconcile/statefulset.go b/pkg/splunk/reconcile/statefulset.go index 12a59f409..569d4cb4a 100644 --- a/pkg/splunk/reconcile/statefulset.go +++ b/pkg/splunk/reconcile/statefulset.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package deploy +package reconcile import ( "context" @@ -28,32 +28,32 @@ import ( // StatefulSetPodManager is used to manage the pods within a StatefulSet type StatefulSetPodManager interface { - // Decommision pod and return true if complete - Decommission(n int32) (bool, error) + // PrepareScaleDown prepares pod to be removed via scale down event; it returns true when ready + PrepareScaleDown(n int32) (bool, error) - // Quarantine pod and return true if complete - Quarantine(n int32) (bool, error) + // PrepareRecycle prepares pod to be recycled for updates; it returns true when ready + PrepareRecycle(n int32) (bool, error) - // ReleaseQuarantine will release a quarantine and return true, if active; it returns false if none active - ReleaseQuarantine(n int32) (bool, error) + // FinishRecycle completes recycle event for pod and returns true, or returns false if nothing to do + FinishRecycle(n int32) (bool, error) } // DefaultStatefulSetPodManager is a simple StatefulSetPodManager that does nothing type DefaultStatefulSetPodManager struct{} -// Decommission for DefaultStatefulSetPodManager does nothing and returns true -func (mgr *DefaultStatefulSetPodManager) Decommission(n int32) (bool, error) { +// PrepareScaleDown for DefaultStatefulSetPodManager does nothing and returns true +func (mgr *DefaultStatefulSetPodManager) PrepareScaleDown(n int32) (bool, error) { return true, nil } -// Quarantine for DefaultStatefulSetPodManager does nothing and returns true -func (mgr *DefaultStatefulSetPodManager) Quarantine(n int32) (bool, error) { +// PrepareRecycle for DefaultStatefulSetPodManager does nothing and returns true +func (mgr *DefaultStatefulSetPodManager) PrepareRecycle(n int32) (bool, error) { return true, nil } -// ReleaseQuarantine for DefaultStatefulSetPodManager does nothing and returns false -func (mgr *DefaultStatefulSetPodManager) ReleaseQuarantine(n int32) (bool, error) { - return false, nil +// FinishRecycle for DefaultStatefulSetPodManager does nothing and returns false +func (mgr *DefaultStatefulSetPodManager) FinishRecycle(n int32) (bool, error) { + return true, nil } // ApplyStatefulSet creates or updates a Kubernetes StatefulSet @@ -78,23 +78,23 @@ func ApplyStatefulSet(c ControllerClient, revised *appsv1.StatefulSet) (enterpri if hasUpdates { // this updates the desired state template, but doesn't actually modify any pods // because we use an "OnUpdate" strategy https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#update-strategies - // note also that this ignores Replicas, which is handled below by ReconcileStatefulSetPods + // note also that this ignores Replicas, which is handled below by UpdateStatefulSetPods return enterprisev1.PhaseUpdating, UpdateResource(c, revised) } - // scaling and pod updates are handled by ReconcileStatefulSetPods + // scaling and pod updates are handled by UpdateStatefulSetPods return enterprisev1.PhaseReady, nil } -// ReconcileStatefulSetPods manages scaling and config updates for StatefulSets -func ReconcileStatefulSetPods(c ControllerClient, statefulSet *appsv1.StatefulSet, mgr StatefulSetPodManager, desiredReplicas int32) (enterprisev1.ResourcePhase, error) { +// UpdateStatefulSetPods manages scaling and config updates for StatefulSets +func UpdateStatefulSetPods(c ControllerClient, statefulSet *appsv1.StatefulSet, mgr StatefulSetPodManager, desiredReplicas int32) (enterprisev1.ResourcePhase, error) { - scopedLog := log.WithName("ReconcileStatefulSetPods").WithValues( + scopedLog := log.WithName("UpdateStatefulSetPods").WithValues( "name", statefulSet.GetObjectMeta().GetName(), "namespace", statefulSet.GetObjectMeta().GetNamespace()) // wait for all replicas ready - replicas := statefulSet.Status.Replicas + replicas := *statefulSet.Spec.Replicas readyReplicas := statefulSet.Status.ReadyReplicas if readyReplicas < replicas { scopedLog.Info("Waiting for pods to become ready") @@ -119,15 +119,15 @@ func ReconcileStatefulSetPods(c ControllerClient, statefulSet *appsv1.StatefulSe // check for scaling down if readyReplicas > desiredReplicas { - // decommission pod to prepare for removal + // prepare pod for removal via scale down n := readyReplicas - 1 - complete, err := mgr.Decommission(n) + ready, err := mgr.PrepareScaleDown(n) if err != nil { podName := fmt.Sprintf("%s-%d", statefulSet.GetName(), n) scopedLog.Error(err, "Unable to decommission Pod", "podName", podName) return enterprisev1.PhaseError, err } - if !complete { + if !ready { // wait until pod quarantine has completed before deleting it return enterprisev1.PhaseScalingDown, nil } @@ -155,13 +155,13 @@ func ReconcileStatefulSetPods(c ControllerClient, statefulSet *appsv1.StatefulSe // terminate pod if it has pending updates; k8s will start a new one with revised template if statefulSet.Status.UpdateRevision != "" && statefulSet.Status.UpdateRevision != pod.GetLabels()["controller-revision-hash"] { - // pod needs to be updated; first, quarantine it to prepare for restart - complete, err := mgr.Quarantine(n) + // pod needs to be updated; first, prepare it to be recycled + ready, err := mgr.PrepareRecycle(n) if err != nil { - scopedLog.Error(err, "Unable to quarantine Pod", "podName", podName) + scopedLog.Error(err, "Unable to prepare Pod for recycling", "podName", podName) return enterprisev1.PhaseError, err } - if !complete { + if !ready { // wait until pod quarantine has completed before deleting it return enterprisev1.PhaseUpdating, nil } @@ -181,14 +181,14 @@ func ReconcileStatefulSetPods(c ControllerClient, statefulSet *appsv1.StatefulSe return enterprisev1.PhaseUpdating, nil } - // check if pod was previously quarantined; if so, it's ok to release it - released, err := mgr.ReleaseQuarantine(n) + // check if pod was previously prepared for recycling; if so, complete + complete, err := mgr.FinishRecycle(n) if err != nil { - scopedLog.Error(err, "Unable to release Pod from quarantine", "podName", podName) + scopedLog.Error(err, "Unable to complete recycling of pod", "podName", podName) return enterprisev1.PhaseError, err } - if released { - // if pod was released, return and wait until next reconcile to let things settle down + if !complete { + // return and wait until next reconcile to let things settle down return enterprisev1.PhaseUpdating, nil } } diff --git a/pkg/splunk/reconcile/statefulset_test.go b/pkg/splunk/reconcile/statefulset_test.go index a37399764..e7dfa794a 100644 --- a/pkg/splunk/reconcile/statefulset_test.go +++ b/pkg/splunk/reconcile/statefulset_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package deploy +package reconcile import ( "testing" diff --git a/pkg/splunk/reconcile/util.go b/pkg/splunk/reconcile/util.go index 3e8707fbb..3f6bde78e 100644 --- a/pkg/splunk/reconcile/util.go +++ b/pkg/splunk/reconcile/util.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package deploy +package reconcile import ( "context" @@ -23,13 +23,19 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + //stdlog "log" + //"github.com/go-logr/stdr" "github.com/splunk/splunk-operator/pkg/splunk/resources" ) -// logger used by splunk.deploy package -var log = logf.Log.WithName("splunk.deploy") +// kubernetes logger used by splunk.reconcile package +var log = logf.Log.WithName("splunk.reconcile") + +// simple stdout logger, used for debugging +//var log = stdr.New(stdlog.New(os.Stderr, "", stdlog.LstdFlags|stdlog.Lshortfile)).WithName("splunk.reconcile") // The ResourceObject type implements methods of runtime.Object and GetObjectMeta() type ResourceObject interface { @@ -47,6 +53,7 @@ func CreateResource(client ControllerClient, obj ResourceObject) error { scopedLog := log.WithName("CreateResource").WithValues( "name", obj.GetObjectMeta().GetName(), "namespace", obj.GetObjectMeta().GetNamespace()) + err := client.Create(context.TODO(), obj) if err != nil && !errors.IsAlreadyExists(err) { diff --git a/pkg/splunk/reconcile/util_test.go b/pkg/splunk/reconcile/util_test.go index 3d38ee011..9e194de0b 100644 --- a/pkg/splunk/reconcile/util_test.go +++ b/pkg/splunk/reconcile/util_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package deploy +package reconcile import ( "context" @@ -258,7 +258,7 @@ func (c *mockClient) checkCalls(t *testing.T, testname string, wantCalls map[str } if notEmptyWantCalls != len(c.calls) { - t.Errorf("%s: MockClient functions called = %d; want %d", testname, len(c.calls), len(wantCalls)) + t.Errorf("%s: MockClient functions called = %d; want %d: calls=%v", testname, len(c.calls), len(wantCalls), c.calls) } }