Skip to content

Commit

Permalink
Merge branch '3810-cordon-control'
Browse files Browse the repository at this point in the history
  • Loading branch information
bboreham committed May 20, 2021
2 parents 51da22c + f800f64 commit fc518cb
Show file tree
Hide file tree
Showing 27 changed files with 4,456 additions and 3 deletions.
11 changes: 10 additions & 1 deletion examples/k8s/cluster-role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ rules:
- pods/log
- replicationcontrollers
- services
- nodes
- namespaces
- persistentvolumes
- persistentvolumeclaims
Expand Down Expand Up @@ -95,3 +94,13 @@ rules:
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- list
- watch
- update
- patch
7 changes: 7 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
camlistore.org v0.0.0-20171230002226-a5a65f0d8b22 h1:VP9VuyosMHmS9zdzd5Co9TJKWPbMTfmtKc/XWctszyQ=
camlistore.org v0.0.0-20171230002226-a5a65f0d8b22/go.mod h1:mzAP6ICVzPdfO0f3N9hAVWhO7qplHF7mbFhGsGdErTI=
cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ=
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/DataDog/datadog-go v2.2.0+incompatible h1:V5BKkxACZLjzHjSgBbr2gvLA2Ae49yhc6CSY7MLy5k4=
github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6nK2Q=
github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
Expand Down Expand Up @@ -39,7 +41,9 @@ github.com/certifi/gocertifi v0.0.0-20150906030631-84c0a38a18fc h1:zSPFItDTOJZPd
github.com/certifi/gocertifi v0.0.0-20150906030631-84c0a38a18fc/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible h1:C29Ae4G5GtYyYMm1aztcyj/J5ckgJm2zwdDajFbx1NY=
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
github.com/circonus-labs/circonusllhist v0.1.3 h1:TJH+oke8D16535+jHExHj4nQvzlZrj7ug5D7I/orNUA=
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
Expand Down Expand Up @@ -97,6 +101,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
github.com/golang/groupcache v0.0.0-20171101203131-84a468cf14b4 h1:6o8aP0LGMKzo3NzwhhX6EJsiJ3ejmj+9yA/3p8Fjjlw=
github.com/golang/groupcache v0.0.0-20171101203131-84a468cf14b4/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
Expand Down Expand Up @@ -134,6 +139,7 @@ github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxB
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v1.1.5 h1:9byZdVjKTe5mce63pRVNP1L7UAmdHOTEMGehn6KvJWs=
github.com/hashicorp/go-msgpack v1.1.5/go.mod h1:gWVc3sv/wbDmR3rQsj1CAktEZzoz1YNK9NfGLXJ69/4=
github.com/hashicorp/go-retryablehttp v0.5.3 h1:QlWt0KvWT0lq8MFppF9tsJGF+ynG7ztc2KIPhzRGk7s=
github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM=
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
Expand Down Expand Up @@ -274,6 +280,7 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ=
github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926 h1:G3dpKMzFDjgEh2q1Z7zUUtKa8ViPtH+ocF0bE0g00O8=
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
github.com/tylerb/graceful v1.2.13 h1:yKdTh6eHcWdD8Jm3wxgJ6pNf8Lb3wwbV4Ip8fHbeMLE=
github.com/tylerb/graceful v1.2.13/go.mod h1:LPYTbOYmUTdabwRt0TGhLllQ0MUNbs0Y5q1WXJOI9II=
Expand Down
34 changes: 34 additions & 0 deletions probe/kubernetes/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,10 @@ type Client interface {
DeleteVolumeSnapshot(namespaceID, volumeSnapshotID string) error
ScaleUp(namespaceID, id string) error
ScaleDown(namespaceID, id string) error
// Cordon or Uncordon a node based on whether `desired` is true or false respectively.
CordonNode(name string, desired bool) error
// Returns a list of kubernetes nodes.
GetNodes() ([]apiv1.Node, error)
}

// ResourceMap is the mapping of resource and their GroupKind
Expand Down Expand Up @@ -624,3 +628,33 @@ func (c *client) modifyScale(namespaceID, id string, f func(*autoscalingv1.Scale
func (c *client) Stop() {
close(c.quit)
}

func (c *client) CordonNode(name string, desired bool) error {
node, err := c.client.CoreV1().Nodes().Get(name, metav1.GetOptions{})
if err != nil {
return err
}

helper := newCordonHelper(node)
if updateRequired := helper.updateIfRequired(desired); !updateRequired {
return nil
}

err, patchErr := helper.patchOrReplace(c.client, false)
if patchErr != nil {
return patchErr
}
if err != nil {
return err
}
return nil
}

func (c *client) GetNodes() ([]apiv1.Node, error) {
l, err := c.client.CoreV1().Nodes().List(metav1.ListOptions{})
if err != nil {
return nil, err
}

return l.Items, nil
}
27 changes: 27 additions & 0 deletions probe/kubernetes/controls.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ const (
DeleteVolumeSnapshot = report.KubernetesDeleteVolumeSnapshot
ScaleUp = report.KubernetesScaleUp
ScaleDown = report.KubernetesScaleDown
CordonNode = report.KubernetesCordonNode
UncordonNode = report.KubernetesUncordonNode
)

// GroupName and version used by CRDs
Expand Down Expand Up @@ -474,6 +476,17 @@ func (r *Reporter) CaptureJob(f func(xfer.Request, string, string) xfer.Response
}
}

// CaptureNode is exported for testing
func (r *Reporter) CaptureNode(f func(xfer.Request, string) xfer.Response) func(xfer.Request) xfer.Response {
return func(req xfer.Request) xfer.Response {
nodeID, ok := report.ParseHostNodeID(req.NodeID)
if !ok {
return xfer.ResponseErrorf("Invalid ID: %s", req.NodeID)
}
return f(req, nodeID)
}
}

// ScaleUp is the control to scale up a deployment
func (r *Reporter) ScaleUp(req xfer.Request, namespace, id string) xfer.Response {
return xfer.ResponseError(r.client.ScaleUp(namespace, id))
Expand All @@ -484,6 +497,16 @@ func (r *Reporter) ScaleDown(req xfer.Request, namespace, id string) xfer.Respon
return xfer.ResponseError(r.client.ScaleDown(namespace, id))
}

// CordonNode is the control to cordon a node.
func (r *Reporter) CordonNode(req xfer.Request, name string) xfer.Response {
return xfer.ResponseError(r.client.CordonNode(name, true))
}

// UncordonNode is the control to un-cordon a node.
func (r *Reporter) UncordonNode(req xfer.Request, name string) xfer.Response {
return xfer.ResponseError(r.client.CordonNode(name, false))
}

func (r *Reporter) registerControls() {
controls := map[string]xfer.ControlHandlerFunc{
CloneVolumeSnapshot: r.CaptureVolumeSnapshot(r.cloneVolumeSnapshot),
Expand All @@ -494,6 +517,8 @@ func (r *Reporter) registerControls() {
DeleteVolumeSnapshot: r.CaptureVolumeSnapshot(r.deleteVolumeSnapshot),
ScaleUp: r.CaptureDeployment(r.ScaleUp),
ScaleDown: r.CaptureDeployment(r.ScaleDown),
CordonNode: r.CaptureNode(r.CordonNode),
UncordonNode: r.CaptureNode(r.UncordonNode),
}
r.handlerRegistry.Batch(nil, controls)
}
Expand All @@ -508,6 +533,8 @@ func (r *Reporter) deregisterControls() {
DeleteVolumeSnapshot,
ScaleUp,
ScaleDown,
CordonNode,
UncordonNode,
}
r.handlerRegistry.Batch(controls, nil)
}
87 changes: 87 additions & 0 deletions probe/kubernetes/cordon.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
/*
Copied from
https://github.com/kubernetes/kubectl/blob/master/pkg/drain/cordon.go
at commit f9460c53339c4bb60b20031e3c6125e8bac679e2 to add node cordon feature.
*/
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubernetes

import (
"encoding/json"

apiv1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/strategicpatch"
"k8s.io/client-go/kubernetes"
)

// CordonHelper wraps functionality to cordon/uncordon nodes
type cordonHelper struct {
node *apiv1.Node
desired bool
}

// NewCordonHelper returns a new CordonHelper
func newCordonHelper(node *apiv1.Node) *cordonHelper {
return &cordonHelper{
node: node,
}
}

// UpdateIfRequired returns true if c.node.Spec.Unschedulable isn't already set,
// or false when no change is needed
func (c *cordonHelper) updateIfRequired(desired bool) bool {
c.desired = desired

return c.node.Spec.Unschedulable != c.desired
}

// PatchOrReplace uses given clientset to update the node status, either by patching or
// updating the given node object; it may return error if the object cannot be encoded as
// JSON, or if either patch or update calls fail; it will also return a second error
// whenever creating a patch has failed
func (c *cordonHelper) patchOrReplace(clientset kubernetes.Interface, serverDryRun bool) (error, error) {
client := clientset.CoreV1().Nodes()

oldData, err := json.Marshal(c.node)
if err != nil {
return err, nil
}

c.node.Spec.Unschedulable = c.desired

newData, err := json.Marshal(c.node)
if err != nil {
return err, nil
}

patchBytes, patchErr := strategicpatch.CreateTwoWayMergePatch(oldData, newData, c.node)
if patchErr == nil {
_, err = client.Patch(c.node.Name, types.StrategicMergePatchType, patchBytes)
} else {
updateOptions := metav1.UpdateOptions{}
if serverDryRun {
updateOptions.DryRun = []string{metav1.DryRunAll}
}
_, err = client.Update(c.node)
}
return err, patchErr
}
50 changes: 50 additions & 0 deletions probe/kubernetes/reporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,21 @@ var (
Icon: "fa fa-file-text",
Rank: 2,
}

CordonControl = []report.Control{
{
ID: CordonNode,
Human: "Cordon",
Icon: "fa fa-toggle-off",
Rank: 1,
},
{
ID: UncordonNode,
Human: "Uncordon",
Icon: "fa fa-toggle-on",
Rank: 0,
},
}
)

// Reporter generate Reports containing Container and ContainerImage topologies
Expand Down Expand Up @@ -345,6 +360,10 @@ func (r *Reporter) Report() (report.Report, error) {
if err != nil {
return result, err
}
hostTopology, err := r.hostTopology()
if err != nil {
return result, err
}

result.Pod = result.Pod.Merge(podTopology)
result.Service = result.Service.Merge(serviceTopology)
Expand All @@ -359,6 +378,8 @@ func (r *Reporter) Report() (report.Report, error) {
result.VolumeSnapshot = result.VolumeSnapshot.Merge(volumeSnapshotTopology)
result.VolumeSnapshotData = result.VolumeSnapshotData.Merge(volumeSnapshotDataTopology)
result.Job = result.Job.Merge(jobTopology)
result.Host = result.Host.Merge(hostTopology)

return result, nil
}

Expand Down Expand Up @@ -678,3 +699,32 @@ func (r *Reporter) namespaceTopology() (report.Topology, error) {
})
return result, err
}

func (r *Reporter) hostTopology() (report.Topology, error) {
result := report.MakeTopology()
// Add buttons for Host view, with the ID of the Kubernetes probe
for _, control := range CordonControl {
control.ProbeID = r.probeID
result.Controls.AddControl(control)
}

nodes, err := r.client.GetNodes()
if err != nil {
return result, err
}

for _, n := range nodes {
var activeControl string
if n.Spec.Unschedulable {
activeControl = UncordonNode
} else {
activeControl = CordonNode
}
result.AddNode(
report.MakeNode(report.MakeHostNodeID(n.Name)).
WithTopology(report.Host).
WithLatestActiveControls(activeControl),
)
}
return result, nil
}
8 changes: 8 additions & 0 deletions probe/kubernetes/reporter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,14 @@ func (c *mockClient) Describe(namespaceID, resourceID string, groupKind schema.G
return nil, nil
}

func (c *mockClient) CordonNode(name string, desired bool) error {
return nil
}

func (c *mockClient) GetNodes() ([]apiv1.Node, error) {
return nil, nil
}

type mockPipeClient map[string]xfer.Pipe

func (c mockPipeClient) PipeConnection(appID, id string, pipe xfer.Pipe) error {
Expand Down
3 changes: 3 additions & 0 deletions render/detailed/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,9 @@ func controlsFor(topology report.Topology, nodeID string) []ControlInstance {
}
for _, controlID := range node.ActiveControls() {
if control, ok := topology.Controls[controlID]; ok {
if control.ProbeID != "" { // does this Control have an override for the node probe?
probeID = control.ProbeID
}
result = append(result, ControlInstance{
ProbeID: probeID,
NodeID: nodeID,
Expand Down
1 change: 1 addition & 0 deletions report/controls.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ type Control struct {
Icon string `json:"icon"` // from https://fortawesome.github.io/Font-Awesome/cheatsheet/ please
Confirmation string `json:"confirmation,omitempty"`
Rank int `json:"rank"`
ProbeID string `json:"probeId,omitempty"`
}

// Merge merges other with cs, returning a fresh Controls.
Expand Down
2 changes: 2 additions & 0 deletions report/map_keys.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,8 @@ const (
KubernetesCloneVolumeSnapshot = "kubernetes_clone_volume_snapshot"
KubernetesDeleteVolumeSnapshot = "kubernetes_delete_volume_snapshot"
KubernetesDescribe = "kubernetes_describe"
KubernetesCordonNode = "kubernetes_cordon_node"
KubernetesUncordonNode = "kubernetes_uncordon_node"
// probe/awsecs
ECSCluster = "ecs_cluster"
ECSCreatedAt = "ecs_created_at"
Expand Down
Loading

0 comments on commit fc518cb

Please sign in to comment.