Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow the node name to be set directly #8

Merged
merged 1 commit into from
Nov 16, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ Notable changes between versions.

## Latest

* Allow the node name to be set directly
* Add `-node` flag to set the Kubernetes node name
* Default to using the `HOSTNAME` environment variable
* Add Slack threading and reactions ([#7](https://github.com/poseidon/scuttle/pull/7))
* Add `-channel-id` flag to set a channel
* Add `-token` flag to set a Slack token
Expand Down
16 changes: 9 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ systemd:
--env KUBECONFIG=/var/lib/kubelet/kubeconfig \
-v /var/lib/kubelet:/var/lib/kubelet:ro,z \
--stop-timeout=60 \
quay.io/poseidon/scuttle:261fc0f \
quay.io/poseidon/scuttle:v0.1.0 \
-platform=aws
ExecStop=/usr/bin/podman stop scuttle
TimeoutStopSec=180
Expand All @@ -91,13 +91,14 @@ Configure via flags.

| flag | description | default |
|-------------|--------------|--------------|
| -node | Kubernetes node name | $HOSTNAME |
| -platform | Platform to poll for termination notices | none |
| -channel-id | Slack Channel ID | "" |
| -token | Slack Bot Token | "" |
| -webhook | Slack Webhook URL | "" |
| -uncordon | Uncordon node on start | true |
| -drain | Drain node on stop | true |
| -delete | Delete node on stop | true |
| -channel-id | Slack Channel ID | "" |
| -token | Slack Bot Token | "" |
| -webhook | Slack Webhook URL | "" |
| -log-level | Logger level | info |
| -version | Show version | NA |
| -help | Show help | NA |
Expand Down Expand Up @@ -177,9 +178,10 @@ INFO[0004] scuttle: stopping... hostname=ip-10-0-35-141
INFO[0004] scuttle: cordoning node hostname=ip-10-0-35-141
INFO[0004] drainer: cordoning node node=ip-10-0-35-141
INFO[0004] scuttle: draining node hostname=ip-10-0-35-141
WARN[0005] WARNING: ignoring DaemonSet-managed Pods: kube-system/cilium-gpdk8, kube-system/kube-proxy-cvwx5, monitoring/promtail-k9slg
INFO[0005] evicting pod default/redacted
INFO[0026] drainer: evicting pod node=ip-10-0-35-141 pod=redacted
INFO[0027] drainer: evicting pod node=ip-10-0-35-141 pod=redacted
...
INFO[0009] scuttle: deleting node hostname=ip-10-0-35-141
INFO[0027] drainer: drained node node=ip-10-0-35-141
INFO[0027] scuttle: deleting node hostname=ip-10-0-35-141
INFO[0009] done
```
14 changes: 9 additions & 5 deletions cmd/scuttle/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ var (

func main() {
flags := struct {
nodeName string
channel string
token string
webhook string
Expand All @@ -40,6 +41,7 @@ func main() {
help bool
}{}

flag.StringVar(&flags.nodeName, "node", "", "Kubernetes node name (defaults to $HOSTNAME)")
flag.StringVar(&flags.platform, "platform", "none", "Set platform (none, aws, azure) to poll termination notices")
flag.BoolVar(&flags.uncordon, "uncordon", true, "Enabling uncordoning node on start")
flag.BoolVar(&flags.drain, "drain", true, "Enabling draining node on stop")
Expand Down Expand Up @@ -91,22 +93,24 @@ func main() {
stop()
}()

// Termination watcher
// termination watcher
scuttle, err := sctl.New(&sctl.Config{
Logger: log,
Channel: flags.channel,
Token: flags.token,
Webhook: flags.webhook,
NodeName: flags.nodeName,
Platform: flags.platform,
ShouldUncordon: flags.uncordon,
ShouldDrain: flags.drain,
ShouldDelete: flags.delete,
// Slack
Channel: flags.channel,
Token: flags.token,
Webhook: flags.webhook,
})
if err != nil {
log.Fatalf("main: scuttle New error: %v", err)
}

// watch for spot termination notice
// watch for termination
log.Infof("main: starting scuttle")
err = scuttle.Run(ctx)
if err != nil {
Expand Down
31 changes: 19 additions & 12 deletions internal/scuttle.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,15 @@ const (
// Config configures a Scuttle
type Config struct {
Logger *logrus.Logger
Channel string
Token string
Webhook string
NodeName string
Platform string
ShouldUncordon bool
ShouldDrain bool
ShouldDelete bool
// Slack
Channel string
Token string
Webhook string
}

// Scuttle watches for termination notices and performs Kubelet teardown actions.
Expand All @@ -55,8 +57,11 @@ func New(config *Config) (*Scuttle, error) {
return nil, fmt.Errorf("scuttle: logger must be non-nil")
}

// Use HOSTNAME to identify Kubelet node
hostname := os.Getenv("HOSTNAME")
hostname := config.NodeName
if hostname == "" {
// fallback to HOSTNAME to identify Kubelet node
hostname = os.Getenv("HOSTNAME")
}

// Kubernetes client from kubeconfig or service account (in-cluster)
kubeconfigPath := os.Getenv("KUBECONFIG")
Expand All @@ -65,20 +70,22 @@ func New(config *Config) (*Scuttle, error) {
return nil, fmt.Errorf("scuttle: error creating Kubernetes client: %v", err)
}

w := &Scuttle{
// initialize Slack client if token is set
var slackClient *slack.Client
if config.Token != "" {
slackClient = slack.New(config.Token)
}

return &Scuttle{
hostname: hostname,
config: config,
log: config.Logger,
client: &http.Client{
Timeout: 2 * time.Second,
},
kubeClient: kubeClient,
}
if config.Token != "" {
w.slack = slack.New(config.Token)
}

return w, nil
slack: slackClient,
}, nil
}

// Run runs the spot termination watch loop.
Expand Down
1 change: 0 additions & 1 deletion internal/slack.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,6 @@ func (w *Scuttle) notifySlack(action Notification, node string, thread string) s
}

opts := []slack.MsgOption{
//slack.MsgOptionText(text, true),
slack.MsgOptionAttachments(attachment),
}
if thread != "" {
Expand Down