From cfc6a23248230e45cfca4adbe79a7f65efe3bdc5 Mon Sep 17 00:00:00 2001 From: Dejan Golja Date: Mon, 1 Aug 2016 23:22:54 +1000 Subject: [PATCH] - refactor aws function to a separate file - add status as a custom type to improve code readability --- aws.go | 210 ++++++++++++++++++++++++++++++++++++++++++++++++ flywheel.go | 212 ++----------------------------------------------- healthcheck.go | 12 +-- 3 files changed, 223 insertions(+), 211 deletions(-) create mode 100644 aws.go diff --git a/aws.go b/aws.go new file mode 100644 index 0000000..54a917a --- /dev/null +++ b/aws.go @@ -0,0 +1,210 @@ +package flywheel + +import ( + "log" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/autoscaling" + "github.com/aws/aws-sdk-go/service/ec2" +) + +// Start all the resources managed by the flywheel. +func (fw *Flywheel) Start() error { + fw.lastStarted = time.Now() + log.Print("Startup beginning") + + var err error + err = fw.startInstances() + + if err == nil { + err = fw.unterminateAutoScaling() + } + if err == nil { + err = fw.startAutoScaling() + } + + if err != nil { + log.Printf("Error starting: %v", err) + return err + } + + fw.ready = false + fw.stopAt = time.Now().Add(fw.idleTimeout) + fw.status = STARTING + return nil +} + +// Start EC2 instances +func (fw *Flywheel) startInstances() error { + if len(fw.config.Instances) == 0 { + return nil + } + log.Printf("Starting instances %v", fw.config.Instances) + _, err := fw.ec2.StartInstances( + &ec2.StartInstancesInput{ + InstanceIds: fw.config.AwsInstances(), + }, + ) + return err +} + +// UnterminateAutoScaling - Restore autoscaling group instances +func (fw *Flywheel) unterminateAutoScaling() error { + var err error + for groupName, size := range fw.config.AutoScaling.Terminate { + log.Printf("Restoring autoscaling group %s", groupName) + _, err = fw.autoscaling.UpdateAutoScalingGroup( + &autoscaling.UpdateAutoScalingGroupInput{ + AutoScalingGroupName: &groupName, + MaxSize: &size, + MinSize: &size, + }, + ) + if err != nil { + return err + } + } + return nil +} + +// Start EC2 instances in a suspended autoscale group +// @note The autoscale group isn't unsuspended here. It's done by the +// healthcheck once all the instances are healthy. +func (fw *Flywheel) startAutoScaling() error { + for _, groupName := range fw.config.AutoScaling.Stop { + log.Printf("Starting autoscaling group %s", groupName) + + resp, err := fw.autoscaling.DescribeAutoScalingGroups( + &autoscaling.DescribeAutoScalingGroupsInput{ + AutoScalingGroupNames: []*string{&groupName}, + }, + ) + if err != nil { + return err + } + + group := resp.AutoScalingGroups[0] + + instanceIds := []*string{} + for _, instance := range group.Instances { + instanceIds = append(instanceIds, instance.InstanceId) + } + + _, err = fw.ec2.StartInstances( + &ec2.StartInstancesInput{ + InstanceIds: instanceIds, + }, + ) + if err != nil { + return err + } + } + + return nil +} + +// Stop all resources managed by the flywheel +func (fw *Flywheel) Stop() error { + fw.lastStopped = time.Now() + + var err error + err = fw.stopInstances() + + if err == nil { + err = fw.terminateAutoScaling() + } + if err == nil { + err = fw.stopAutoScaling() + } + + if err != nil { + log.Printf("Error stopping: %v", err) + return err + } + + fw.ready = false + fw.status = STOPPING + fw.stopAt = fw.lastStopped + return nil +} + +// Stop EC2 instances +func (fw *Flywheel) stopInstances() error { + if len(fw.config.Instances) == 0 { + return nil + } + log.Printf("Stopping instances %v", fw.config.Instances) + _, err := fw.ec2.StopInstances( + &ec2.StopInstancesInput{ + InstanceIds: fw.config.AwsInstances(), + }, + ) + return err +} + +// Suspend ReplaceUnhealthy in an autoscale group and stop the instances. +func (fw *Flywheel) stopAutoScaling() error { + for _, groupName := range fw.config.AutoScaling.Stop { + log.Printf("Stopping autoscaling group %s", groupName) + + resp, err := fw.autoscaling.DescribeAutoScalingGroups( + &autoscaling.DescribeAutoScalingGroupsInput{ + AutoScalingGroupNames: []*string{&groupName}, + }, + ) + if err != nil { + return err + } + + group := resp.AutoScalingGroups[0] + + _, err = fw.autoscaling.SuspendProcesses( + &autoscaling.ScalingProcessQuery{ + AutoScalingGroupName: group.AutoScalingGroupName, + ScalingProcesses: []*string{ + aws.String("ReplaceUnhealthy"), + }, + }, + ) + if err != nil { + return err + } + + instanceIds := []*string{} + for _, instance := range group.Instances { + instanceIds = append(instanceIds, instance.InstanceId) + } + + _, err = fw.ec2.StopInstances( + &ec2.StopInstancesInput{ + InstanceIds: instanceIds, + }, + ) + if err != nil { + return err + } + } + + return nil +} + +// Reduce autoscaling min/max instances to 0, causing the instances to be terminated. +func (fw *Flywheel) terminateAutoScaling() error { + var err error + var zero int64 + for groupName := range fw.config.AutoScaling.Terminate { + log.Printf("Terminating autoscaling group %s", groupName) + _, err = fw.autoscaling.UpdateAutoScalingGroup( + &autoscaling.UpdateAutoScalingGroupInput{ + AutoScalingGroupName: &groupName, + MaxSize: &zero, + MinSize: &zero, + }, + ) + if err != nil { + return err + } + } + return nil +} diff --git a/flywheel.go b/flywheel.go index daa0a17..d35fbfc 100644 --- a/flywheel.go +++ b/flywheel.go @@ -29,7 +29,7 @@ type Ping struct { // Pong - result of the ping request type Pong struct { - Status int `json:"-"` + Status Status `json:"-"` StatusName string `json:"status"` Err error `json:"error,omitempty"` LastStarted time.Time `json:"last-started,omitempty"` @@ -42,7 +42,7 @@ type Flywheel struct { config *Config running bool pings chan Ping - status int + status Status ready bool stopAt time.Time lastStarted time.Time @@ -81,7 +81,7 @@ func (fw *Flywheel) ProxyEndpoint(hostname string) string { // Spin - Runs the main loop for the Flywheel. func (fw *Flywheel) Spin() { - hchan := make(chan int, 1) + hchan := make(chan Status, 1) go fw.HealthWatcher(hchan) @@ -94,7 +94,7 @@ func (fw *Flywheel) Spin() { fw.Poll() case status := <-hchan: if fw.status != status { - log.Printf("Healthcheck - status is now %v", StatusString(status)) + log.Printf("Healthcheck - status is now %v", status) // Status may change from STARTED to UNHEALTHY to STARTED due // to things like AWS RequestLimitExceeded errors. // If there is an active timeout, keep it instead of resetting. @@ -136,7 +136,7 @@ func (fw *Flywheel) RecvPing(ping *Ping) { } pong.Status = fw.status - pong.StatusName = StatusString(fw.status) + pong.StatusName = fw.status.String() pong.LastStarted = fw.lastStarted pong.LastStopped = fw.lastStopped pong.StopAt = fw.stopAt @@ -170,206 +170,6 @@ func (fw *Flywheel) Poll() { } } -// Start all the resources managed by the flywheel. -func (fw *Flywheel) Start() error { - fw.lastStarted = time.Now() - log.Print("Startup beginning") - - var err error - err = fw.startInstances() - - if err == nil { - err = fw.unterminateAutoScaling() - } - if err == nil { - err = fw.startAutoScaling() - } - - if err != nil { - log.Printf("Error starting: %v", err) - return err - } - - fw.ready = false - fw.stopAt = time.Now().Add(fw.idleTimeout) - fw.status = STARTING - return nil -} - -// Start EC2 instances -func (fw *Flywheel) startInstances() error { - if len(fw.config.Instances) == 0 { - return nil - } - log.Printf("Starting instances %v", fw.config.Instances) - _, err := fw.ec2.StartInstances( - &ec2.StartInstancesInput{ - InstanceIds: fw.config.AwsInstances(), - }, - ) - return err -} - -// UnterminateAutoScaling - Restore autoscaling group instances -func (fw *Flywheel) unterminateAutoScaling() error { - var err error - for groupName, size := range fw.config.AutoScaling.Terminate { - log.Printf("Restoring autoscaling group %s", groupName) - _, err = fw.autoscaling.UpdateAutoScalingGroup( - &autoscaling.UpdateAutoScalingGroupInput{ - AutoScalingGroupName: &groupName, - MaxSize: &size, - MinSize: &size, - }, - ) - if err != nil { - return err - } - } - return nil -} - -// Start EC2 instances in a suspended autoscale group -// @note The autoscale group isn't unsuspended here. It's done by the -// healthcheck once all the instances are healthy. -func (fw *Flywheel) startAutoScaling() error { - for _, groupName := range fw.config.AutoScaling.Stop { - log.Printf("Starting autoscaling group %s", groupName) - - resp, err := fw.autoscaling.DescribeAutoScalingGroups( - &autoscaling.DescribeAutoScalingGroupsInput{ - AutoScalingGroupNames: []*string{&groupName}, - }, - ) - if err != nil { - return err - } - - group := resp.AutoScalingGroups[0] - - instanceIds := []*string{} - for _, instance := range group.Instances { - instanceIds = append(instanceIds, instance.InstanceId) - } - - _, err = fw.ec2.StartInstances( - &ec2.StartInstancesInput{ - InstanceIds: instanceIds, - }, - ) - if err != nil { - return err - } - } - - return nil -} - -// Stop all resources managed by the flywheel -func (fw *Flywheel) Stop() error { - fw.lastStopped = time.Now() - - var err error - err = fw.stopInstances() - - if err == nil { - err = fw.terminateAutoScaling() - } - if err == nil { - err = fw.stopAutoScaling() - } - - if err != nil { - log.Printf("Error stopping: %v", err) - return err - } - - fw.ready = false - fw.status = STOPPING - fw.stopAt = fw.lastStopped - return nil -} - -// Stop EC2 instances -func (fw *Flywheel) stopInstances() error { - if len(fw.config.Instances) == 0 { - return nil - } - log.Printf("Stopping instances %v", fw.config.Instances) - _, err := fw.ec2.StopInstances( - &ec2.StopInstancesInput{ - InstanceIds: fw.config.AwsInstances(), - }, - ) - return err -} - -// Suspend ReplaceUnhealthy in an autoscale group and stop the instances. -func (fw *Flywheel) stopAutoScaling() error { - for _, groupName := range fw.config.AutoScaling.Stop { - log.Printf("Stopping autoscaling group %s", groupName) - - resp, err := fw.autoscaling.DescribeAutoScalingGroups( - &autoscaling.DescribeAutoScalingGroupsInput{ - AutoScalingGroupNames: []*string{&groupName}, - }, - ) - if err != nil { - return err - } - - group := resp.AutoScalingGroups[0] - - _, err = fw.autoscaling.SuspendProcesses( - &autoscaling.ScalingProcessQuery{ - AutoScalingGroupName: group.AutoScalingGroupName, - ScalingProcesses: []*string{ - aws.String("ReplaceUnhealthy"), - }, - }, - ) - if err != nil { - return err - } - - instanceIds := []*string{} - for _, instance := range group.Instances { - instanceIds = append(instanceIds, instance.InstanceId) - } - - _, err = fw.ec2.StopInstances( - &ec2.StopInstancesInput{ - InstanceIds: instanceIds, - }, - ) - if err != nil { - return err - } - } - - return nil -} - -// Reduce autoscaling min/max instances to 0, causing the instances to be terminated. -func (fw *Flywheel) terminateAutoScaling() error { - var err error - var zero int64 - for groupName := range fw.config.AutoScaling.Terminate { - log.Printf("Terminating autoscaling group %s", groupName) - _, err = fw.autoscaling.UpdateAutoScalingGroup( - &autoscaling.UpdateAutoScalingGroupInput{ - AutoScalingGroupName: &groupName, - MaxSize: &zero, - MinSize: &zero, - }, - ) - if err != nil { - return err - } - } - return nil -} - // WriteStatusFile - Before we exit the application we write the current state func (fw *Flywheel) WriteStatusFile(statusFile string) { var pong Pong @@ -382,7 +182,7 @@ func (fw *Flywheel) WriteStatusFile(statusFile string) { defer fd.Close() pong.Status = fw.status - pong.StatusName = StatusString(fw.status) + pong.StatusName = fw.status.String() pong.LastStarted = fw.lastStarted pong.LastStopped = fw.lastStopped diff --git a/healthcheck.go b/healthcheck.go index 3dd5583..e8c3487 100644 --- a/healthcheck.go +++ b/healthcheck.go @@ -9,18 +9,20 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" ) +type Status uint + // Different state of the systems const ( - STOPPED = iota + STOPPED Status = iota STARTING STARTED STOPPING UNHEALTHY ) -// StatusString - Working with integer statuses is mostly better, but it's +// String - Working with integer statuses is mostly better, but it's // occasionally necessary to output the status name. -func StatusString(n int) string { +func (n Status) String() string { switch n { case STOPPED: return "STOPPED" @@ -39,7 +41,7 @@ func StatusString(n int) string { // HealthWatcher - Check the status of the instances. Currently checks if they are "ready"; all // stopped or all started. Will need to be extended to determine actual status. -func (fw *Flywheel) HealthWatcher(out chan<- int) { +func (fw *Flywheel) HealthWatcher(out chan<- Status) { out <- fw.CheckAll() ticker := time.NewTicker(fw.hcInterval) @@ -53,7 +55,7 @@ func (fw *Flywheel) HealthWatcher(out chan<- int) { // CheckAll - check asg/instance state // TODO - add more information what is unhealthy -func (fw *Flywheel) CheckAll() int { +func (fw *Flywheel) CheckAll() Status { health := make(map[string]int) err := fw.checkInstances(health)