Merge branch 'master' into issue194

.github/CHANGELOG.md

@@ -1,6 +1,20 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
+## [v0.18] - 2021-03-23
+
+8b2d320 Fix for Launch definition validation. Consider only the "InService" instances. (#197)
+42f810c Fail the CR for drain failures, when IgnoreDrainFailures isn't set. (#185)
+f5c9457 output can contain other messages from API Server, so be more relaxed (#174)
+391b2fb Expose template list and other execution errors to logs (#166)
+757b669 Bump golang and busybox (#172)
+b8f69e8 Add instance id to the logs (#173)
+ac7be6b Fix namespaced name order (#170)
+51f469d use standard fmt.Errorf to format error message; unify error format (#171)
+b552c69 Bump dependencies. (#169)
+36a2784 Remove separate module for pkg/log (#168)
+237f93d Move constants to types so that they can be reused (#167)
+
 ## [v0.17] - 2020-12-11
 
 * aa2b73b - use NamespacedName (#160)

.github/CONTRIBUTING.md

@@ -1,14 +1,18 @@
 # How to contribute
 
-1. Fork it (<https://github.com/keikoproj/upgrade-manager/fork>)
-2. Add your fork as a git remote named, say `reviews`.
-2. Open an issue and discuss the feature / bug
-3. Create your feature branch in your fork (`git checkout -b feature/fooBar`)
-4. Commit your changes (`git commit -am 'Add some fooBar'`)
-5. Add a "Testing Done" section in the commit message. Be as explicit as possible about all the manual and automated tests performed.
-6. Push to the branch (`git push reviews feature/fooBar`)
-7. Make sure unit tests and any static-analysis (linting) tests are passing
-8. Create a new Pull Request
+## Development
+- Open an issue and discuss the feature / bug you want
+- Fork (<https://github.com/keikoproj/upgrade-manager/fork>)
+- Clone your fork
+- Install [kubebuilder](https://book.kubebuilder.io/quick-start.html)
+- `make test` to ensure everything is working
+- Create your feature branch (`git checkout -b feature/fooBar`)
+- Implement your change and add tests 
+- `make test` to ensure everything is working 
+- Commit your changes (`git commit -am 'Add some fooBar'`)
+- Add a "Testing Done" section in the commit message. Be as explicit as possible about all the manual and automated tests performed.
+- Push to the branch (`git push origin feature/fooBar`)
+- Create a new Pull Request
 
 ## How to report a bug
 

.gitignore

@@ -12,9 +12,9 @@ bin
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+coverage.txt
 
 # Kubernetes Generated files - skip generated files, except for vendored files
-
 !vendor/**/zz_generated.*
 
 # editor and IDE paraphernalia
@@ -26,4 +26,4 @@ vendor/
 config/default/manager_image_patch.yaml-e
 .vscode/
 cover.html
-.DS_Store
+.DS_Store

.go-version

@@ -0,0 +1 @@
+1.15.6

Makefile

@@ -1,4 +1,4 @@
-VERSION=0.18-dev
+VERSION=0.19-dev
 # Image URL to use all building/pushing image targets
 IMG ?= keikoproj/rolling-upgrade-controller:${VERSION}
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)

api/v1alpha1/rollingupgrade_types.go

@@ -17,7 +17,9 @@ package v1alpha1
 
 import (
 	"fmt"
+	"time"
 
+	"github.com/keikoproj/upgrade-manager/controllers/common"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -71,24 +73,181 @@ type RollingUpgradeStatus struct {
 	NodesProcessed      int    `json:"nodesProcessed,omitempty"`
 	TotalNodes          int    `json:"totalNodes,omitempty"`
 
-	Conditions []RollingUpgradeCondition `json:"conditions,omitempty"`
+	Conditions              []RollingUpgradeCondition `json:"conditions,omitempty"`
+	LastNodeTerminationTime metav1.Time               `json:"lastTerminationTime,omitempty"`
+	LastNodeDrainTime       metav1.Time               `json:"lastDrainTime,omitempty"`
+
+	Statistics     []*RollingUpgradeStatistics `json:"statistics,omitempty"`
+	LastBatchNodes []string                    `json:"lastBatchNodes,omitempty"`
+}
+
+// RollingUpgrade Statistics, includes summary(sum/count) from each step
+type RollingUpgradeStatistics struct {
+	StepName      RollingUpgradeStep `json:"stepName,omitempty"`
+	DurationSum   metav1.Duration    `json:"durationSum,omitempty"`
+	DurationCount int32              `json:"durationCount,omitempty"`
 }
 
+type RollingUpgradeStep string
+
 const (
 	// StatusRunning marks the CR to be running.
 	StatusRunning = "running"
 	// StatusComplete marks the CR as completed.
 	StatusComplete = "completed"
 	// StatusError marks the CR as errored out.
 	StatusError = "error"
+
+	NodeRotationTotal RollingUpgradeStep = "total"
+
+	NodeRotationKickoff          RollingUpgradeStep = "kickoff"
+	NodeRotationDesiredNodeReady RollingUpgradeStep = "desired_node_ready"
+	NodeRotationPredrainScript   RollingUpgradeStep = "predrain_script"
+	NodeRotationDrain            RollingUpgradeStep = "drain"
+	NodeRotationPostdrainScript  RollingUpgradeStep = "postdrain_script"
+	NodeRotationPostWait         RollingUpgradeStep = "post_wait"
+	NodeRotationTerminate        RollingUpgradeStep = "terminate"
+	NodeRotationPostTerminate    RollingUpgradeStep = "post_terminate"
+	NodeRotationCompleted        RollingUpgradeStep = "completed"
 )
 
+var NodeRotationStepOrders = map[RollingUpgradeStep]int{
+	NodeRotationKickoff:          10,
+	NodeRotationDesiredNodeReady: 20,
+	NodeRotationPredrainScript:   30,
+	NodeRotationDrain:            40,
+	NodeRotationPostdrainScript:  50,
+	NodeRotationPostWait:         60,
+	NodeRotationTerminate:        70,
+	NodeRotationPostTerminate:    80,
+	NodeRotationCompleted:        1000,
+}
+
 // RollingUpgradeCondition describes the state of the RollingUpgrade
 type RollingUpgradeCondition struct {
 	Type   UpgradeConditionType   `json:"type,omitempty"`
 	Status corev1.ConditionStatus `json:"status,omitempty"`
 }
 
+// RollingUpgrade Node step information
+type NodeStepDuration struct {
+	GroupName string             `json:"groupName,omitempty"`
+	NodeName  string             `json:"nodeName,omitempty"`
+	StepName  RollingUpgradeStep `json:"stepName,omitempty"`
+	Duration  metav1.Duration    `json:"duration,omitempty"`
+}
+
+// Node In-processing
+type NodeInProcessing struct {
+	NodeName         string             `json:"nodeName,omitempty"`
+	StepName         RollingUpgradeStep `json:"stepName,omitempty"`
+	UpgradeStartTime metav1.Time        `json:"upgradeStartTime,omitempty"`
+	StepStartTime    metav1.Time        `json:"stepStartTime,omitempty"`
+	StepEndTime      metav1.Time        `json:"stepEndTime,omitempty"`
+}
+
+// Update last batch nodes
+func (s *RollingUpgradeStatus) UpdateLastBatchNodes(batchNodes map[string]*NodeInProcessing) {
+	keys := make([]string, 0, len(batchNodes))
+	for k := range batchNodes {
+		keys = append(keys, k)
+	}
+	s.LastBatchNodes = keys
+}
+
+// Update Node Statistics
+func (s *RollingUpgradeStatus) UpdateStatistics(nodeSteps map[string][]NodeStepDuration) {
+	for _, v := range nodeSteps {
+		for _, step := range v {
+			s.AddNodeStepDuration(step)
+		}
+	}
+}
+
+// Add one step duration
+func (s *RollingUpgradeStatus) AddNodeStepDuration(nsd NodeStepDuration) {
+	// if step exists, add count and sum, otherwise append
+	for _, s := range s.Statistics {
+		if s.StepName == nsd.StepName {
+			s.DurationSum = metav1.Duration{
+				Duration: s.DurationSum.Duration + nsd.Duration.Duration,
+			}
+			s.DurationCount += 1
+			return
+		}
+	}
+	s.Statistics = append(s.Statistics, &RollingUpgradeStatistics{
+		StepName: nsd.StepName,
+		DurationSum: metav1.Duration{
+			Duration: nsd.Duration.Duration,
+		},
+		DurationCount: 1,
+	})
+}
+
+// Node turns onto step
+func (s *RollingUpgradeStatus) NodeStep(InProcessingNodes map[string]*NodeInProcessing,
+	nodeSteps map[string][]NodeStepDuration, groupName, nodeName string, stepName RollingUpgradeStep) {
+
+	var inProcessingNode *NodeInProcessing
+	if n, ok := InProcessingNodes[nodeName]; !ok {
+		inProcessingNode = &NodeInProcessing{
+			NodeName:         nodeName,
+			StepName:         stepName,
+			UpgradeStartTime: metav1.Now(),
+			StepStartTime:    metav1.Now(),
+		}
+		InProcessingNodes[nodeName] = inProcessingNode
+	} else {
+		inProcessingNode = n
+	}
+
+	inProcessingNode.StepEndTime = metav1.Now()
+	var duration = inProcessingNode.StepEndTime.Sub(inProcessingNode.StepStartTime.Time)
+	if stepName == NodeRotationCompleted {
+		//Add overall and remove the node from in-processing map
+		var total = inProcessingNode.StepEndTime.Sub(inProcessingNode.UpgradeStartTime.Time)
+		duration1 := s.ToStepDuration(groupName, nodeName, inProcessingNode.StepName, duration)
+		duration2 := s.ToStepDuration(groupName, nodeName, NodeRotationTotal, total)
+		s.addNodeStepDuration(nodeSteps, nodeName, duration1)
+		s.addNodeStepDuration(nodeSteps, nodeName, duration2)
+	} else if inProcessingNode.StepName != stepName { //Still same step
+		var oldOrder = NodeRotationStepOrders[inProcessingNode.StepName]
+		var newOrder = NodeRotationStepOrders[stepName]
+		if newOrder > oldOrder { //Make sure the steps running in order
+			stepDuration := s.ToStepDuration(groupName, nodeName, inProcessingNode.StepName, duration)
+			inProcessingNode.StepStartTime = metav1.Now()
+			inProcessingNode.StepName = stepName
+			s.addNodeStepDuration(nodeSteps, nodeName, stepDuration)
+		}
+	}
+}
+
+func (s *RollingUpgradeStatus) addNodeStepDuration(steps map[string][]NodeStepDuration, nodeName string, nsd NodeStepDuration) {
+	if stepDuration, ok := steps[nodeName]; !ok {
+		steps[nodeName] = []NodeStepDuration{
+			nsd,
+		}
+	} else {
+		stepDuration = append(stepDuration, nsd)
+		steps[nodeName] = stepDuration
+	}
+}
+
+// Add one step duration
+func (s *RollingUpgradeStatus) ToStepDuration(groupName, nodeName string, stepName RollingUpgradeStep, duration time.Duration) NodeStepDuration {
+	//Add to system level statistics
+	common.AddStepDuration(groupName, string(stepName), duration)
+	return NodeStepDuration{
+		GroupName: groupName,
+		NodeName:  nodeName,
+		StepName:  stepName,
+		Duration: metav1.Duration{
+			Duration: duration,
+		},
+	}
+}
+
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
 // +kubebuilder:resource:path=rollingupgrades,scope=Namespaced,shortName=ru

api/v1alpha1/rollingupgrade_types_test.go

@@ -16,7 +16,10 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"testing"
+
 	. "github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
 	. "github.com/onsi/gomega"
 
 	"golang.org/x/net/context"
@@ -81,3 +84,54 @@ var _ = Describe("RollingUpgrade", func() {
 	})
 
 })
+
+// Test
+func TestNodeTurnsOntoStep(t *testing.T) {
+	g := gomega.NewGomegaWithT(t)
+
+	r := &RollingUpgradeStatus{}
+	//A map to retain the steps for multiple nodes
+	nodeSteps := make(map[string][]NodeStepDuration)
+	inProcessingNodes := make(map[string]*NodeInProcessing)
+
+	r.NodeStep(inProcessingNodes, nodeSteps, "test-asg", "node-1", NodeRotationKickoff)
+
+	g.Expect(inProcessingNodes).NotTo(gomega.BeNil())
+	g.Expect(nodeSteps["node-1"]).To(gomega.BeNil())
+
+	r.NodeStep(inProcessingNodes, nodeSteps, "test-asg", "node-1", NodeRotationDesiredNodeReady)
+
+	g.Expect(len(nodeSteps["node-1"])).To(gomega.Equal(1))
+	g.Expect(nodeSteps["node-1"][0].StepName).To(gomega.Equal(NodeRotationKickoff))
+
+	//Retry desired_node_ready
+	r.NodeStep(inProcessingNodes, nodeSteps, "test-asg", "node-1", NodeRotationDesiredNodeReady)
+	g.Expect(len(nodeSteps["node-1"])).To(gomega.Equal(1))
+	g.Expect(nodeSteps["node-1"][0].StepName).To(gomega.Equal(NodeRotationKickoff))
+
+	//Retry desired_node_ready again
+	r.NodeStep(inProcessingNodes, nodeSteps, "test-asg", "node-1", NodeRotationDesiredNodeReady)
+	g.Expect(len(nodeSteps["node-1"])).To(gomega.Equal(1))
+	g.Expect(nodeSteps["node-1"][0].StepName).To(gomega.Equal(NodeRotationKickoff))
+
+	//Completed
+	r.NodeStep(inProcessingNodes, nodeSteps, "test-asg", "node-1", NodeRotationCompleted)
+	g.Expect(len(nodeSteps["node-1"])).To(gomega.Equal(3))
+	g.Expect(nodeSteps["node-1"][1].StepName).To(gomega.Equal(NodeRotationDesiredNodeReady))
+	g.Expect(nodeSteps["node-1"][2].StepName).To(gomega.Equal(NodeRotationTotal))
+
+	//Second node
+	r.NodeStep(inProcessingNodes, nodeSteps, "test-asg", "node-2", NodeRotationKickoff)
+	g.Expect(len(nodeSteps["node-1"])).To(gomega.Equal(3))
+
+	r.NodeStep(inProcessingNodes, nodeSteps, "test-asg", "node-2", NodeRotationDesiredNodeReady)
+	g.Expect(len(nodeSteps["node-1"])).To(gomega.Equal(3))
+
+	r.UpdateLastBatchNodes(inProcessingNodes)
+	g.Expect(len(r.LastBatchNodes)).To(gomega.Equal(2))
+
+	r.UpdateStatistics(nodeSteps)
+	g.Expect(r.Statistics).ToNot(gomega.BeEmpty())
+	g.Expect(len(r.Statistics)).To(gomega.Equal(3))
+
+}