Use UID for pod/container matching and only consider running container

ipamd/ipamd.go

@@ -210,6 +210,11 @@ func (c *IPAMContext) nodeInit() error {
 				ip.Name, ip.Namespace)
 			continue
 		}
+		if ip.IP == "" {
+			log.Infof("Skipping Pod %s, Namespace %s due to no IP",
+				ip.Name, ip.Namespace)
+			continue
+		}
 		log.Infof("Recovered AddNetwork for Pod %s, Namespace %s, Container %s",
 			ip.Name, ip.Namespace, ip.Container)
 		_, _, err = c.dataStore.AssignPodIPv4Address(ip)
@@ -243,21 +248,23 @@ func (c *IPAMContext) getLocalPodsWithRetry() ([]*k8sapi.K8SPodInfo, error) {
 		return nil, errors.New("unable to get local pods, giving up")
 	}
 
-	containers, err := c.dockerClient.GetRunningContainers()
+	var containers map[string]*docker.ContainerInfo
 
-	if err != nil {
-		log.Errorf("Not able to get container info from docker: %v", err)
-		return nil, errors.Wrapf(err, "not able to get container info from docker")
+	for retry := 1; retry <= maxK8SRetries; retry++ {
+		containers, err = c.dockerClient.GetRunningContainers()
+		if err == nil {
+			break
+		}
+		log.Infof("Not able to get local containers yet (attempt %d/%d): %v", retry, maxK8SRetries, err)
+		time.Sleep(retryK8SInterval)
 	}
 
 	// TODO consider using map
 	for _, pod := range pods {
 		// needs to find the container ID
 		for _, container := range containers {
-			//e.g. /k8s_POD_worker-hello-5974f49799-q9vct_default_c31721a2-5dfb-11e8-b09c-022ad646a21e_0
-			k8sName := "/k8s_POD_" + pod.Name + "_" + pod.Namespace + "_" + pod.UID + "_0"
-			if container.Name == k8sName {
-				log.Debugf("Found pod(%v)'s container ID: %v ", k8sName, container.ID)
+			if container.K8SUID == pod.UID {
+				log.Debugf("Found pod(%v)'s container ID: %v ", container.Name, container.ID)
 				pod.Container = container.ID
 				break
 			}

ipamd/ipamd_test.go

@@ -135,8 +135,10 @@ func TestNodeInit(t *testing.T) {
 	mockK8S.EXPECT().K8SGetLocalPodIPs().Return([]*k8sapi.K8SPodInfo{&k8sapi.K8SPodInfo{Name: "pod1",
 		Namespace: "default", UID: "pod-uid", IP: ipaddr02}}, nil)
 
-	mockDocker.EXPECT().GetRunningContainers().Return([]*docker.ContainerInfo{&docker.ContainerInfo{ID: "docker-id",
-		Name: k8sName, K8SUID: "pod-uid"}}, nil)
+	var dockerList = make(map[string]*docker.ContainerInfo, 0)
+	dockerList["pod-uid"] = &docker.ContainerInfo{ID: "docker-id",
+		Name: k8sName, K8SUID: "pod-uid"}
+	mockDocker.EXPECT().GetRunningContainers().Return(dockerList, nil)
 
 	err := mockContext.nodeInit()
 	assert.NoError(t, err)

main.go

@@ -26,7 +26,7 @@ import (
 
 const (
 	defaultLogFilePath = "/host/var/log/aws-routed-eni/ipamd.log"
-	version            = "0.2.3"
+	version            = "0.2.4"
 )
 
 func main() {

misc/aws-k8s-cni-calico.yaml

@@ -63,7 +63,7 @@ spec:
       - key: CriticalAddonsOnly
         operator: Exists
       containers:
-      - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:0.2.3
+      - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:0.2.4
         name: aws-node
         env:
           - name: AWS_VPC_K8S_CNI_LOGLEVEL

misc/aws-k8s-cni.yaml

@@ -63,7 +63,7 @@ spec:
       - key: CriticalAddonsOnly
         operator: Exists
       containers:
-      - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:0.2.3
+      - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:0.2.4
         name: aws-node
         env:
           - name: AWS_VPC_K8S_CNI_LOGLEVEL

pkg/docker/docker.go

@@ -5,6 +5,8 @@ import (
 	"github.com/docker/docker/client"
 	"golang.org/x/net/context"
 
+	"github.com/pkg/errors"
+
 	log "github.com/cihub/seelog"
 )
 
@@ -17,7 +19,7 @@ type ContainerInfo struct {
 
 // APIs provides Docker API
 type APIs interface {
-	GetRunningContainers() ([]*ContainerInfo, error)
+	GetRunningContainers() (map[string]*ContainerInfo, error)
 }
 
 type Client struct{}
@@ -26,8 +28,8 @@ func New() *Client {
 	return &Client{}
 }
 
-func (c *Client) GetRunningContainers() ([]*ContainerInfo, error) {
-	var containerInfos []*ContainerInfo
+func (c *Client) GetRunningContainers() (map[string]*ContainerInfo, error) {
+	var containerInfos = make(map[string]*ContainerInfo)
 
 	cli, err := client.NewEnvClient()
 	if err != nil {
@@ -41,13 +43,48 @@ func (c *Client) GetRunningContainers() ([]*ContainerInfo, error) {
 	}
 
 	for _, container := range containers {
-		log.Infof("GetRunningContainers: Discovered running docker: %s %s %s",
-			container.ID, container.Names[0], container.Labels["io.kubernetes.pod.uid"])
-		containerInfos = append(containerInfos,
-			&ContainerInfo{
+		log.Infof("GetRunningContainers: Discovered running docker: %s %s %s State: %s Status: %s ",
+			container.ID, container.Names[0], container.Labels["io.kubernetes.pod.uid"], container.State, container.Status)
+
+		containerType, ok := container.Labels["io.kubernetes.docker.type"]
+		if !ok {
+			log.Infof("GetRunningContainers: skip non pause container")
+			continue
+		}
+
+		if containerType != "podsandbox" {
+			log.Infof("GetRunningContainers: skip container type: %s", containerType)
+			continue
+		}
+
+		log.Debugf("GetRunningContainers: containerType %s", containerType)
+
+		if container.State != "running" {
+			log.Infof("GetRunningContainers: skip container who is not running")
+			continue
+		}
+
+		uid := container.Labels["io.kubernetes.pod.uid"]
+		_, ok = containerInfos[uid]
+		if !ok {
+			containerInfos[uid] = &ContainerInfo{
 				ID:     container.ID,
 				Name:   container.Names[0],
-				K8SUID: container.Labels["io.kubernetes.pod.uid"]})
+				K8SUID: uid}
+			continue
+		}
+
+		if container.Names[0] != containerInfos[uid].Name {
+			log.Infof("GetRunningContainers: same uid matched by container:%s, %s container id %s",
+				containerInfos[uid].Name, container.Names[0], container.ID)
+			continue
+		}
+
+		if container.Names[0] == containerInfos[uid].Name {
+			log.Errorf("GetRunningContainers: Conflict container id %s for container %s",
+				container.ID, containerInfos[uid].Name)
+			return nil, errors.New("conflict docker runtime info")
+		}
 	}
 
 	return containerInfos, nil

pkg/k8sapi/discovery.go

@@ -200,7 +200,7 @@ func (d *Controller) handlePodUpdate(key string) error {
 	}
 
 	if !exists {
-		log.Infof(" Pods deleted on my node: %s", key)
+		log.Infof(" Pods deleted on my node: %v", key)
 		d.workerPodsLock.Lock()
 		defer d.workerPodsLock.Unlock()
 		delete(d.workerPods, key)