diff --git a/ipamd/ipamd.go b/ipamd/ipamd.go index 0993918c07..2d6af4ecfb 100644 --- a/ipamd/ipamd.go +++ b/ipamd/ipamd.go @@ -210,6 +210,11 @@ func (c *IPAMContext) nodeInit() error { ip.Name, ip.Namespace) continue } + if ip.IP == "" { + log.Infof("Skipping Pod %s, Namespace %s due to no IP", + ip.Name, ip.Namespace) + continue + } log.Infof("Recovered AddNetwork for Pod %s, Namespace %s, Container %s", ip.Name, ip.Namespace, ip.Container) _, _, err = c.dataStore.AssignPodIPv4Address(ip) @@ -243,21 +248,23 @@ func (c *IPAMContext) getLocalPodsWithRetry() ([]*k8sapi.K8SPodInfo, error) { return nil, errors.New("unable to get local pods, giving up") } - containers, err := c.dockerClient.GetRunningContainers() + var containers map[string]*docker.ContainerInfo - if err != nil { - log.Errorf("Not able to get container info from docker: %v", err) - return nil, errors.Wrapf(err, "not able to get container info from docker") + for retry := 1; retry <= maxK8SRetries; retry++ { + containers, err = c.dockerClient.GetRunningContainers() + if err == nil { + break + } + log.Infof("Not able to get local containers yet (attempt %d/%d): %v", retry, maxK8SRetries, err) + time.Sleep(retryK8SInterval) } // TODO consider using map for _, pod := range pods { // needs to find the container ID for _, container := range containers { - //e.g. /k8s_POD_worker-hello-5974f49799-q9vct_default_c31721a2-5dfb-11e8-b09c-022ad646a21e_0 - k8sName := "/k8s_POD_" + pod.Name + "_" + pod.Namespace + "_" + pod.UID + "_0" - if container.Name == k8sName { - log.Debugf("Found pod(%v)'s container ID: %v ", k8sName, container.ID) + if container.K8SUID == pod.UID { + log.Debugf("Found pod(%v)'s container ID: %v ", container.Name, container.ID) pod.Container = container.ID break } diff --git a/ipamd/ipamd_test.go b/ipamd/ipamd_test.go index a27feacddf..22bdc1de79 100644 --- a/ipamd/ipamd_test.go +++ b/ipamd/ipamd_test.go @@ -135,8 +135,10 @@ func TestNodeInit(t *testing.T) { mockK8S.EXPECT().K8SGetLocalPodIPs().Return([]*k8sapi.K8SPodInfo{&k8sapi.K8SPodInfo{Name: "pod1", Namespace: "default", UID: "pod-uid", IP: ipaddr02}}, nil) - mockDocker.EXPECT().GetRunningContainers().Return([]*docker.ContainerInfo{&docker.ContainerInfo{ID: "docker-id", - Name: k8sName, K8SUID: "pod-uid"}}, nil) + var dockerList = make(map[string]*docker.ContainerInfo, 0) + dockerList["pod-uid"] = &docker.ContainerInfo{ID: "docker-id", + Name: k8sName, K8SUID: "pod-uid"} + mockDocker.EXPECT().GetRunningContainers().Return(dockerList, nil) err := mockContext.nodeInit() assert.NoError(t, err) diff --git a/main.go b/main.go index b5a6615db5..532fe5ee85 100644 --- a/main.go +++ b/main.go @@ -26,7 +26,7 @@ import ( const ( defaultLogFilePath = "/host/var/log/aws-routed-eni/ipamd.log" - version = "0.2.3" + version = "0.2.4" ) func main() { diff --git a/misc/aws-k8s-cni-calico.yaml b/misc/aws-k8s-cni-calico.yaml index 2227008fda..1858020203 100644 --- a/misc/aws-k8s-cni-calico.yaml +++ b/misc/aws-k8s-cni-calico.yaml @@ -63,7 +63,7 @@ spec: - key: CriticalAddonsOnly operator: Exists containers: - - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:0.2.3 + - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:0.2.4 name: aws-node env: - name: AWS_VPC_K8S_CNI_LOGLEVEL diff --git a/misc/aws-k8s-cni.yaml b/misc/aws-k8s-cni.yaml index 174ce279fc..bc16f9ce82 100644 --- a/misc/aws-k8s-cni.yaml +++ b/misc/aws-k8s-cni.yaml @@ -63,7 +63,7 @@ spec: - key: CriticalAddonsOnly operator: Exists containers: - - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:0.2.3 + - image: 602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni:0.2.4 name: aws-node env: - name: AWS_VPC_K8S_CNI_LOGLEVEL diff --git a/pkg/docker/docker.go b/pkg/docker/docker.go index a0a2e6d779..4ab18137e5 100644 --- a/pkg/docker/docker.go +++ b/pkg/docker/docker.go @@ -5,6 +5,8 @@ import ( "github.com/docker/docker/client" "golang.org/x/net/context" + "github.com/pkg/errors" + log "github.com/cihub/seelog" ) @@ -17,7 +19,7 @@ type ContainerInfo struct { // APIs provides Docker API type APIs interface { - GetRunningContainers() ([]*ContainerInfo, error) + GetRunningContainers() (map[string]*ContainerInfo, error) } type Client struct{} @@ -26,8 +28,8 @@ func New() *Client { return &Client{} } -func (c *Client) GetRunningContainers() ([]*ContainerInfo, error) { - var containerInfos []*ContainerInfo +func (c *Client) GetRunningContainers() (map[string]*ContainerInfo, error) { + var containerInfos = make(map[string]*ContainerInfo) cli, err := client.NewEnvClient() if err != nil { @@ -41,13 +43,48 @@ func (c *Client) GetRunningContainers() ([]*ContainerInfo, error) { } for _, container := range containers { - log.Infof("GetRunningContainers: Discovered running docker: %s %s %s", - container.ID, container.Names[0], container.Labels["io.kubernetes.pod.uid"]) - containerInfos = append(containerInfos, - &ContainerInfo{ + log.Infof("GetRunningContainers: Discovered running docker: %s %s %s State: %s Status: %s ", + container.ID, container.Names[0], container.Labels["io.kubernetes.pod.uid"], container.State, container.Status) + + containerType, ok := container.Labels["io.kubernetes.docker.type"] + if !ok { + log.Infof("GetRunningContainers: skip non pause container") + continue + } + + if containerType != "podsandbox" { + log.Infof("GetRunningContainers: skip container type: %s", containerType) + continue + } + + log.Debugf("GetRunningContainers: containerType %s", containerType) + + if container.State != "running" { + log.Infof("GetRunningContainers: skip container who is not running") + continue + } + + uid := container.Labels["io.kubernetes.pod.uid"] + _, ok = containerInfos[uid] + if !ok { + containerInfos[uid] = &ContainerInfo{ ID: container.ID, Name: container.Names[0], - K8SUID: container.Labels["io.kubernetes.pod.uid"]}) + K8SUID: uid} + continue + } + + if container.Names[0] != containerInfos[uid].Name { + log.Infof("GetRunningContainers: same uid matched by container:%s, %s container id %s", + containerInfos[uid].Name, container.Names[0], container.ID) + continue + } + + if container.Names[0] == containerInfos[uid].Name { + log.Errorf("GetRunningContainers: Conflict container id %s for container %s", + container.ID, containerInfos[uid].Name) + return nil, errors.New("conflict docker runtime info") + } } return containerInfos, nil diff --git a/pkg/docker/mocks/docker_mocks.go b/pkg/docker/mocks/docker_mocks.go index aab1ef33f3..84337af9e0 100644 --- a/pkg/docker/mocks/docker_mocks.go +++ b/pkg/docker/mocks/docker_mocks.go @@ -48,9 +48,9 @@ func (m *MockAPIs) EXPECT() *MockAPIsMockRecorder { } // GetRunningContainers mocks base method -func (m *MockAPIs) GetRunningContainers() ([]*docker.ContainerInfo, error) { +func (m *MockAPIs) GetRunningContainers() (map[string]*docker.ContainerInfo, error) { ret := m.ctrl.Call(m, "GetRunningContainers") - ret0, _ := ret[0].([]*docker.ContainerInfo) + ret0, _ := ret[0].(map[string]*docker.ContainerInfo) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/pkg/k8sapi/discovery.go b/pkg/k8sapi/discovery.go index 5723082bf8..5908f57e86 100644 --- a/pkg/k8sapi/discovery.go +++ b/pkg/k8sapi/discovery.go @@ -200,7 +200,7 @@ func (d *Controller) handlePodUpdate(key string) error { } if !exists { - log.Infof(" Pods deleted on my node: %s", key) + log.Infof(" Pods deleted on my node: %v", key) d.workerPodsLock.Lock() defer d.workerPodsLock.Unlock() delete(d.workerPods, key)