Update Cortex versions (eksctl, EKS, AWS IAM, Python, etc) (#2438)

* Update versions

* Update resource metadata

* eksctl fixes

* Discard deprecated ubuntu images

* Update k8s pkgs to match k8s version

* Revert k8s.io pkg versions

* Test commit

* Upgrade k8s.io packages

* Upgrade black and re-format

* Revert black because of dependency conflict

* Fix operator failing to create the Docker client

* Fixes

* Fix GPU examples

* Increase the no output timeout

* Lock click version

Co-authored-by: David Eliahu <deliahu@users.noreply.github.com>

Author: RobertLucian

.circleci/config.yml

@@ -28,7 +28,7 @@ commands:
 jobs:
   lint:
     docker:
-      - image: cimg/python:3.6
+      - image: cimg/python:3.7
     resource_class: medium
     steps:
       - checkout
@@ -42,18 +42,19 @@ jobs:
           command: |
             go get -u -v golang.org/x/lint/golint
             go get -u -v github.com/kyoh86/looppointer/cmd/looppointer
-            pip3 install black aiohttp
+            pip3 install aiohttp black==20.8b1 click==8.0.4
       - run:
           name: Lint
           command: make lint
+          no_output_timeout: 20m
       - save_cache:
           key: go-mod-v1-{{ checksum "go.sum" }}
           paths:
             - "~/go/pkg/mod"
 
   test:
     machine:
-      image: ubuntu-1604:202104-01 # machine executor necessary to run go integration tests
+      image: ubuntu-2004:202201-02 # machine executor necessary to run go integration tests
     resource_class: medium
     steps:
       - checkout
@@ -90,7 +91,7 @@ jobs:
 
   build-and-upload-cli:
     docker:
-      - image: cimg/python:3.6
+      - image: cimg/python:3.7
     resource_class: medium
     steps:
       - checkout
@@ -117,7 +118,7 @@ jobs:
 
   build-and-push-images-arm64:
     machine:
-      image: ubuntu-2004:202101-01
+      image: ubuntu-2004:202201-02
     resource_class: arm.medium
     steps:
       - checkout
@@ -133,7 +134,7 @@ jobs:
 
   amend-images:
     docker:
-      - image: cimg/python:3.6
+      - image: cimg/python:3.7
     environment:
       DOCKER_CLI_EXPERIMENTAL: enabled
     resource_class: medium
@@ -148,7 +149,7 @@ jobs:
 
   cluster-up:
     docker:
-      - image: cimg/python:3.6
+      - image: cimg/python:3.7
     steps:
       - setup_remote_docker
       - checkout
@@ -211,7 +212,7 @@ jobs:
 
   e2e-tests:
     docker:
-      - image: cimg/python:3.6
+      - image: cimg/python:3.7
     steps:
       - checkout
       - run:
@@ -242,7 +243,7 @@ jobs:
 
   cluster-down:
     docker:
-      - image: cimg/python:3.6
+      - image: cimg/python:3.7
     steps:
       - setup_remote_docker
       - checkout

Makefile

@@ -171,7 +171,7 @@ tools:
 	@go get -u -v github.com/kyoh86/looppointer/cmd/looppointer
 	@go get -u -v github.com/VojtechVitek/rerun/cmd/rerun
 	@go get -u -v github.com/go-delve/delve/cmd/dlv
-	@python3 -m pip install aiohttp black 'pydoc-markdown>=3.0.0,<4.0.0' boto3 pyyaml
+	@python3 -m pip install aiohttp boto3 pyyaml pydoc-markdown==3.* black==20.8b1 -U
 	@python3 -m pip install -e test/e2e
 
 format:

build/generate_ami_mapping.go

@@ -222,7 +222,7 @@ func main() {
 		json.Unmarshal(jsonBytes, &k8sVersionMap)
 	}
 
-	k8sVersion := "1.21"
+	k8sVersion := "1.22"
 
 	if k8sVersionMap[k8sVersion] == nil {
 		k8sVersionMap[k8sVersion] = map[string]map[string]string{}

dev/minimum_aws_policy.json

@@ -56,6 +56,10 @@
                 "logs:ListTagsLogGroup",
                 "logs:DescribeLogStreams",
                 "iam:TagRole",
+                "iam:GetPolicy",
+                "iam:CreatePolicy",
+                "iam:DeletePolicy",
+                "iam:ListPolicyVersions",
                 "iam:RemoveRoleFromInstanceProfile",
                 "iam:CreateRole",
                 "iam:AttachRolePolicy",
@@ -69,6 +73,7 @@
                 "iam:ListAttachedRolePolicies",
                 "iam:DeleteRolePolicy",
                 "iam:DeleteOpenIDConnectProvider",
+                "iam:TagOpenIDConnectProvider",
                 "iam:DeleteInstanceProfile",
                 "iam:GetRole",
                 "iam:GetInstanceProfile",
@@ -84,6 +89,7 @@
             "Resource": [
                 "arn:*:iam::$CORTEX_ACCOUNT_ID:instance-profile/eksctl-*",
                 "arn:*:iam::$CORTEX_ACCOUNT_ID:role/eksctl-*",
+                "arn:*:iam::$CORTEX_ACCOUNT_ID:policy/eksctl-*",
                 "arn:*:iam::$CORTEX_ACCOUNT_ID:role/aws-service-role/eks-nodegroup.amazonaws.com/AWSServiceRoleForAmazonEKSNodegroup",
                 "arn:*:iam::$CORTEX_ACCOUNT_ID:role/eksctl-managed-*",
                 "arn:*:iam::$CORTEX_ACCOUNT_ID:oidc-provider/*",
@@ -119,7 +125,8 @@
                 "eks:*",
                 "kms:CreateGrant",
                 "acm:DescribeCertificate",
-                "servicequotas:ListServiceQuotas"
+                "servicequotas:ListServiceQuotas",
+                "logs:PutRetentionPolicy"
             ],
             "Resource": "*"
         },
@@ -139,4 +146,4 @@
             "Resource": "arn:*:s3:::$CORTEX_CLUSTER_NAME*/*"
         }
     ]
-}
+}

dev/versions.md

@@ -7,8 +7,8 @@
 1. Update `generate_eks.py` if necessary
 1. Check that `eksctl utils write-kubeconfig` log filter still behaves as desired, and logs in `cortex cluster up` look good.
 1. Update eksctl on your dev
-   machine: `curl --location "https://github.com/weaveworks/eksctl/releases/download/v0.67.0/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp && sudo mv -f /tmp/eksctl /usr/local/bin`
-1. Check if eksctl iam polices changed by comparing the previous version of the eksctl policy docs to the new version's and update `./dev/minimum_aws_policy.json` and `docs/clusters/management/auth.md` accordingly. https://github.com/weaveworks/eksctl/blob/v0.67.0/userdocs/src/usage/minimum-iam-policies.md
+   machine: `curl --location "https://github.com/weaveworks/eksctl/releases/download/v0.107.0/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp && sudo mv -f /tmp/eksctl /usr/local/bin`
+1. Check if eksctl iam polices changed by comparing the previous version of the eksctl policy docs to the new version's and update `./dev/minimum_aws_policy.json` and `docs/clusters/management/auth.md` accordingly. https://github.com/weaveworks/eksctl/blob/v0.107.0/userdocs/src/usage/minimum-iam-policies.md
 
 ## Kubernetes
 
@@ -57,10 +57,10 @@
 
 1. Find the latest [release](https://istio.io/latest/news/releases) and check the release notes (here are
    the [latest IstioOperator Options](https://istio.io/latest/docs/reference/config/istio.operator.v1alpha1/))
-1. Update the version in `images/manager/Dockerfile`
-1. Update the version in all `images/istio-*` Dockerfiles
-1. Update `istio.yaml.j2`, `apis.yaml.j2`, `operator.yaml.j2`, and `pkg/lib/k8s` as necessary
-1. Update `install.sh` as necessary
+1. Update the version in `images/manager/Dockerfile`.
+1. Update the version in all `images/istio-*` Dockerfiles.
+1. Update `istio.yaml.j2`, `apis.yaml.j2`, `operator.yaml.j2`, and `pkg/lib/k8s` as necessary.
+1. Update `install.sh` as necessary.
 
 ## AWS CNI
 
@@ -71,7 +71,7 @@
 
 ```bash
 PREV_RELEASE=1.10.1
-NEW_RELEASE=1.10.1
+NEW_RELEASE=1.11.0
 wget -q -O cni_supported_instances_prev.txt https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v${PREV_RELEASE}/pkg/awsutils/vpc_ip_resource_limit.go; wget -q -O cni_supported_instances_new.txt https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/v${NEW_RELEASE}/pkg/awsutils/vpc_ip_resource_limit.go; git diff --no-index cni_supported_instances_prev.txt cni_supported_instances_new.txt; rm -rf cni_supported_instances_prev.txt; rm -rf cni_supported_instances_new.txt
 ```
 
@@ -145,9 +145,9 @@ see https://github.com/moby/moby/issues/39302#issuecomment-639687466_
 ### Non-versioned modules
 
 1. `rm -rf go.mod go.sum && go mod init && go clean -modcache`
-1. `go get k8s.io/client-go@v0.21.6 && go get k8s.io/apimachinery@v0.21.6 && go get k8s.io/api@v0.21.6`
-1. `go get istio.io/client-go@v1.11.4 && go get istio.io/api@1.11.4`
-1. `go get github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils@v1.9.3`
+1. `go get k8s.io/client-go@v0.20.15 && go get k8s.io/apimachinery@v0.20.15 && go get k8s.io/api@v0.20.15`
+1. `go get istio.io/client-go@v1.11.8 && go get istio.io/api@1.11.8`
+1. `go get github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils@v1.11.0`
 1. `go get github.com/cortexlabs/yaml@31e52ba8433b683c471ef92cf1711fe67671dac5`
 1. `go get github.com/cortexlabs/go-input@8b67a7a7b28d1c45f5c588171b3b50148462b247`
 1. `go get github.com/xlab/treeprint@v1.0.0`
@@ -212,14 +212,14 @@ see https://github.com/moby/moby/issues/39302#issuecomment-639687466_
 1. `git stash`
 1. `git remote add upstream https://github.com/kubernetes/autoscaler.git`
 1. `git fetch upstream`
-1. Checkout the appropriate version tag, e.g. `git checkout cluster-autoscaler-1.21.1 -b cluster-autoscaler-1.21.1-cortex`
+1. Checkout the appropriate version tag, e.g. `git checkout cluster-autoscaler-1.22.2 -b cluster-autoscaler-1.22.2-cortex`
 1. `git stash pop`
 1. Resolve any merge conflicts
 1. Unstage and check the diff
-1. `git commit -am "Add rate limiter"`
-1. `git push origin cluster-autoscaler-1.21.1-cortex`
-1. Update `images/cluster-autoscaler/Dockerfile` to use the new branch name (e.g. "cluster-autoscaler-1.21.1") in the `-b` flag's value from `git clone`.
-1. Match the Go version of the builder in `images/cluster-autoscaler/Dockerfile` with that of the [cluster autoscaler](https://github.com/kubernetes/autoscaler)'s Dockerfile.
+1. `git add *; git commit -am "Add rate limiter"`
+1. `git push origin cluster-autoscaler-1.22.2-cortex`
+1. Update `images/cluster-autoscaler/Dockerfile` to use the new branch name (e.g. "cluster-autoscaler-1.22.2") in the `-b` flag's value from `git clone`.
+1. Match the Go version of the builder in `images/cluster-autoscaler/Dockerfile` with that of the [cluster autoscaler's Dockerfile](https://github.com/kubernetes/autoscaler/blob/master/builder/Dockerfile).
 
 ## FluentBit
 

docs/clusters/management/auth.md

@@ -121,6 +121,10 @@ Replace the following placeholders with their respective values in the policy te
                 "logs:ListTagsLogGroup",
                 "logs:DescribeLogStreams",
                 "iam:TagRole",
+                "iam:GetPolicy",
+                "iam:CreatePolicy",
+                "iam:DeletePolicy",
+                "iam:ListPolicyVersions",
                 "iam:RemoveRoleFromInstanceProfile",
                 "iam:CreateRole",
                 "iam:AttachRolePolicy",
@@ -134,6 +138,7 @@ Replace the following placeholders with their respective values in the policy te
                 "iam:ListAttachedRolePolicies",
                 "iam:DeleteRolePolicy",
                 "iam:DeleteOpenIDConnectProvider",
+                "iam:TagOpenIDConnectProvider",
                 "iam:DeleteInstanceProfile",
                 "iam:GetRole",
                 "iam:GetInstanceProfile",
@@ -149,6 +154,7 @@ Replace the following placeholders with their respective values in the policy te
             "Resource": [
                 "arn:*:iam::$CORTEX_ACCOUNT_ID:instance-profile/eksctl-*",
                 "arn:*:iam::$CORTEX_ACCOUNT_ID:role/eksctl-*",
+                "arn:*:iam::$CORTEX_ACCOUNT_ID:policy/eksctl-*",
                 "arn:*:iam::$CORTEX_ACCOUNT_ID:role/aws-service-role/eks-nodegroup.amazonaws.com/AWSServiceRoleForAmazonEKSNodegroup",
                 "arn:*:iam::$CORTEX_ACCOUNT_ID:role/eksctl-managed-*",
                 "arn:*:iam::$CORTEX_ACCOUNT_ID:oidc-provider/*",
@@ -184,7 +190,8 @@ Replace the following placeholders with their respective values in the policy te
                 "eks:*",
                 "kms:CreateGrant",
                 "acm:DescribeCertificate",
-                "servicequotas:ListServiceQuotas"
+                "servicequotas:ListServiceQuotas",
+                "logs:PutRetentionPolicy"
             ],
             "Resource": "*"
         },