[meta] update docker images (elastic#1604)

* [meta] update docker images - use python slim base images - bump python version - bump python dependencies - bump tools version * cleanup dependencies * lint test files with new black version * link gcloud to path * do not build helm-tester docker image by default
fbacker · Mar 7, 2022 · ae6c5cc · ae6c5cc
1 parent 6714c16
commit ae6c5cc
Show file tree

Hide file tree

Showing 12 changed files with 180 additions and 205 deletions.
diff --git a/.ci/jobs/elastic+helm-charts+main+template-testing.yml b/.ci/jobs/elastic+helm-charts+main+template-testing.yml
@@ -4,22 +4,22 @@
     display-name: elastic / helm-charts - main - template testing
     description: Main - template testing
     scm:
-    - git:
-        wipe-workspace: 'True'
+      - git:
+          wipe-workspace: "True"
     axes:
-    - axis:
-        type: slave
-        name: label
-        values:
-        - docker&&virtual
-    - axis:
-        type: yaml
-        name: CHART
-        filename: helpers/matrix.yml
+      - axis:
+          type: slave
+          name: label
+          values:
+            - docker&&virtual
+      - axis:
+          type: yaml
+          name: CHART
+          filename: helpers/matrix.yml
     builders:
-    - shell: |-
-        #!/usr/local/bin/runbld
-        set -euo pipefail
+      - shell: |-
+          #!/usr/local/bin/runbld
+          set -euo pipefail
 
-        cd ${CHART}
-        make test
+          cd ${CHART}
+          make build test
diff --git a/.ci/jobs/elastic+helm-charts+pull-request+template-testing.yml b/.ci/jobs/elastic+helm-charts+pull-request+template-testing.yml
@@ -23,4 +23,4 @@
         set -euo pipefail
 
         cd ${CHART}
-        make test
+        make build test
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -216,7 +216,7 @@ You can also use `make template` (equivalent to `helm template` ) to look at the
 YAML being generated:
 
 It is possible to run all of the tests and linting inside of a Docker container
-using `make test`
+using `make build test`
 
 Note that templating tests are formatted using [Black][], you should run
 `make lint-python` (equivalent to `black --diff --check .` ) to validate them or

diff --git a/elasticsearch/tests/elasticsearch_test.py b/elasticsearch/tests/elasticsearch_test.py
@@ -807,12 +807,12 @@ def test_dont_add_data_volume_when_persistance_is_disabled():
 """
     r = helm_template(config)
     assert "volumeClaimTemplates" not in r["statefulset"][uname]["spec"]
-    assert (
-        {"name": "elasticsearch-master", "mountPath": "/usr/share/elasticsearch/data"}
-        not in r["statefulset"][uname]["spec"]["template"]["spec"]["containers"][0][
-            "volumeMounts"
-        ]
-    )
+    assert {
+        "name": "elasticsearch-master",
+        "mountPath": "/usr/share/elasticsearch/data",
+    } not in r["statefulset"][uname]["spec"]["template"]["spec"]["containers"][0][
+        "volumeMounts"
+    ]
 
 
 def test_priority_class_name():
@@ -1471,7 +1471,11 @@ def test_network_policy():
         {"podSelector": {"matchLabels": {"app": "elasticsearch-master"}}},
     ]
     assert transport["ports"][0]["port"] == 9300
-    assert pod_selector == {"matchLabels": {"app": "elasticsearch-master",}}
+    assert pod_selector == {
+        "matchLabels": {
+            "app": "elasticsearch-master",
+        }
+    }
 
 
 def test_default_automount_sa_token():

diff --git a/filebeat/tests/filebeat_test.py b/filebeat/tests/filebeat_test.py
@@ -685,29 +685,23 @@ def test_adding_a_secret_mount():
       path: /usr/share/filebeat/config/certs
 """
     r = helm_template(config)
-    assert (
-        {
-            "mountPath": "/usr/share/filebeat/config/certs",
-            "name": "elastic-certificates",
-        }
-        in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][
-            "volumeMounts"
-        ]
-    )
+    assert {
+        "mountPath": "/usr/share/filebeat/config/certs",
+        "name": "elastic-certificates",
+    } in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][
+        "volumeMounts"
+    ]
     assert {
         "name": "elastic-certificates",
         "secret": {"secretName": "elastic-certificates-name"},
     } in r["daemonset"][name]["spec"]["template"]["spec"]["volumes"]
 
-    assert (
-        {
-            "mountPath": "/usr/share/filebeat/config/certs",
-            "name": "elastic-certificates",
-        }
-        not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][
-            "volumeMounts"
-        ]
-    )
+    assert {
+        "mountPath": "/usr/share/filebeat/config/certs",
+        "name": "elastic-certificates",
+    } not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][
+        "volumeMounts"
+    ]
     assert {
         "name": "elastic-certificates",
         "secret": {"secretName": "elastic-certificates-name"},
@@ -722,29 +716,23 @@ def test_adding_a_secret_mount():
       path: /usr/share/filebeat/config/certs
 """
     r = helm_template(config)
-    assert (
-        {
-            "mountPath": "/usr/share/filebeat/config/certs",
-            "name": "elastic-certificates",
-        }
-        in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][
-            "volumeMounts"
-        ]
-    )
+    assert {
+        "mountPath": "/usr/share/filebeat/config/certs",
+        "name": "elastic-certificates",
+    } in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][
+        "volumeMounts"
+    ]
     assert {
         "name": "elastic-certificates",
         "secret": {"secretName": "elastic-certificates-name"},
     } in r["deployment"][name]["spec"]["template"]["spec"]["volumes"]
 
-    assert (
-        {
-            "mountPath": "/usr/share/filebeat/config/certs",
-            "name": "elastic-certificates",
-        }
-        not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][
-            "volumeMounts"
-        ]
-    )
+    assert {
+        "mountPath": "/usr/share/filebeat/config/certs",
+        "name": "elastic-certificates",
+    } not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][
+        "volumeMounts"
+    ]
     assert {
         "name": "elastic-certificates",
         "secret": {"secretName": "elastic-certificates-name"},
@@ -761,15 +749,12 @@ def test_adding_a_deprecated_secret_mount():
     path: /usr/share/filebeat/config/certs
 """
     r = helm_template(config)
-    assert (
-        {
-            "mountPath": "/usr/share/filebeat/config/certs",
-            "name": "elastic-certificates",
-        }
-        in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][
-            "volumeMounts"
-        ]
-    )
+    assert {
+        "mountPath": "/usr/share/filebeat/config/certs",
+        "name": "elastic-certificates",
+    } in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][
+        "volumeMounts"
+    ]
     assert {
         "name": "elastic-certificates",
         "secret": {"secretName": "elastic-certificates-name"},
@@ -810,12 +795,13 @@ def test_adding_a_extra_volume_with_volume_mount():
     assert {"name": "extras", "emptyDir": {}} not in r["deployment"][name]["spec"][
         "template"
     ]["spec"]["volumes"]
-    assert (
-        {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,}
-        not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][
-            "volumeMounts"
-        ]
-    )
+    assert {
+        "name": "extras",
+        "mountPath": "/usr/share/extras",
+        "readOnly": True,
+    } not in r["deployment"][name]["spec"]["template"]["spec"]["containers"][0][
+        "volumeMounts"
+    ]
 
     config = """
 deployment:
@@ -838,12 +824,13 @@ def test_adding_a_extra_volume_with_volume_mount():
     assert {"name": "extras", "emptyDir": {}} not in r["daemonset"][name]["spec"][
         "template"
     ]["spec"]["volumes"]
-    assert (
-        {"name": "extras", "mountPath": "/usr/share/extras", "readOnly": True,}
-        not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][
-            "volumeMounts"
-        ]
-    )
+    assert {
+        "name": "extras",
+        "mountPath": "/usr/share/extras",
+        "readOnly": True,
+    } not in r["daemonset"][name]["spec"]["template"]["spec"]["containers"][0][
+        "volumeMounts"
+    ]
 
 
 def test_adding_a_deprecated_extra_volume_with_volume_mount():

diff --git a/helpers/common.mk b/helpers/common.mk
@@ -32,7 +32,7 @@ template: ## Render chart templates
 	helm template ./
 
 .PHONY: test
-test: build ## Run all tests in a docker container
+test: ## Run all tests in a docker container
 	docker run --rm -i --user "$$(id -u):$$(id -g)" -v $$(pwd)/../:/app -w /app/$$(basename $$(pwd)) helm-tester make test-all
 
 .PHONY: test-all ## Run all tests

diff --git a/helpers/helm-tester/Dockerfile b/helpers/helm-tester/Dockerfile
@@ -1,7 +1,11 @@
-FROM python:3.7
+FROM python:3.10.2-slim
 
 ENV HELM_VERSION=3.8.0
 
+RUN apt-get update && apt-get upgrade -y && \
+  apt-get install --no-install-recommends -y make wget && \
+  rm -rf /var/lib/apt/lists/*
+
 RUN wget --no-verbose https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz && \
   tar xfv helm-v${HELM_VERSION}-linux-amd64.tar.gz && \
   mv linux-amd64/helm /usr/local/bin/ && \

diff --git a/helpers/helm-tester/requirements.txt b/helpers/helm-tester/requirements.txt
@@ -1,10 +1,2 @@
-atomicwrites==1.3.0
-attrs==19.1.0
-importlib-metadata==0.23
-more-itertools==7.2.0
-pluggy==0.13.0
-py==1.10.0
-pytest==4.1.0
-PyYAML==5.4
-six==1.12.0
-zipp==0.6.0
+pytest==7.0.1
+PyYAML==6.0
diff --git a/helpers/terraform/Dockerfile b/helpers/terraform/Dockerfile
@@ -1,53 +1,48 @@
-FROM centos:7
+FROM python:3.10.2-slim
 
-ENV VAULT_VERSION 0.9.3
-ENV TERRAFORM_VERSION=0.11.7
-ENV KUBECTL_VERSION=1.21.6
+ENV DOCKER_VERSION=20.10.12
+ENV GCLOUD_VERSION=375.0.0
 ENV HELM_VERSION=3.8.0
-ENV DOCKER_VERSION=18.09.7
-ENV JQ_VERSION=1.6
+ENV KUBECTL_VERSION=1.21.6
+ENV TERRAFORM_VERSION=0.11.7
+ENV VAULT_VERSION=1.9.3
 
-RUN yum -y install \
-    make \
-    openssl \
-    unzip \
-    which
+RUN apt-get update && apt-get upgrade -y && \
+  apt-get install --no-install-recommends -y curl jq make openssl unzip && \
+  rm -rf /var/lib/apt/lists/*
 
-RUN yum -y install \
-    https://packages.cloud.google.com/yum/pool/654962a43cb6bc6b47bc312424410b7c203e1beb73386d868d9e614839c82609-google-cloud-sdk-215.0.0-1.el7.noarch.rpm && \
-    gcloud config set core/disable_usage_reporting true && \
-    gcloud config set component_manager/disable_update_check true && \
-    gcloud version
+RUN curl -O  https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${GCLOUD_VERSION}-linux-x86_64.tar.gz && \
+  tar xvf google-cloud-sdk-${GCLOUD_VERSION}-linux-x86_64.tar.gz && \
+  ./google-cloud-sdk/install.sh --quiet --usage-reporting=false --path-update=true && \
+  rm -fr google-cloud-sdk-${GCLOUD_VERSION}-linux-x86_64.tar.gz && \
+  ln -s /google-cloud-sdk/bin/gcloud /usr/local/bin/gcloud && \
+  gcloud config set component_manager/disable_update_check true && \
+  gcloud version
 
 RUN curl -O https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_amd64.zip && \
-    unzip vault_${VAULT_VERSION}_linux_amd64.zip -d /usr/local/bin/ && \
-    chmod +x /usr/local/bin/vault && \
-    rm -f vault_${VAULT_VERSION}_linux_amd64.zip && \
-    vault version
+  unzip vault_${VAULT_VERSION}_linux_amd64.zip -d /usr/local/bin/ && \
+  chmod +x /usr/local/bin/vault && \
+  rm -f vault_${VAULT_VERSION}_linux_amd64.zip && \
+  vault version
 
 RUN curl -O https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \
-    unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /usr/local/bin/ && \
-    rm -f terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \
-    terraform version
+  unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /usr/local/bin/ && \
+  rm -f terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \
+  terraform version
 
 RUN curl -O https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl && \
-    mv kubectl /usr/local/bin/ && \
-    chmod a+x /usr/local/bin/kubectl && \
-    kubectl version --client
+  mv kubectl /usr/local/bin/ && \
+  chmod a+x /usr/local/bin/kubectl && \
+  kubectl version --client
 
 RUN curl -O https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz && \
-    tar xfv helm-v${HELM_VERSION}-linux-amd64.tar.gz && \
-    mv linux-amd64/helm /usr/local/bin/ && \
-    rm -rf helm-v${HELM_VERSION}-linux-amd64.tar.gz linux-amd64 && \
-    helm version --client
+  tar xfv helm-v${HELM_VERSION}-linux-amd64.tar.gz && \
+  mv linux-amd64/helm /usr/local/bin/ && \
+  rm -rf helm-v${HELM_VERSION}-linux-amd64.tar.gz linux-amd64 && \
+  helm version --client
 
 RUN curl -O https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz && \
-    tar xfv docker* && \
-    mv docker/docker /usr/local/bin && \
-    rm -rf docker-${DOCKER_VERSION}.tgz docker/ && \
-    docker -v
-
-RUN curl -O -L https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64 && \
-    mv jq-linux64 /usr/local/bin/jq && \
-    chmod a+x /usr/local/bin/jq && \
-    jq --version
+  tar xfv docker* && \
+  mv docker/docker /usr/local/bin && \
+  rm -rf docker-${DOCKER_VERSION}.tgz docker/ && \
+  docker -v
diff --git a/logstash/tests/logstash_test.py b/logstash/tests/logstash_test.py
@@ -243,7 +243,10 @@ def test_adding_a_extra_port():
     extraPorts = r["statefulset"][name]["spec"]["template"]["spec"]["containers"][0][
         "ports"
     ]
-    assert {"name": "foo", "containerPort": 30000,} in extraPorts
+    assert {
+        "name": "foo",
+        "containerPort": 30000,
+    } in extraPorts
 
 
 def test_adding_a_extra_init_container():
@@ -660,9 +663,10 @@ def test_adding_in_logstash_config():
 
     s = r["statefulset"][name]["spec"]["template"]["spec"]
 
-    assert {"configMap": {"name": name + "-config"}, "name": "logstashconfig",} in s[
-        "volumes"
-    ]
+    assert {
+        "configMap": {"name": name + "-config"},
+        "name": "logstashconfig",
+    } in s["volumes"]
     assert {
         "mountPath": "/usr/share/logstash/config/logstash.yml",
         "name": "logstashconfig",