[stable/hadoop] Hadoop bump to 2.9.0 and some cleanup (helm#10199)

* bump hadoop to 2.9.0 and some cleanup Signed-off-by: cpanato <ctadeu@gmail.com> * remove heritage label Signed-off-by: cpanato <ctadeu@gmail.com>
davidkarlsen · Jan 2, 2019 · 967250d · 967250d
1 parent 5070b5f
commit 967250d
Show file tree

Hide file tree

Showing 22 changed files with 128 additions and 138 deletions.
diff --git a/stable/hadoop/Chart.yaml b/stable/hadoop/Chart.yaml
@@ -1,8 +1,8 @@
 apiVersion: v1
 description: The Apache Hadoop software library is a framework that allows for the distributed processing of large data sets across clusters of computers using simple programming models.
 name: hadoop
-version: 1.0.8
-appVersion: 2.7.3
+version: 1.1.0
+appVersion: 2.9.0
 home: https://hadoop.apache.org/
 sources:
 - https://github.com/apache/hadoop

diff --git a/stable/hadoop/README.md b/stable/hadoop/README.md
@@ -39,9 +39,10 @@ The following table lists the configurable parameters of the Hadoop chart and th
 
 | Parameter                                         | Description                                                                        | Default                                                          |
 | ------------------------------------------------- | -------------------------------                                                    | ---------------------------------------------------------------- |
-| `image`                                           | Hadoop image ([source](https://github.com/Comcast/kube-yarn/tree/master/image))    | `danisla/hadoop:{VERSION}`                                       |
-| `imagePullPolicy`                                 | Pull policy for the images                                                         | `IfNotPresent`                                                   |
-| `hadoopVersion`                                   | Version of hadoop libraries being used                                              | `{VERSION}`                                                      |
+| `image.repository`                                | Hadoop image ([source](https://github.com/Comcast/kube-yarn/tree/master/image))    | `danisla/hadoop`                                                 |
+| `image.tag`                                       | Hadoop image tag                                                                   | `2.9.0`                                                          |
+| `imagee.pullPolicy`                               | Pull policy for the images                                                         | `IfNotPresent`                                                   |
+| `hadoopVersion`                                   | Version of hadoop libraries being used                                             | `2.9.0`                                                          |
 | `antiAffinity`                                    | Pod antiaffinity, `hard` or `soft`                                                 | `hard`                                                           |
 | `hdfs.nameNode.pdbMinAvailable`                   | PDB for HDFS NameNode                                                              | `1`                                                              |
 | `hdfs.nameNode.resources`                         | resources for the HDFS NameNode                                                    | `requests:memory=256Mi,cpu=10m,limits:memory=2048Mi,cpu=1000m`   |

diff --git a/stable/hadoop/templates/NOTES.txt b/stable/hadoop/templates/NOTES.txt
@@ -1,24 +1,24 @@
 1. You can check the status of HDFS by running this command:
-   kubectl exec -n {{ .Release.Namespace }} -it {{ template "hadoop.fullname" . }}-hdfs-nn-0 -- /usr/local/hadoop/bin/hdfs dfsadmin -report
+   kubectl exec -n {{ .Release.Namespace }} -it {{ include "hadoop.fullname" . }}-hdfs-nn-0 -- /usr/local/hadoop/bin/hdfs dfsadmin -report
 
 2. You can list the yarn nodes by running this command:
-   kubectl exec -n {{ .Release.Namespace }} -it {{ template "hadoop.fullname" . }}-yarn-rm-0 -- /usr/local/hadoop/bin/yarn node -list
+   kubectl exec -n {{ .Release.Namespace }} -it {{ include "hadoop.fullname" . }}-yarn-rm-0 -- /usr/local/hadoop/bin/yarn node -list
 
 3. Create a port-forward to the yarn resource manager UI:
-   kubectl port-forward -n {{ .Release.Namespace }} {{ template "hadoop.fullname" . }}-yarn-rm-0 8088:8088 
+   kubectl port-forward -n {{ .Release.Namespace }} {{ include "hadoop.fullname" . }}-yarn-rm-0 8088:8088
 
    Then open the ui in your browser:
-   
+
    open http://localhost:8088
 
 4. You can run included hadoop tests like this:
-   kubectl exec -n {{ .Release.Namespace }} -it {{ template "hadoop.fullname" . }}-yarn-nm-0 -- /usr/local/hadoop/bin/hadoop jar /usr/local/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-{{ .Values.hadoopVersion }}-tests.jar TestDFSIO -write -nrFiles 5 -fileSize 128MB -resFile /tmp/TestDFSIOwrite.txt
+   kubectl exec -n {{ .Release.Namespace }} -it {{ include "hadoop.fullname" . }}-yarn-nm-0 -- /usr/local/hadoop/bin/hadoop jar /usr/local/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-{{ .Values.hadoopVersion }}-tests.jar TestDFSIO -write -nrFiles 5 -fileSize 128MB -resFile /tmp/TestDFSIOwrite.txt
 
 5. You can list the mapreduce jobs like this:
-   kubectl exec -n {{ .Release.Namespace }} -it {{ template "hadoop.fullname" . }}-yarn-rm-0 -- /usr/local/hadoop/bin/mapred job -list
+   kubectl exec -n {{ .Release.Namespace }} -it {{ include "hadoop.fullname" . }}-yarn-rm-0 -- /usr/local/hadoop/bin/mapred job -list
 
 6. This chart can also be used with the zeppelin chart
-    helm install --namespace {{ .Release.Namespace }} --set hadoop.useConfigMap=true,hadoop.configMapName={{ template "hadoop.fullname" . }} stable/zeppelin
+    helm install --namespace {{ .Release.Namespace }} --set hadoop.useConfigMap=true,hadoop.configMapName={{ include "hadoop.fullname" . }} stable/zeppelin
 
 7. You can scale the number of yarn nodes like this:
    helm upgrade {{ .Release.Name }} --set yarn.nodeManager.replicas=4 stable/hadoop

diff --git a/stable/hadoop/templates/_helpers.yaml → stable/hadoop/templates/_helpers.tpl b/stable/hadoop/templates/_helpers.yaml → stable/hadoop/templates/_helpers.tpl
@@ -14,3 +14,10 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- $name := default .Chart.Name .Values.nameOverride -}}
 {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "hadoop.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
diff --git a/stable/hadoop/templates/hadoop-configmap.yaml b/stable/hadoop/templates/hadoop-configmap.yaml
@@ -1,12 +1,11 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ template "hadoop.fullname" . }}
+  name: {{ include "hadoop.fullname" . }}
   labels:
-    app: {{ template "hadoop.name" . }}
-    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app: {{ include "hadoop.name" . }}
+    chart: {{ include "hadoop.chart" . }}
     release: {{ .Release.Name }}
-    heritage: {{ .Release.Service }}
 data:
   bootstrap.sh: |
     #!/bin/bash
@@ -41,8 +40,8 @@ data:
     if [[ "${HOSTNAME}" =~ "hdfs-dn" ]]; then
       mkdir -p /root/hdfs/datanode
 
-      #  wait up to 30 seconds for namenode 
-      (while [[ $count -lt 15 && -z `curl -sf http://{{ template "hadoop.fullname" . }}-hdfs-nn:50070` ]]; do ((count=count+1)) ; echo "Waiting for {{ template "hadoop.fullname" . }}-hdfs-nn" ; sleep 2; done && [[ $count -lt 15 ]])
+      #  wait up to 30 seconds for namenode
+      (while [[ $count -lt 15 && -z `curl -sf http://{{ include "hadoop.fullname" . }}-hdfs-nn:50070` ]]; do ((count=count+1)) ; echo "Waiting for {{ include "hadoop.fullname" . }}-hdfs-nn" ; sleep 2; done && [[ $count -lt 15 ]])
       [[ $? -ne 0 ]] && echo "Timeout waiting for hdfs-nn, exiting." && exit 1
 
       $HADOOP_PREFIX/sbin/hadoop-daemon.sh start datanode
@@ -74,7 +73,7 @@ data:
       chmod +x start-yarn-nm.sh
 
       #  wait up to 30 seconds for resourcemanager
-      (while [[ $count -lt 15 && -z `curl -sf http://{{ template "hadoop.fullname" . }}-yarn-rm:8088/ws/v1/cluster/info` ]]; do ((count=count+1)) ; echo "Waiting for {{ template "hadoop.fullname" . }}-yarn-rm" ; sleep 2; done && [[ $count -lt 15 ]])
+      (while [[ $count -lt 15 && -z `curl -sf http://{{ include "hadoop.fullname" . }}-yarn-rm:8088/ws/v1/cluster/info` ]]; do ((count=count+1)) ; echo "Waiting for {{ include "hadoop.fullname" . }}-yarn-rm" ; sleep 2; done && [[ $count -lt 15 ]])
       [[ $? -ne 0 ]] && echo "Timeout waiting for yarn-rm, exiting." && exit 1
 
       ./start-yarn-nm.sh
@@ -96,7 +95,7 @@ data:
     <configuration>
       <property>
             <name>fs.defaultFS</name>
-            <value>hdfs://{{ template "hadoop.fullname" . }}-hdfs-nn:9000/</value>
+            <value>hdfs://{{ include "hadoop.fullname" . }}-hdfs-nn:9000/</value>
             <description>NameNode URI</description>
         </property>
     </configuration>
@@ -147,7 +146,7 @@ data:
         <value>0.0.0.0</value>
       </property>
       <!-- /Bind to all interfaces -->
-      
+
     </configuration>
 
   mapred-site.xml: |
@@ -161,11 +160,11 @@ data:
       </property>
       <property>
         <name>mapreduce.jobhistory.address</name>
-        <value>{{ template "hadoop.fullname" . }}-yarn-rm-0.{{ template "hadoop.fullname" . }}-yarn-rm.{{ .Release.Namespace }}.svc.cluster.local:10020</value>
+        <value>{{ include "hadoop.fullname" . }}-yarn-rm-0.{{ include "hadoop.fullname" . }}-yarn-rm.{{ .Release.Namespace }}.svc.cluster.local:10020</value>
       </property>
       <property>
         <name>mapreduce.jobhistory.webapp.address</name>
-        <value>{{ template "hadoop.fullname" . }}-yarn-rm-0.{{ template "hadoop.fullname" . }}-yarn-rm.{{ .Release.Namespace }}.svc.cluster.local:19888</value>
+        <value>{{ include "hadoop.fullname" . }}-yarn-rm-0.{{ include "hadoop.fullname" . }}-yarn-rm.{{ .Release.Namespace }}.svc.cluster.local:19888</value>
       </property>
     </configuration>
 
@@ -253,7 +252,7 @@ data:
     <configuration>
       <property>
         <name>yarn.resourcemanager.hostname</name>
-        <value>{{ template "hadoop.fullname" . }}-yarn-rm</value>
+        <value>{{ include "hadoop.fullname" . }}-yarn-rm</value>
       </property>
 
       <!-- Bind to all interfaces -->

diff --git a/stable/hadoop/templates/hdfs-dn-pdb.yaml b/stable/hadoop/templates/hdfs-dn-pdb.yaml
@@ -1,18 +1,16 @@
 apiVersion: policy/v1beta1
 kind: PodDisruptionBudget
 metadata:
-  name: {{ template "hadoop.fullname" . }}-hdfs-dn
+  name: {{ include "hadoop.fullname" . }}-hdfs-dn
   labels:
-    app: {{ template "hadoop.name" . }}
-    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app: {{ include "hadoop.name" . }}
+    chart: {{ include "hadoop.chart" . }}
     release: {{ .Release.Name }}
-    heritage: {{ .Release.Service }}
     component: hdfs-dn
 spec:
   selector:
     matchLabels:
-      app: {{ template "hadoop.name" . }}
+      app: {{ include "hadoop.name" . }}
       release: {{ .Release.Name }}
       component: hdfs-dn
   minAvailable: {{ .Values.hdfs.dataNode.pdbMinAvailable }}
-
diff --git a/stable/hadoop/templates/hdfs-dn-pvc.yaml b/stable/hadoop/templates/hdfs-dn-pvc.yaml
@@ -2,12 +2,11 @@
 kind: PersistentVolumeClaim
 apiVersion: v1
 metadata:
-  name: {{ template "hadoop.fullname" . }}-hdfs-dn
+  name: {{ include "hadoop.fullname" . }}-hdfs-dn
   labels:
-    app: {{ template "hadoop.name" . }}
-    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app: {{ include "hadoop.name" . }}
+    chart: {{ include "hadoop.chart" . }}
     release: {{ .Release.Name }}
-    heritage: {{ .Release.Service }}
     component: hdfs-dn
 spec:
   accessModes:

diff --git a/stable/hadoop/templates/hdfs-dn-statefulset.yaml b/stable/hadoop/templates/hdfs-dn-statefulset.yaml
@@ -1,22 +1,21 @@
 apiVersion: apps/v1beta1
 kind: StatefulSet
 metadata:
-  name: {{ template "hadoop.fullname" . }}-hdfs-dn
+  name: {{ include "hadoop.fullname" . }}-hdfs-dn
   annotations:
     checksum/config: {{ include (print $.Template.BasePath "/hadoop-configmap.yaml") . | sha256sum }}
   labels:
-    app: {{ template "hadoop.name" . }}
-    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app: {{ include "hadoop.name" . }}
+    chart: {{ include "hadoop.chart" . }}
     release: {{ .Release.Name }}
-    heritage: {{ .Release.Service }}
     component: hdfs-dn
 spec:
-  serviceName: {{ template "hadoop.fullname" . }}-hdfs-dn
+  serviceName: {{ include "hadoop.fullname" . }}-hdfs-dn
   replicas: {{ .Values.hdfs.dataNode.replicas }}
   template:
     metadata:
       labels:
-        app: {{ template "hadoop.name" . }}
+        app: {{ include "hadoop.name" . }}
         release: {{ .Release.Name }}
         component: hdfs-dn
     spec:
@@ -27,7 +26,7 @@ spec:
           - topologyKey: "kubernetes.io/hostname"
             labelSelector:
               matchLabels:
-                app:  {{ template "hadoop.name" . }}
+                app:  {{ include "hadoop.name" . }}
                 release: {{ .Release.Name | quote }}
                 component: hdfs-dn
         {{- else if eq .Values.antiAffinity "soft" }}
@@ -37,20 +36,20 @@ spec:
               topologyKey: "kubernetes.io/hostname"
               labelSelector:
                 matchLabels:
-                  app:  {{ template "hadoop.name" . }}
+                  app:  {{ include "hadoop.name" . }}
                   release: {{ .Release.Name | quote }}
                   component: hdfs-dn
         {{- end }}
       terminationGracePeriodSeconds: 0
       containers:
       - name: hdfs-dn
-        image: {{ .Values.image }}
-        imagePullPolicy: {{ .Values.imagePullPolicy }}
+        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+        imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
         command:
            - "/bin/bash"
            - "/tmp/hadoop-config/bootstrap.sh"
            - "-d"
-        resources:       
+        resources:
 {{ toYaml .Values.hdfs.dataNode.resources | indent 10 }}
         readinessProbe:
           httpGet:
@@ -72,11 +71,11 @@ spec:
       volumes:
       - name: hadoop-config
         configMap:
-          name: {{ template "hadoop.fullname" . }}
+          name: {{ include "hadoop.fullname" . }}
       - name: dfs
       {{- if .Values.persistence.dataNode.enabled }}
         persistentVolumeClaim:
-          claimName: {{ template "hadoop.fullname" . }}-hdfs-dn
-      {{- else }}        
+          claimName: {{ include "hadoop.fullname" . }}-hdfs-dn
+      {{- else }}
         emptyDir: {}
       {{- end }}
diff --git a/stable/hadoop/templates/hdfs-dn-svc.yaml b/stable/hadoop/templates/hdfs-dn-svc.yaml
@@ -2,12 +2,11 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ template "hadoop.fullname" . }}-hdfs-dn
+  name: {{ include "hadoop.fullname" . }}-hdfs-dn
   labels:
-    app: {{ template "hadoop.name" . }}
-    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app: {{ include "hadoop.name" . }}
+    chart: {{ include "hadoop.chart" . }}
     release: {{ .Release.Name }}
-    heritage: {{ .Release.Service }}
     component: hdfs-dn
 spec:
   ports:
@@ -18,6 +17,6 @@ spec:
     port: 50075
   clusterIP: None
   selector:
-    app: {{ template "hadoop.name" . }}
+    app: {{ include "hadoop.name" . }}
     release: {{ .Release.Name }}
-    component: hdfs-dn
+    component: hdfs-dn
diff --git a/stable/hadoop/templates/hdfs-nn-pdb.yaml b/stable/hadoop/templates/hdfs-nn-pdb.yaml
@@ -1,18 +1,16 @@
 apiVersion: policy/v1beta1
 kind: PodDisruptionBudget
 metadata:
-  name: {{ template "hadoop.fullname" . }}-hdfs-nn
+  name: {{ include "hadoop.fullname" . }}-hdfs-nn
   labels:
-    app: {{ template "hadoop.name" . }}
-    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    app: {{ include "hadoop.name" . }}
+    chart: {{ include "hadoop.chart" . }}
     release: {{ .Release.Name }}
-    heritage: {{ .Release.Service }}
     component: hdfs-nn
 spec:
   selector:
     matchLabels:
-      app: {{ template "hadoop.name" . }}
+      app: {{ include "hadoop.name" . }}
       release: {{ .Release.Name }}
       component: hdfs-nn
   minAvailable: {{ .Values.hdfs.nameNode.pdbMinAvailable }}
-
diff --git a/stable/hadoop/templates/hdfs-nn-pvc.yaml b/stable/hadoop/templates/hdfs-nn-pvc.yaml
@@ -2,12 +2,11 @@
 kind: PersistentVolumeClaim
 apiVersion: v1
 metadata:
-  name: {{ template "hadoop.fullname" . }}-hdfs-nn
+  name: {{ include "hadoop.fullname" . }}-hdfs-nn
   labels:
-    app: {{ template "hadoop.name" . }}
-    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app: {{ include "hadoop.name" . }}
+    chart: {{ include "hadoop.chart" . }}
     release: {{ .Release.Name }}
-    heritage: {{ .Release.Service }}
     component: hdfs-nn
 spec:
   accessModes:

diff --git a/stable/hadoop/templates/hdfs-nn-statefulset.yaml b/stable/hadoop/templates/hdfs-nn-statefulset.yaml
@@ -1,22 +1,21 @@
 apiVersion: apps/v1beta1
 kind: StatefulSet
 metadata:
-  name: {{ template "hadoop.fullname" . }}-hdfs-nn
+  name: {{ include "hadoop.fullname" . }}-hdfs-nn
   annotations:
     checksum/config: {{ include (print $.Template.BasePath "/hadoop-configmap.yaml") . | sha256sum }}
   labels:
-    app: {{ template "hadoop.name" . }}
-    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app: {{ include "hadoop.name" . }}
+    chart: {{ include "hadoop.chart" . }}
     release: {{ .Release.Name }}
-    heritage: {{ .Release.Service }}
     component: hdfs-nn
 spec:
-  serviceName: {{ template "hadoop.fullname" . }}-hdfs-nn
+  serviceName: {{ include "hadoop.fullname" . }}-hdfs-nn
   replicas: 1
   template:
     metadata:
       labels:
-        app: {{ template "hadoop.name" . }}
+        app: {{ include "hadoop.name" . }}
         release: {{ .Release.Name }}
         component: hdfs-nn
     spec:
@@ -27,7 +26,7 @@ spec:
           - topologyKey: "kubernetes.io/hostname"
             labelSelector:
               matchLabels:
-                app:  {{ template "hadoop.name" . }}
+                app:  {{ include "hadoop.name" . }}
                 release: {{ .Release.Name | quote }}
                 component: hdfs-nn
         {{- else if eq .Values.antiAffinity "soft" }}
@@ -37,15 +36,15 @@ spec:
               topologyKey: "kubernetes.io/hostname"
               labelSelector:
                 matchLabels:
-                  app:  {{ template "hadoop.name" . }}
+                  app:  {{ include "hadoop.name" . }}
                   release: {{ .Release.Name | quote }}
                   component: hdfs-nn
         {{- end }}
       terminationGracePeriodSeconds: 0
       containers:
       - name: hdfs-nn
-        image: {{ .Values.image }}
-        imagePullPolicy: {{ .Values.imagePullPolicy }}
+        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+        imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
         command:
         - "/bin/bash"
         - "/tmp/hadoop-config/bootstrap.sh"
@@ -72,11 +71,11 @@ spec:
       volumes:
       - name: hadoop-config
         configMap:
-          name: {{ template "hadoop.fullname" . }}
+          name: {{ include "hadoop.fullname" . }}
       - name: dfs
       {{- if .Values.persistence.nameNode.enabled }}
         persistentVolumeClaim:
-          claimName: {{ template "hadoop.fullname" . }}-hdfs-nn
-      {{- else }}        
+          claimName: {{ include "hadoop.fullname" . }}-hdfs-nn
+      {{- else }}
         emptyDir: {}
       {{- end }}