feat(nocode): Add datahub-upgrade job to helm chart and set version t…

…o v0.8.1 (#2643)
datahub-project · Jun 4, 2021 · ee454eb · ee454eb
1 parent a483933
commit ee454eb
Show file tree

Hide file tree

Showing 21 changed files with 349 additions and 77 deletions.
diff --git a/datahub-kubernetes/README.md b/datahub-kubernetes/README.md
@@ -84,10 +84,10 @@ prerequisites-zookeeper-0                          1/1     Running     0
 deploy Datahub by running the following
 
 ```(shell)
-helm install datahub datahub/ --values datahub/quickstart-values.yaml
+helm install datahub datahub/
 ```
 
-Values in [quickstart-values.yaml](https://github.com/linkedin/datahub/tree/master/datahub-kubernetes/datahub/quickstart-values.yaml) 
+Values in [values.yaml](https://github.com/linkedin/datahub/tree/master/datahub-kubernetes/datahub/values.yaml) 
 have been preset to point to the dependencies deployed using the [prerequisites](https://github.com/linkedin/datahub/tree/master/datahub-kubernetes/prerequisites) 
 chart with release name "prerequisites". If you deployed the helm chart using a different release name, update the quickstart-values.yaml file accordingly before installing. 
 

diff --git a/datahub-kubernetes/datahub/Chart.yaml b/datahub-kubernetes/datahub/Chart.yaml
@@ -4,10 +4,10 @@ description: A Helm chart for LinkedIn DataHub
 type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 0.1.2
+version: 0.2.0
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: latest #0.3.1
+appVersion: 0.8.1 #0.3.1
 dependencies:
   - name: datahub-gms
     version: 0.2.1

diff --git a/datahub-kubernetes/datahub/README.md b/datahub-kubernetes/datahub/README.md
@@ -17,44 +17,44 @@ helm install datahub datahub/
 |-----|------|---------|-------------|
 | datahub-frontend.enabled | bool | `true` | Enable Datahub Front-end |
 | datahub-frontend.image.repository | string | `"linkedin/datahub-frontend-react"` | Image repository for datahub-frontend |
-| datahub-frontend.image.tag | string | `"latest"` | Image tag for datahub-frontend |
+| datahub-frontend.image.tag | string | `"v0.8.1"` | Image tag for datahub-frontend |
 | datahub-gms.enabled | bool | `true` | Enable GMS |
 | datahub-gms.image.repository | string | `"linkedin/datahub-gms"` | Image repository for datahub-gms |
-| datahub-gms.image.tag | string | `"latest"` | Image tag for datahub-gms |
+| datahub-gms.image.tag | string | `"v0.8.1"` | Image tag for datahub-gms |
 | datahub-mae-consumer.enabled | bool | `true` | Enable MAE Consumer |
 | datahub-mae-consumer.image.repository | string | `"linkedin/datahub-mae-consumer"` | Image repository for datahub-mae-consumer |
-| datahub-mae-consumer.image.tag | string | `"latest"` | Image tag for datahub-mae-consumer |
+| datahub-mae-consumer.image.tag | string | `"v0.8.1"` | Image tag for datahub-mae-consumer |
 | datahub-mce-consumer.enabled | bool | `true` | Enable MCE Consumer |
 | datahub-mce-consumer.image.repository | string | `"linkedin/datahub-mce-consumer"` | Image repository for datahub-mce-consumer |
-| datahub-mce-consumer.image.tag | string | `"latest"` | Image tag for datahub-mce-consumer |
+| datahub-mce-consumer.image.tag | string | `"v0.8.1"` | Image tag for datahub-mce-consumer |
 | datahub-ingestion-cron.enabled | bool | `false` | Enable cronjob for periodic ingestion |
 | elasticsearchSetupJob.enabled | bool | `true` | Enable setup job for elasicsearch |
 | elasticsearchSetupJob.image.repository | string | `"linkedin/datahub-elasticsearch-setup"` | Image repository for elasticsearchSetupJob |
-| elasticsearchSetupJob.image.tag | string | `"latest"` | Image repository for elasticsearchSetupJob |
+| elasticsearchSetupJob.image.tag | string | `"v0.8.1"` | Image repository for elasticsearchSetupJob |
 | kafkaSetupJob.enabled | bool | `true` | Enable setup job for kafka |
 | kafkaSetupJob.image.repository | string | `"linkedin/datahub-kafka-setup"` | Image repository for kafkaSetupJob |
-| kafkaSetupJob.image.tag | string | `"latest"` | Image repository for kafkaSetupJob |
+| kafkaSetupJob.image.tag | string | `"v0.8.1"` | Image repository for kafkaSetupJob |
 | mysqlSetupJob.enabled | bool | `false` | Enable setup job for mysql |
 | mysqlSetupJob.image.repository | string | `""` | Image repository for mysqlSetupJob |
 | mysqlSetupJob.image.tag | string | `""` | Image repository for mysqlSetupJob |
 | global.datahub_analytics_enabled | boolean | true | Enable datahub usage analytics |
 | global.datahub.appVersion | string | `"1.0"` | App version for annotation |
 | global.datahub.gms.port | string | `"8080"` | Port of GMS service |
-| global.elasticsearch.host | string | `"elasticsearch"` | Elasticsearch host name (endpoint) |
+| global.elasticsearch.host | string | `"elasticsearch-master"` | Elasticsearch host name (endpoint) |
 | global.elasticsearch.port | string | `"9200"` | Elasticsearch port |
-| global.kafka.bootstrap.server | string | `"broker:9092"` | Kafka bootstrap servers (with port) |
-| global.kafka.zookeeper.server | string | `"zookeeper:2181"` | Kafka zookeeper servers (with port) |
-| global.kafka.schemaregistry.url | string | `"http://schema-registry:8081"` | URL to kafka schema registry |
-| global.neo4j.host | string | `"neo4j:7474"` | Neo4j host address (with port) |
-| global.neo4j.uri | string | `"bolt://neo4j"` | Neo4j URI |
+| global.kafka.bootstrap.server | string | `"prerequisites-broker:9092"` | Kafka bootstrap servers (with port) |
+| global.kafka.zookeeper.server | string | `"prerequisites-zookeeper:2181"` | Kafka zookeeper servers (with port) |
+| global.kafka.schemaregistry.url | string | `"http://prerequisites-cp-schema-registry:8081"` | URL to kafka schema registry |
+| global.neo4j.host | string | `"prerequisites-neo4j:7474"` | Neo4j host address (with port) |
+| global.neo4j.uri | string | `"bolt://prerequisites-neo4j"` | Neo4j URI |
 | global.neo4j.username | string | `"neo4j"` | Neo4j user name |
 | global.neo4j.password.secretRef | string | `"neo4j-secrets"` | Secret that contains the Neo4j password |
 | global.neo4j.password.secretKey | string | `"neo4j-password"` | Secret key that contains the Neo4j password |
 | global.sql.datasource.driver | string | `"com.mysql.jdbc.Driver"` | Driver for the SQL database |
-| global.sql.datasource.host | string | `"mysql:3306"` | SQL database host (with port) |
-| global.sql.datasource.hostForMysqlClient | string | `"mysql"` | SQL database host (without port) |
-| global.sql.datasource.url | string | `"jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false\u0026useSSL=true"` | URL to access SQL database |
-| global.sql.datasource.username | string | `"datahub"` | SQL user name |
+| global.sql.datasource.host | string | `"prerequisites-mysql:3306"` | SQL database host (with port) |
+| global.sql.datasource.hostForMysqlClient | string | `"prerequisites-mysql"` | SQL database host (without port) |
+| global.sql.datasource.url | string | `"jdbc:mysql://prerequisites-mysql:3306/datahub?verifyServerCertificate=false\u0026useSSL=true"` | URL to access SQL database |
+| global.sql.datasource.username | string | `"root"` | SQL user name |
 | global.sql.datasource.password.secretRef | string | `"mysql-secrets"` | Secret that contains the MySQL password |
 | global.sql.datasource.password.secretKey | string | `"mysql-password"` | Secret key that contains the MySQL password |
 

diff --git a/datahub-kubernetes/datahub/charts/datahub-frontend/README.md b/datahub-kubernetes/datahub/charts/datahub-frontend/README.md
@@ -20,7 +20,7 @@ Current chart version is `0.2.0`
 | global.datahub.gms.port | string | `"8080"` |  |
 | image.pullPolicy | string | `"IfNotPresent"` |  |
 | image.repository | string | `"linkedin/datahub-frontend-react"` |  |
-| image.tag | string | `"latest"` |  |
+| image.tag | string | `"v0.8.1"` |  |
 | imagePullSecrets | list | `[]` |  |
 | ingress.annotations | object | `{}` |  |
 | ingress.enabled | bool | `false` |  |

diff --git a/datahub-kubernetes/datahub/charts/datahub-frontend/values.yaml b/datahub-kubernetes/datahub/charts/datahub-frontend/values.yaml
@@ -6,7 +6,7 @@ replicaCount: 1
 
 image:
   repository: linkedin/datahub-frontend-react
-  tag: "latest"
+  tag: "v0.8.1"
   pullPolicy: Always
 
 imagePullSecrets: []

diff --git a/datahub-kubernetes/datahub/charts/datahub-gms/README.md b/datahub-kubernetes/datahub/charts/datahub-gms/README.md
@@ -38,7 +38,7 @@ Current chart version is `0.2.0`
 | global.sql.datasource.password.secretKey | string | `"mysql-password"` |  |
 | image.pullPolicy | string | `"IfNotPresent"` |  |
 | image.repository | string | `"linkedin/datahub-gms"` |  |
-| image.tag | string | `"latest"` |  |
+| image.tag | string | `"v0.8.1"` |  |
 | imagePullSecrets | list | `[]` |  |
 | ingress.annotations | object | `{}` |  |
 | ingress.enabled | bool | `false` |  |

diff --git a/datahub-kubernetes/datahub/charts/datahub-gms/values.yaml b/datahub-kubernetes/datahub/charts/datahub-gms/values.yaml
@@ -7,7 +7,7 @@ replicaCount: 1
 image:
   repository: linkedin/datahub-gms
   pullPolicy: Always
-  tag: latest
+  tag: v0.8.1
 
 imagePullSecrets: []
 nameOverride: ""

diff --git a/datahub-kubernetes/datahub/charts/datahub-ingestion-cron/README.md b/datahub-kubernetes/datahub/charts/datahub-ingestion-cron/README.md
@@ -8,7 +8,7 @@ A Helm chart for datahub's metadata-ingestion framework with kerberos authentica
 |-----|------|---------|-------------|
 | image.pullPolicy | string | `"Always"` | Image pull policy |
 | image.repository | string | `"linkedin/datahub-ingestion"` | DataHub Ingestion image repository |
-| image.tag | string | `"latest"` | DataHub Ingestion image tag |
+| image.tag | string | `"v0.8.1"` | DataHub Ingestion image tag |
 | imagePullSecrets | array | `[]` (does not add image pull secrets to deployed pods) | Docker registry secret names as an array |
 | labels | string | `{}` | Metadata labels to be added to each crawling cron job |
 | crons | type | `{}` | A map of crawling parameters per different technology being crawler, the key in the object will be used as the name for the new cron job |

diff --git a/datahub-kubernetes/datahub/charts/datahub-ingestion-cron/templates/cron.yaml b/datahub-kubernetes/datahub/charts/datahub-ingestion-cron/templates/cron.yaml
@@ -28,10 +28,10 @@ spec:
           - name: {{ $jobName }}-crawler
             image: "{{ $.Values.image.repository }}:{{ $.Values.image.tag }}"
             imagePullPolicy: {{ $.Values.image.pullPolicy }}
-            {{- if .extraVolumeMounts }}
             volumeMounts:
               - name: recipe
                 mountPath: /etc/recipe
+            {{- if .extraVolumeMounts }}
               {{- toYaml .extraVolumeMounts | nindent 14 }}
             {{- end }}
             command: ["/bin/sh", "-c", {{ default $defaultCommand .command }} ]

diff --git a/datahub-kubernetes/datahub/charts/datahub-ingestion-cron/values.yaml b/datahub-kubernetes/datahub/charts/datahub-ingestion-cron/values.yaml
@@ -4,7 +4,7 @@
 
 image:
   repository: linkedin/datahub-ingestion
-  tag: latest
+  tag: v0.8.1
   pullPolicy: Always
 
 imagePullSecrets: []

diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/README.md b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/README.md
@@ -31,7 +31,7 @@ Current chart version is `0.2.0`
 | global.hostAliases[0].ip | string | `"192.168.0.104"` |  |
 | image.pullPolicy | string | `"IfNotPresent"` |  |
 | image.repository | string | `"linkedin/datahub-mae-consumer"` |  |
-| image.tag | string | `"latest"` |  |
+| image.tag | string | `"v0.8.1"` |  |
 | imagePullSecrets | list | `[]` |  |
 | ingress.annotations | object | `{}` |  |
 | ingress.enabled | bool | `false` |  |

diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/deployment.yaml
@@ -54,17 +54,21 @@ spec:
             {{- toYaml .Values.securityContext | nindent 12 }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: 9091
+              protocol: TCP
           livenessProbe:
             httpGet:
               path: /actuator/health
-              port: 9091
+              port: http
             initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
             failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
           readinessProbe:
             httpGet:
               path: /actuator/health
-              port: 9091
+              port: http
             initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
             failureThreshold: {{ .Values.readinessProbe.failureThreshold }}

diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/service.yaml b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/templates/service.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ printf "%s-%s" .Release.Name "datahub-mae-consumer" }}
+  labels:
+    {{- include "datahub-mae-consumer.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.global.datahub.mae_consumer.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "datahub-mae-consumer.selectorLabels" . | nindent 4 }}
diff --git a/datahub-kubernetes/datahub/charts/datahub-mae-consumer/values.yaml b/datahub-kubernetes/datahub/charts/datahub-mae-consumer/values.yaml
@@ -7,7 +7,7 @@ replicaCount: 1
 image:
   repository: linkedin/datahub-mae-consumer
   pullPolicy: Always
-  tag: latest
+  tag: v0.8.1
 
 imagePullSecrets: []
 nameOverride: ""
@@ -165,6 +165,10 @@ global:
       secretRef: "neo4j-secrets"
       secretKey: "neo4j-password"
 
+  datahub:
+    mae_consumer:
+      port: "9091"
+
   hostAliases:
     - ip: "192.168.0.104"
       hostnames:

diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/README.md b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/README.md
@@ -24,7 +24,7 @@ Current chart version is `0.2.0`
 | global.hostAliases[0].ip | string | `"192.168.0.104"` |  |
 | image.pullPolicy | string | `"IfNotPresent"` |  |
 | image.repository | string | `"linkedin/datahub-mce-consumer"` |  |
-| image.tag | string | `"latest"` |  |
+| image.tag | string | `"v0.8.1"` |  |
 | imagePullSecrets | list | `[]` |  |
 | ingress.annotations | object | `{}` |  |
 | ingress.enabled | bool | `false` |  |

diff --git a/datahub-kubernetes/datahub/charts/datahub-mce-consumer/values.yaml b/datahub-kubernetes/datahub/charts/datahub-mce-consumer/values.yaml
@@ -7,7 +7,7 @@ replicaCount: 1
 image:
   repository: linkedin/datahub-mce-consumer
   pullPolicy: Always
-  tag: latest
+  tag: v0.8.1
 
 imagePullSecrets: []
 nameOverride: ""

diff --git a/datahub-kubernetes/datahub/quickstart-values.yaml b/datahub-kubernetes/datahub/quickstart-values.yaml
@@ -4,13 +4,13 @@ datahub-gms:
   enabled: true
   image:
     repository: linkedin/datahub-gms
-    tag: "latest"
+    tag: "v0.8.1"
 
 datahub-frontend:
   enabled: true
   image:
     repository: linkedin/datahub-frontend-react
-    tag: "latest"
+    tag: "v0.8.1"
   # Set up ingress to expose react front-end
   ingress:
     enabled: false
@@ -19,31 +19,37 @@ datahub-mae-consumer:
   enabled: true
   image:
     repository: linkedin/datahub-mae-consumer
-    tag: "latest"
+    tag: "v0.8.1"
 
 datahub-mce-consumer:
   enabled: true
   image:
     repository: linkedin/datahub-mce-consumer
-    tag: "latest"
+    tag: "v0.8.1"
 
 elasticsearchSetupJob:
   enabled: true
   image:
     repository: linkedin/datahub-elasticsearch-setup
-    tag: "latest"
+    tag: "v0.8.1"
 
 kafkaSetupJob:
   enabled: true
   image:
     repository: linkedin/datahub-kafka-setup
-    tag: "latest"
+    tag: "v0.8.1"
 
 mysqlSetupJob:
   enabled: true
   image:
     repository: acryldata/datahub-mysql-setup
-    tag: "latest"
+    tag: "v0.8.1"
+
+datahubUpgrade:
+  enabled: true
+  image:
+    repository: acryldata/datahub-upgrade
+    tag: "v0.8.1"
 
 datahub-ingestion-cron:
   enabled: false
@@ -53,12 +59,6 @@ global:
     host: "elasticsearch-master"
     port: "9200"
     indexPrefix: demo
-    useSSL: "true"
-    auth:
-      username: test
-      password:
-        secretRef: test
-        secretKey: test
 
   kafka:
     bootstrap:
@@ -91,4 +91,6 @@ global:
   datahub:
     gms:
       port: "8080"
+    mae_consumer:
+      port: "9091"
     appVersion: "1.0"