feat(backup): Add restore indices and restore backup tasks #2779
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,119 @@ | ||
| {{- if .Values.datahubUpgrade.enabled -}} | ||
|
There was a problem hiding this comment. Is this correct? Why does this depend on datahubUpgrade flag? There was a problem hiding this comment. All k8s files under datahub-upgrade are all enabled by the same flag There was a problem hiding this comment. Enabling it doesn't do much until you actually run the job since these cronjobs are suspended by default |
||
| # Job template for restoring indices by sending MAE corresponding to all entities in the local db | ||
| # Creates a suspended cronJob that you can use to create an adhoc job when ready to run clean up. | ||
| # Run the following command to do so | ||
| # kubectl create job --from=cronjob/<<release-name>>-datahub-restore-indices-job-template datahub-restore-indices-job | ||
| apiVersion: batch/v1beta1 | ||
| kind: CronJob | ||
| metadata: | ||
| name: {{ .Release.Name }}-datahub-restore-indices-job-template | ||
| labels: | ||
| app.kubernetes.io/managed-by: {{ .Release.Service | quote }} | ||
| app.kubernetes.io/instance: {{ .Release.Name | quote }} | ||
| app.kubernetes.io/version: {{ .Chart.AppVersion }} | ||
| helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" | ||
| spec: | ||
| schedule: "* * * * *" | ||
| suspend: true | ||
| jobTemplate: | ||
| spec: | ||
| template: | ||
| spec: | ||
| {{- with .Values.global.hostAliases }} | ||
| hostAliases: | ||
| {{- toYaml . | nindent 12 }} | ||
| {{- end }} | ||
| {{- with .Values.datahubUpgrade.serviceAccount }} | ||
| serviceAccountName: {{ . }} | ||
| {{- end }} | ||
| {{- with .Values.imagePullSecrets }} | ||
| imagePullSecrets: | ||
| {{- toYaml . | nindent 12 }} | ||
| {{- end }} | ||
| volumes: | ||
| {{- with .Values.datahubUpgrade.extraVolumes }} | ||
| {{- toYaml . | nindent 12 }} | ||
| {{- end }} | ||
| restartPolicy: Never | ||
| securityContext: | ||
| runAsUser: 1000 | ||
| fsGroup: 1000 | ||
| containers: | ||
| - name: datahub-upgrade-job | ||
| image: "{{ .Values.datahubUpgrade.image.repository }}:{{ .Values.datahubUpgrade.image.tag }}" | ||
| args: | ||
| - "-u" | ||
| - "RestoreIndices" | ||
| env: | ||
| - name: DATAHUB_GMS_HOST | ||
| value: {{ printf "%s-%s" .Release.Name "datahub-gms" }} | ||
| - name: DATAHUB_GMS_PORT | ||
| value: "{{ .Values.global.datahub.gms.port }}" | ||
| - name: DATAHUB_MAE_CONSUMER_HOST | ||
| value: {{ printf "%s-%s" .Release.Name "datahub-mae-consumer" }} | ||
| - name: DATAHUB_MAE_CONSUMER_PORT | ||
| value: "{{ .Values.global.datahub.mae_consumer.port }}" | ||
| - name: EBEAN_DATASOURCE_USERNAME | ||
| value: "{{ .Values.global.sql.datasource.username }}" | ||
| - name: EBEAN_DATASOURCE_PASSWORD | ||
| valueFrom: | ||
| secretKeyRef: | ||
| name: "{{ .Values.global.sql.datasource.password.secretRef }}" | ||
| key: "{{ .Values.global.sql.datasource.password.secretKey }}" | ||
| - name: EBEAN_DATASOURCE_HOST | ||
| value: "{{ .Values.global.sql.datasource.host }}" | ||
| - name: EBEAN_DATASOURCE_URL | ||
| value: "{{ .Values.global.sql.datasource.url }}" | ||
| - name: EBEAN_DATASOURCE_DRIVER | ||
| value: "{{ .Values.global.sql.datasource.driver }}" | ||
| - name: KAFKA_BOOTSTRAP_SERVER | ||
| value: "{{ .Values.global.kafka.bootstrap.server }}" | ||
| - name: KAFKA_SCHEMAREGISTRY_URL | ||
| value: "{{ .Values.global.kafka.schemaregistry.url }}" | ||
| - name: ELASTICSEARCH_HOST | ||
| value: {{ .Values.global.elasticsearch.host | quote }} | ||
| - name: ELASTICSEARCH_PORT | ||
| value: {{ .Values.global.elasticsearch.port | quote }} | ||
| {{- with .Values.global.elasticsearch.useSSL }} | ||
| - name: ELASTICSEARCH_USE_SSL | ||
| value: {{ . | quote }} | ||
| {{- end }} | ||
| {{- with .Values.global.elasticsearch.auth }} | ||
| - name: ELASTICSEARCH_USERNAME | ||
| value: {{ .username }} | ||
| - name: ELASTICSEARCH_PASSWORD | ||
| valueFrom: | ||
| secretKeyRef: | ||
| name: "{{ .password.secretRef }}" | ||
| key: "{{ .password.secretKey }}" | ||
| {{- end }} | ||
| - name: GRAPH_SERVICE_IMPL | ||
| value: {{ .Values.global.graph_service_impl }} | ||
| {{- if eq .Values.global.graph_service_impl "neo4j" }} | ||
| - name: NEO4J_HOST | ||
| value: "{{ .Values.global.neo4j.host }}" | ||
| - name: NEO4J_URI | ||
| value: "{{ .Values.global.neo4j.uri }}" | ||
| - name: NEO4J_USERNAME | ||
| value: "{{ .Values.global.neo4j.username }}" | ||
| - name: NEO4J_PASSWORD | ||
| valueFrom: | ||
| secretKeyRef: | ||
| name: "{{ .Values.global.neo4j.password.secretRef }}" | ||
| key: "{{ .Values.global.neo4j.password.secretKey }}" | ||
| {{- end }} | ||
| {{- with .Values.datahubUpgrade.extraEnvs }} | ||
| {{- toYaml . | nindent 16 }} | ||
| {{- end }} | ||
| volumeMounts: | ||
| {{- with .Values.datahubUpgrade.extraVolumeMounts }} | ||
| {{- toYaml . | nindent 16 }} | ||
| {{- end }} | ||
| resources: | ||
| limits: | ||
| cpu: 500m | ||
| memory: 512Mi | ||
| requests: | ||
| cpu: 300m | ||
| memory: 256Mi | ||
| {{- end -}} | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| package com.linkedin.datahub.upgrade.common.steps; | ||
|
|
||
| import com.linkedin.datahub.upgrade.UpgradeContext; | ||
| import com.linkedin.datahub.upgrade.UpgradeStep; | ||
| import com.linkedin.datahub.upgrade.UpgradeStepResult; | ||
| import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; | ||
| import com.linkedin.datahub.upgrade.nocode.NoCodeUpgrade; | ||
| import com.linkedin.metadata.graph.GraphService; | ||
| import java.util.function.Function; | ||
|
|
||
|
|
||
| public class ClearGraphServiceStep implements UpgradeStep { | ||
|
|
||
| private final String deletePattern = ".*"; | ||
|
|
||
| private final GraphService _graphService; | ||
| private final boolean _alwaysRun; | ||
|
|
||
| public ClearGraphServiceStep(final GraphService graphService, final boolean alwaysRun) { | ||
| _graphService = graphService; | ||
| _alwaysRun = alwaysRun; | ||
| } | ||
|
|
||
| @Override | ||
| public String id() { | ||
| return "ClearGraphServiceStep"; | ||
| } | ||
|
|
||
| @Override | ||
| public boolean skip(UpgradeContext context) { | ||
| if (_alwaysRun) { | ||
| return false; | ||
| } | ||
| if (context.parsedArgs().containsKey(NoCodeUpgrade.CLEAN_ARG_NAME)) { | ||
|
There was a problem hiding this comment. Should we add a "cleanup has been requested" report line here? There was a problem hiding this comment. That log is added in the execute function. I can move it to here? There was a problem hiding this comment. no lets just keep that |
||
| return false; | ||
| } | ||
| context.report().addLine("Cleanup has not been requested."); | ||
| return true; | ||
| } | ||
|
|
||
| @Override | ||
| public int retryCount() { | ||
| return 1; | ||
| } | ||
|
|
||
| @Override | ||
| public Function<UpgradeContext, UpgradeStepResult> executable() { | ||
| return (context) -> { | ||
| try { | ||
| _graphService.clear(); | ||
| } catch (Exception e) { | ||
| context.report().addLine(String.format("Failed to clear graph indices: %s", e.toString())); | ||
| return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); | ||
| } | ||
| return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); | ||
| }; | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| package com.linkedin.datahub.upgrade.common.steps; | ||
|
|
||
| import com.linkedin.datahub.upgrade.UpgradeContext; | ||
| import com.linkedin.datahub.upgrade.UpgradeStep; | ||
| import com.linkedin.datahub.upgrade.UpgradeStepResult; | ||
| import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult; | ||
| import com.linkedin.datahub.upgrade.nocode.NoCodeUpgrade; | ||
| import com.linkedin.metadata.search.SearchService; | ||
| import java.util.function.Function; | ||
|
|
||
|
|
||
| public class ClearSearchServiceStep implements UpgradeStep { | ||
|
|
||
| private final SearchService _searchService; | ||
| private final boolean _alwaysRun; | ||
|
|
||
| public ClearSearchServiceStep(final SearchService searchService, final boolean alwaysRun) { | ||
| _searchService = searchService; | ||
| _alwaysRun = alwaysRun; | ||
| } | ||
|
|
||
| @Override | ||
| public String id() { | ||
| return "ClearSearchServiceStep"; | ||
| } | ||
|
|
||
| @Override | ||
| public boolean skip(UpgradeContext context) { | ||
| if (_alwaysRun) { | ||
| return false; | ||
| } | ||
| if (context.parsedArgs().containsKey(NoCodeUpgrade.CLEAN_ARG_NAME)) { | ||
| return false; | ||
| } | ||
| context.report().addLine("Cleanup has not been requested."); | ||
| return true; | ||
| } | ||
|
|
||
| @Override | ||
| public int retryCount() { | ||
| return 1; | ||
| } | ||
|
|
||
| @Override | ||
| public Function<UpgradeContext, UpgradeStepResult> executable() { | ||
| return (context) -> { | ||
| try { | ||
| _searchService.clear(); | ||
| } catch (Exception e) { | ||
| context.report().addLine(String.format("Failed to clear search service: %s", e.toString())); | ||
| return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.FAILED); | ||
| } | ||
| return new DefaultUpgradeStepResult(id(), UpgradeStepResult.Result.SUCCEEDED); | ||
| }; | ||
| } | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Just to confirm: Have we run any sec scans since adding these deps?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
good point
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
looks good