Skip to content

Commit 950b000

Browse files
almeidabbmalmeidabbm
authored
Merge pull request #87 from lightdash/07-08-feat_add_add_query_specific_workers
feat: add add query specific workers
2 parents 8cf74e9 + a6700a4 commit 950b000

File tree

6 files changed

+175
-115
lines changed

6 files changed

+175
-115
lines changed

‎charts/lightdash/Chart.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ type: application
66
# This is the chart version. This version number should be incremented each time you make changes
77
# to the chart and its templates, including the app version.
88
# Versions are expected to follow Semantic Versioning (https://semver.org/)
9-
version: 1.5.5
9+
version: 1.6.0
1010

1111
# This is the version number of the application being deployed. This version number should be
1212
# incremented each time you make changes to the application. Versions are not expected to

‎charts/lightdash/README.md

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
A Helm chart to deploy lightdash on kubernetes
44

5-
![Version: 1.5.5](https://img.shields.io/badge/Version-1.5.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.1121.0](https://img.shields.io/badge/AppVersion-0.1121.0-informational?style=flat-square)
5+
![Version: 1.6.0](https://img.shields.io/badge/Version-1.6.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.1121.0](https://img.shields.io/badge/AppVersion-0.1121.0-informational?style=flat-square)
66

77
## Prerequisites
88

@@ -111,8 +111,25 @@ If you don't want helm to manage this, you may wish to separately create a secre
111111
| postgresql.commonAnnotations."helm.sh/hook" | string | `"pre-install,pre-upgrade"` | |
112112
| postgresql.commonAnnotations."helm.sh/hook-weight" | string | `"-1"` | |
113113
| postgresql.enabled | bool | `true` | |
114+
| queryWorker.concurrency | int | `3` | |
115+
| queryWorker.enabled | bool | `false` | |
116+
| queryWorker.livenessProbe.initialDelaySeconds | int | `10` | |
117+
| queryWorker.livenessProbe.periodSeconds | int | `10` | |
118+
| queryWorker.livenessProbe.timeoutSeconds | int | `5` | |
119+
| queryWorker.port | int | `8080` | |
120+
| queryWorker.readinessProbe.initialDelaySeconds | int | `35` | |
121+
| queryWorker.readinessProbe.periodSeconds | int | `35` | |
122+
| queryWorker.readinessProbe.timeoutSeconds | int | `30` | |
123+
| queryWorker.replicas | int | `1` | |
124+
| queryWorker.resources.requests.cpu | string | `"475m"` | |
125+
| queryWorker.resources.requests.ephemeral-storage | string | `"1Gi"` | |
126+
| queryWorker.resources.requests.memory | string | `"725Mi"` | |
127+
| queryWorker.tasks.exclude | string | `nil` | |
128+
| queryWorker.tasks.include | string | `"runAsyncWarehouseQuery"` | |
129+
| queryWorker.terminationGracePeriodSeconds | int | `90` | |
114130
| replicaCount | int | `1` | Specify the number of lightdash instances. |
115131
| resources | object | `{}` | |
132+
| scheduler.concurrency | int | `3` | |
116133
| scheduler.enabled | bool | `false` | |
117134
| scheduler.livenessProbe.initialDelaySeconds | int | `10` | |
118135
| scheduler.livenessProbe.periodSeconds | int | `10` | |
@@ -125,9 +142,8 @@ If you don't want helm to manage this, you may wish to separately create a secre
125142
| scheduler.resources.requests.cpu | string | `"475m"` | |
126143
| scheduler.resources.requests.ephemeral-storage | string | `"1Gi"` | |
127144
| scheduler.resources.requests.memory | string | `"725Mi"` | |
128-
| scheduler.sqlProxy.resources.requests.cpu | string | `"25m"` | |
129-
| scheduler.sqlProxy.resources.requests.ephemeral-storage | string | `"10Mi"` | |
130-
| scheduler.sqlProxy.resources.requests.memory | string | `"25Mi"` | |
145+
| scheduler.tasks.exclude | string | `"runAsyncWarehouseQuery"` | |
146+
| scheduler.tasks.include | string | `nil` | |
131147
| scheduler.terminationGracePeriodSeconds | int | `90` | |
132148
| schedulerExtraEnv | list | `[]` | |
133149
| secrets.LIGHTDASH_SECRET | string | `"changeme"` | This is the secret used to sign the session ID cookie and to encrypt sensitive information. Do not share this secret! |

‎charts/lightdash/templates/_worker-deployment.tpl

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
{{/*
2+
Worker deployment template that can be reused for different scheduler components
3+
Usage: {{- include "lightdash.workerDeployment" (dict "root" . "component" "worker" "workerConfig" .Values.scheduler) }}
4+
*/}}
5+
{{- define "lightdash.workerDeployment" -}}
6+
{{- $root := .root -}}
7+
{{- $component := .component -}}
8+
{{- $workerConfig := .workerConfig -}}
9+
{{- if $workerConfig.enabled }}
10+
apiVersion: apps/v1
11+
kind: Deployment
12+
metadata:
13+
name: {{ include "lightdash.fullname" $root }}-{{ $component }}
14+
labels:
15+
{{- include "lightdash.labels" $root | nindent 4 }}
16+
app.kubernetes.io/component: {{ $component }}
17+
spec:
18+
replicas: {{ $workerConfig.replicas }}
19+
selector:
20+
matchLabels:
21+
{{- include "lightdash.selectorLabels" $root | nindent 6 }}
22+
app.kubernetes.io/component: {{ $component }}
23+
template:
24+
metadata:
25+
annotations:
26+
{{- with $root.Values.podAnnotations }}
27+
{{- toYaml . | nindent 8 }}
28+
{{- end }}
29+
checksum/config: {{ include (print $root.Template.BasePath "/configmap.yaml") $root | sha256sum }}
30+
checksum/secrets: {{ include (print $root.Template.BasePath "/secrets.yaml") $root | sha256sum }}
31+
labels:
32+
{{- include "lightdash.selectorLabels" $root | nindent 8 }}
33+
app.kubernetes.io/component: {{ $component }}
34+
spec:
35+
{{- with $root.Values.imagePullSecrets }}
36+
imagePullSecrets:
37+
{{- toYaml . | nindent 8 }}
38+
{{- end }}
39+
securityContext:
40+
{{- toYaml $root.Values.podSecurityContext | nindent 8 }}
41+
serviceAccountName: {{ include "lightdash.serviceAccountName" $root }}
42+
containers:
43+
{{- if $root.Values.extraContainers }}
44+
{{- toYaml $root.Values.extraContainers | nindent 8 }}
45+
{{- end }}
46+
- name: {{ $root.Chart.Name }}
47+
securityContext:
48+
{{- toYaml $root.Values.securityContext | nindent 12 }}
49+
image: "{{ $root.Values.image.repository }}:{{ $root.Values.image.tag | default $root.Chart.AppVersion }}"
50+
imagePullPolicy: {{ $root.Values.image.pullPolicy }}
51+
command: ["node", "dist/scheduler.js"]
52+
args: {{ $root.Values.image.args }}
53+
env:
54+
- name: PGPASSWORD
55+
valueFrom:
56+
secretKeyRef:
57+
name: {{ (include "lightdash.database.secretName" $root) }}
58+
key: {{ (include "lightdash.database.secret.passwordKey" $root) }}
59+
- name: PORT
60+
value: {{ $workerConfig.port | quote }}
61+
{{- if $workerConfig.tasks.include }}
62+
- name: SCHEDULER_INCLUDE_TASKS
63+
value: "{{ $workerConfig.tasks.include }}"
64+
{{- end }}
65+
{{- if $workerConfig.tasks.exclude }}
66+
- name: SCHEDULER_EXCLUDE_TASKS
67+
value: "{{ $workerConfig.tasks.exclude }}"
68+
{{- end }}
69+
- name: SCHEDULER_CONCURRENCY
70+
value: {{ $workerConfig.concurrency | default 3 }}
71+
{{- if $root.Values.extraEnv }}
72+
{{- toYaml $root.Values.extraEnv | nindent 12 }}
73+
{{- end }}
74+
{{- if $root.Values.schedulerExtraEnv }}
75+
{{- toYaml $root.Values.schedulerExtraEnv | nindent 12 }}
76+
{{- end }}
77+
envFrom:
78+
- configMapRef:
79+
name: {{ template "lightdash.fullname" $root }}
80+
{{ if $root.Values.secrets }}
81+
- secretRef:
82+
name: {{ template "lightdash.fullname" $root }}
83+
{{ end }}
84+
livenessProbe:
85+
initialDelaySeconds: {{ $workerConfig.livenessProbe.initialDelaySeconds }}
86+
timeoutSeconds: {{ $workerConfig.livenessProbe.timeoutSeconds }}
87+
periodSeconds: {{ $workerConfig.livenessProbe.periodSeconds }}
88+
httpGet:
89+
path: /api/v1/health
90+
port: {{ $workerConfig.port }}
91+
readinessProbe:
92+
initialDelaySeconds: {{ $workerConfig.readinessProbe.initialDelaySeconds }}
93+
periodSeconds: {{ $workerConfig.readinessProbe.periodSeconds }}
94+
timeoutSeconds: {{ $workerConfig.readinessProbe.timeoutSeconds }}
95+
httpGet:
96+
path: /api/v1/health
97+
port: {{ $workerConfig.port }}
98+
resources:
99+
{{- toYaml $workerConfig.resources | nindent 12 }}
100+
{{- if $root.Values.initContainers }}
101+
initContainers:
102+
{{- toYaml $root.Values.initContainers | nindent 8 }}
103+
{{- end }}
104+
{{- with $root.Values.nodeSelector }}
105+
nodeSelector:
106+
{{- toYaml . | nindent 8 }}
107+
{{- end }}
108+
{{- with $root.Values.affinity }}
109+
affinity:
110+
{{- toYaml . | nindent 8 }}
111+
{{- end }}
112+
{{- with $root.Values.tolerations }}
113+
tolerations:
114+
{{- toYaml . | nindent 8 }}
115+
{{- end }}
116+
terminationGracePeriodSeconds: {{ $workerConfig.terminationGracePeriodSeconds | default 90 }}
117+
{{- end }}
118+
{{- end }}

‎charts/lightdash/templates/queryWorkerDeployment.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
{{- /* Query worker deployment using reusable template - DO NOT REMOVE THIS COMMENT OTHERWISE FORMATTING WILL BREAK */ -}}
2+
{{- include "lightdash.workerDeployment" (dict "root" . "component" "query-worker" "workerConfig" .Values.queryWorker) -}}

‎charts/lightdash/templates/workerDeployment.yaml

Lines changed: 2 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -1,99 +1,2 @@
1-
{{- if .Values.scheduler.enabled }}
2-
apiVersion: apps/v1
3-
kind: Deployment
4-
metadata:
5-
name: {{ include "lightdash.fullname" . }}-worker
6-
labels:
7-
{{- include "lightdash.labels" . | nindent 4 }}
8-
app.kubernetes.io/component: worker
9-
spec:
10-
replicas: {{ .Values.scheduler.replicas }}
11-
selector:
12-
matchLabels:
13-
{{- include "lightdash.selectorLabels" . | nindent 6 }}
14-
app.kubernetes.io/component: worker
15-
template:
16-
metadata:
17-
annotations:
18-
{{- with .Values.podAnnotations }}
19-
{{- toYaml . | nindent 8 }}
20-
{{- end }}
21-
checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
22-
checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }}
23-
labels:
24-
{{- include "lightdash.selectorLabels" . | nindent 8 }}
25-
app.kubernetes.io/component: worker
26-
spec:
27-
{{- with .Values.imagePullSecrets }}
28-
imagePullSecrets:
29-
{{- toYaml . | nindent 8 }}
30-
{{- end }}
31-
securityContext:
32-
{{- toYaml .Values.podSecurityContext | nindent 8 }}
33-
serviceAccountName: {{ include "lightdash.serviceAccountName" . }}
34-
containers:
35-
{{- if .Values.extraContainers }}
36-
{{- toYaml .Values.extraContainers | nindent 8 }}
37-
{{- end }}
38-
- name: {{ .Chart.Name }}
39-
securityContext:
40-
{{- toYaml .Values.securityContext | nindent 12 }}
41-
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
42-
imagePullPolicy: {{ .Values.image.pullPolicy }}
43-
command: ["node", "dist/scheduler.js"]
44-
args: {{ .Values.image.args }}
45-
env:
46-
- name: PGPASSWORD
47-
valueFrom:
48-
secretKeyRef:
49-
name: {{ (include "lightdash.database.secretName" .) }}
50-
key: {{ (include "lightdash.database.secret.passwordKey" .) }}
51-
- name: PORT
52-
value: {{ .Values.scheduler.port | quote }}
53-
{{- if .Values.extraEnv }}
54-
{{- toYaml .Values.extraEnv | nindent 12 }}
55-
{{- end }}
56-
{{- if .Values.schedulerExtraEnv }}
57-
{{- toYaml .Values.schedulerExtraEnv | nindent 12 }}
58-
{{- end }}
59-
envFrom:
60-
- configMapRef:
61-
name: {{ template "lightdash.fullname" $ }}
62-
{{ if .Values.secrets }}
63-
- secretRef:
64-
name: {{ template "lightdash.fullname" $ }}
65-
{{ end }}
66-
livenessProbe:
67-
initialDelaySeconds: {{ .Values.scheduler.livenessProbe.initialDelaySeconds }}
68-
timeoutSeconds: {{ .Values.scheduler.livenessProbe.timeoutSeconds }}
69-
periodSeconds: {{ .Values.scheduler.livenessProbe.periodSeconds }}
70-
httpGet:
71-
path: /api/v1/health
72-
port: {{ .Values.scheduler.port }}
73-
readinessProbe:
74-
initialDelaySeconds: {{ .Values.scheduler.readinessProbe.initialDelaySeconds }}
75-
periodSeconds: {{ .Values.scheduler.readinessProbe.periodSeconds }}
76-
timeoutSeconds: {{ .Values.scheduler.readinessProbe.timeoutSeconds }}
77-
httpGet:
78-
path: /api/v1/health
79-
port: {{ .Values.scheduler.port }}
80-
resources:
81-
{{- toYaml .Values.scheduler.resources | nindent 12 }}
82-
{{- if .Values.initContainers }}
83-
initContainers:
84-
{{- toYaml .Values.initContainers | nindent 8 }}
85-
{{- end }}
86-
{{- with .Values.nodeSelector }}
87-
nodeSelector:
88-
{{- toYaml . | nindent 8 }}
89-
{{- end }}
90-
{{- with .Values.affinity }}
91-
affinity:
92-
{{- toYaml . | nindent 8 }}
93-
{{- end }}
94-
{{- with .Values.tolerations }}
95-
tolerations:
96-
{{- toYaml . | nindent 8 }}
97-
{{- end }}
98-
terminationGracePeriodSeconds: {{ .Values.scheduler.terminationGracePeriodSeconds | default 90 }}
99-
{{- end }}
1+
{{- /* Worker deployment using reusable template - DO NOT REMOVE THIS COMMENT OTHERWISE FORMATTING WILL BREAK */ -}}
2+
{{- include "lightdash.workerDeployment" (dict "root" . "component" "worker" "workerConfig" .Values.scheduler) -}}

‎charts/lightdash/values.yaml

Lines changed: 32 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -125,11 +125,11 @@ browserless-chrome:
125125
replicaCount: 1
126126
resources:
127127
limits:
128-
memory: '512Mi'
129-
cpu: '500m'
128+
memory: "512Mi"
129+
cpu: "500m"
130130
requests:
131-
memory: '512Mi'
132-
cpu: '500m'
131+
memory: "512Mi"
132+
cpu: "500m"
133133
image:
134134
repository: "ghcr.io/browserless/chromium"
135135
tag: "v2.24.3"
@@ -205,7 +205,6 @@ serviceAccount:
205205
backendConfig:
206206
create: false
207207

208-
209208
## Backend deployment specific configuration
210209
lightdashBackend:
211210
terminationGracePeriodSeconds: 90
@@ -225,17 +224,36 @@ scheduler:
225224
port: 8080
226225
enabled: false
227226
replicas: 1
227+
concurrency: 3
228+
resources:
229+
requests:
230+
memory: 725Mi
231+
cpu: 475m
232+
ephemeral-storage: 1Gi
233+
terminationGracePeriodSeconds: 90
234+
livenessProbe:
235+
initialDelaySeconds: 10
236+
timeoutSeconds: 5
237+
periodSeconds: 10
238+
readinessProbe:
239+
initialDelaySeconds: 35
240+
timeoutSeconds: 30
241+
periodSeconds: 35
242+
tasks:
243+
exclude: "runAsyncWarehouseQuery"
244+
include:
245+
246+
## Query worker deployment specific configuration
247+
queryWorker:
248+
port: 8080
249+
enabled: false
250+
replicas: 1
251+
concurrency: 3
228252
resources:
229253
requests:
230254
memory: 725Mi
231255
cpu: 475m
232256
ephemeral-storage: 1Gi
233-
sqlProxy:
234-
resources:
235-
requests:
236-
memory: 25Mi
237-
cpu: 25m
238-
ephemeral-storage: 10Mi
239257
terminationGracePeriodSeconds: 90
240258
livenessProbe:
241259
initialDelaySeconds: 10
@@ -245,3 +263,6 @@ scheduler:
245263
initialDelaySeconds: 35
246264
timeoutSeconds: 30
247265
periodSeconds: 35
266+
tasks:
267+
include: "runAsyncWarehouseQuery"
268+
exclude:

0 commit comments

Comments
 (0)