Skip to content

Commit

Permalink
Index worker (#3817)
Browse files Browse the repository at this point in the history
* feat: helm: Add aleph-index-worker

* Bump version: 4.0.0-rc41 → 4.0.0-rc42

* values namespace workerindex

* Bump version: 4.0.0-rc42 → 4.0.0-rc43
  • Loading branch information
stchris authored Jul 22, 2024
1 parent 554852f commit b9159bd
Show file tree
Hide file tree
Showing 10 changed files with 208 additions and 19 deletions.
2 changes: 1 addition & 1 deletion .bumpversion.cfg
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[bumpversion]
current_version = 4.0.0-rc41
current_version = 4.0.0-rc43
tag_name = {new_version}
commit = True
tag = True
Expand Down
8 changes: 4 additions & 4 deletions contrib/aleph-traefik-minio-keycloak/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ services:
- "traefik.enable=false"

worker:
image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc41}
image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc43}
command: aleph worker
restart: on-failure
links:
Expand All @@ -79,7 +79,7 @@ services:
- "traefik.enable=false"

shell:
image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc41}
image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc43}
command: /bin/bash
depends_on:
- postgres
Expand All @@ -99,7 +99,7 @@ services:
- "traefik.enable=false"

api:
image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc41}
image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc43}
command: gunicorn -w 6 -b 0.0.0.0:8000 --log-level debug --log-file - aleph.wsgi:app
expose:
- 8000
Expand All @@ -121,7 +121,7 @@ services:
- "traefik.enable=false"

ui:
image: ghcr.io/alephdata/aleph-ui-production:${ALEPH_TAG:-4.0.0-rc41}
image: ghcr.io/alephdata/aleph-ui-production:${ALEPH_TAG:-4.0.0-rc43}
depends_on:
- api
- traefik
Expand Down
8 changes: 4 additions & 4 deletions contrib/keycloak/docker-compose.dev-keycloak.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ services:
elasticsearch:
build:
context: services/elasticsearch
image: ghcr.io/alephdata/aleph-elasticsearch:${ALEPH_TAG:-4.0.0-rc41}
image: ghcr.io/alephdata/aleph-elasticsearch:${ALEPH_TAG:-4.0.0-rc43}
hostname: elasticsearch
environment:
- discovery.type=single-node
Expand Down Expand Up @@ -55,7 +55,7 @@ services:
app:
build:
context: .
image: alephdata/aleph:${ALEPH_TAG:-4.0.0-rc41}
image: alephdata/aleph:${ALEPH_TAG:-4.0.0-rc43}
hostname: aleph
command: /bin/bash
links:
Expand Down Expand Up @@ -83,7 +83,7 @@ services:
api:
build:
context: .
image: alephdata/aleph:${ALEPH_TAG:-4.0.0-rc41}
image: alephdata/aleph:${ALEPH_TAG:-4.0.0-rc43}
command: aleph run -h 0.0.0.0 -p 5000 --with-threads --reload --debugger
ports:
- "127.0.0.1:5000:5000"
Expand Down Expand Up @@ -117,7 +117,7 @@ services:
ui:
build:
context: ui
image: alephdata/aleph-ui:${ALEPH_TAG:-4.0.0-rc41}
image: alephdata/aleph-ui:${ALEPH_TAG:-4.0.0-rc43}
links:
- api
command: npm run start
Expand Down
8 changes: 4 additions & 4 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ services:
- aleph.env

worker:
image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc41}
image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc43}
command: aleph worker
restart: on-failure
depends_on:
Expand All @@ -62,7 +62,7 @@ services:
- aleph.env

shell:
image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc41}
image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc43}
command: /bin/bash
depends_on:
- postgres
Expand All @@ -80,7 +80,7 @@ services:
- aleph.env

api:
image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc41}
image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc43}
expose:
- 8000
depends_on:
Expand All @@ -97,7 +97,7 @@ services:
- aleph.env

ui:
image: ghcr.io/alephdata/aleph-ui-production:${ALEPH_TAG:-4.0.0-rc41}
image: ghcr.io/alephdata/aleph-ui-production:${ALEPH_TAG:-4.0.0-rc43}
depends_on:
- api
ports:
Expand Down
4 changes: 2 additions & 2 deletions helm/charts/aleph/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@ apiVersion: v2
name: aleph
description: Helm chart for Aleph
type: application
version: 4.0.0-rc41
appVersion: 4.0.0-rc41
version: 4.0.0-rc43
appVersion: 4.0.0-rc43
2 changes: 1 addition & 1 deletion helm/charts/aleph/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ Helm chart for Aleph
| global.amazon | bool | `true` | Are we using AWS services like s3? |
| global.google | bool | `false` | Are we using GCE services like storage, vision api? |
| global.image.repository | string | `"alephdata/aleph"` | Aleph docker image repo |
| global.image.tag | string | `"4.0.0-rc41"` | Aleph docker image tag |
| global.image.tag | string | `"4.0.0-rc43"` | Aleph docker image tag |
| global.image.tag | string | `"Always"` | |
| global.namingPrefix | string | `"aleph"` | Prefix for the names of k8s resources |

Expand Down
149 changes: 149 additions & 0 deletions helm/charts/aleph/templates/worker-index.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ .Values.global.namingPrefix }}-worker-index
labels:
app: {{ .Values.global.namingPrefix }}-worker-index
spec:
replicas: {{ .Values.workerindex.replicas }}
selector:
matchLabels:
app: {{ .Values.global.namingPrefix }}-worker-index
strategy: {{- toYaml .Values.workerindex.strategy | nindent 4 }}
template:
metadata:
labels:
app: {{ .Values.global.namingPrefix }}-worker-index
annotations: {{- toYaml .Values.workerindex.podAnnotations | nindent 8 }}
spec:
nodeSelector: {{- toYaml .Values.workerindex.nodeSelector | nindent 8 }}
restartPolicy: Always
securityContext: {{- toYaml .Values.workerindex.podSecurityContext | nindent 8 }}
containers:
- name: {{ .Chart.Name }}
image: "{{ .Values.global.image.repository }}:{{ .Values.global.image.tag }}"
imagePullPolicy: {{ .Values.global.image.pullPolicy }}
command: ["aleph", "worker"]
volumeMounts:
{{ if .Values.global.google }}
- mountPath: /var/secrets/google
name: service-account-aleph
readOnly: true
{{ end }}
- mountPath: /tmp
name: tmp-volume
- mountPath: /home/app
name: home-volume
securityContext:
{{- toYaml .Values.workerindex.containerSecurityContext | nindent 12 }}
resources:
{{- toYaml .Values.workerindex.containerResources | nindent 12 }}
env:
{{- range $key, $value := .Values.global.commonEnv }}
- name: {{ $key }}
value: {{ $value | quote }}
{{- end }}
{{- range $key, $value := .Values.global.env }}
- name: {{ $key }}
value: {{ $value | quote }}
{{- end }}
{{- range $key, $value := .Values.workerindex.env }}
- name: {{ $key }}
value: {{ $value | quote }}
{{- end }}
- name: ALEPH_DATABASE_URI
valueFrom:
secretKeyRef:
name: aleph-secrets
key: ALEPH_DATABASE_URI
- name: ALEPH_OAUTH_SECRET
valueFrom:
secretKeyRef:
name: aleph-secrets
key: ALEPH_OAUTH_SECRET
- name: ALEPH_SECRET_KEY
valueFrom:
secretKeyRef:
name: aleph-secrets
key: ALEPH_SECRET_KEY
- name: FTM_STORE_URI
valueFrom:
secretKeyRef:
name: aleph-secrets
key: FTM_STORE_URI
- name: RABBITMQ_PASSWORD
valueFrom:
secretKeyRef:
name: aleph-secrets
key: RABBITMQ_PASSWORD
optional: true
- name: SENTRY_DSN
valueFrom:
secretKeyRef:
name: aleph-secrets
key: SENTRY_DSN
optional: true
{{ if .Values.global.google }}
- name: GOOGLE_APPLICATION_CREDENTIALS
value: /var/secrets/google/service-account.json
{{ end }}
{{ if .Values.global.amazon }}
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: aleph-secrets
key: AWS_ACCESS_KEY_ID
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: aleph-secrets
key: AWS_SECRET_ACCESS_KEY
{{ end }}
{{ if .Values.global.prometheus.enabled }}
- name: PROMETHEUS_ENABLED
value: "true"
{{ end }}
volumes:
{{ if .Values.global.google }}
- name: service-account-aleph
secret:
secretName: service-account-aleph
{{ end }}
- name: home-volume
emptyDir:
medium: Memory
- name: tmp-volume
emptyDir: {}
---
{{ if .Values.global.prometheus.enabled }}
apiVersion: v1
kind: Service
metadata:
name: {{ .Values.global.namingPrefix }}-worker-index
labels:
app: {{ .Values.global.namingPrefix }}-worker-index
spec:
selector:
app: {{ .Values.global.namingPrefix }}-worker-index
ports:
- port: 9100
targetPort: 9100
protocol: "TCP"
name: metrics
{{ end }}
---
{{ if .Values.global.serviceMonitor.enabled }}
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: {{ .Values.global.namingPrefix }}-worker-index
labels:
app: {{ .Values.global.namingPrefix }}-worker-index
spec:
selector:
matchLabels:
app: {{ .Values.global.namingPrefix }}-worker-index
endpoints:
- interval: "30s"
port: metrics
{{ end }}
42 changes: 41 additions & 1 deletion helm/charts/aleph/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ global:

image:
repository: ghcr.io/alephdata/aleph
tag: "4.0.0-rc41"
tag: "4.0.0-rc43"
pullPolicy: Always

commonEnv:
Expand Down Expand Up @@ -268,8 +268,48 @@ worker:
memory: 800Mi

env:
ALEPH_WORKER_STAGES: "xref:reingest:reindex:loadmapping:flushmapping:exportsearch:exportxref:updateentity:pruneentity"
WORKER_THREADS: 0

# Aleph Indexing Worker - templates/worker-index.yaml
worker-index:
strategy:
rollingUpdate:
maxSurge: 2
maxUnavailable: 100%
type: RollingUpdate

podAnnotations: {}

env:
ALEPH_WORKER_STAGES: "index"
WORKER_THREADS: 0

nodeSelector: {}

podSecurityContext:
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000

image: {}

# Scaling:
replicas: 2

containerSecurityContext:
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false

containerResources:
requests:
cpu: 301m
memory: 300Mi
limits:
# cpu: 1000m
memory: 800Mi


# Aleph Prometheus Exporter - templates/exporter.yaml
exporter:
enabled: false
Expand Down
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

setup(
name="aleph",
version="4.0.0-rc41",
version="4.0.0-rc43",
description="Document sifting web frontend",
classifiers=[
"Intended Audience :: Developers",
Expand Down
2 changes: 1 addition & 1 deletion ui/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "aleph-ui",
"version": "4.0.0-rc41",
"version": "4.0.0-rc43",
"private": true,
"dependencies": {
"@alephdata/followthemoney": "^3.5.5",
Expand Down

0 comments on commit b9159bd

Please sign in to comment.