alephdata · stchris · Jul 22, 2024 · Jul 22, 2024 · Jul 22, 2024 · Jul 22, 2024
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 4.0.0-rc41
+current_version = 4.0.0-rc43
 tag_name = {new_version}
 commit = True
 tag = True

diff --git a/contrib/aleph-traefik-minio-keycloak/docker-compose.yml b/contrib/aleph-traefik-minio-keycloak/docker-compose.yml
@@ -54,7 +54,7 @@ services:
       - "traefik.enable=false"
 
   worker:
-    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc41}
+    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc43}
     command: aleph worker
     restart: on-failure
     links:
@@ -79,7 +79,7 @@ services:
       - "traefik.enable=false"
 
   shell:
-    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc41}
+    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc43}
     command: /bin/bash
     depends_on:
       - postgres
@@ -99,7 +99,7 @@ services:
       - "traefik.enable=false"
 
   api:
-    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc41}
+    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc43}
     command: gunicorn -w 6 -b 0.0.0.0:8000 --log-level debug --log-file - aleph.wsgi:app
     expose:
       - 8000
@@ -121,7 +121,7 @@ services:
       - "traefik.enable=false"
 
   ui:
-    image: ghcr.io/alephdata/aleph-ui-production:${ALEPH_TAG:-4.0.0-rc41}
+    image: ghcr.io/alephdata/aleph-ui-production:${ALEPH_TAG:-4.0.0-rc43}
     depends_on:
       - api
       - traefik

diff --git a/contrib/keycloak/docker-compose.dev-keycloak.yml b/contrib/keycloak/docker-compose.dev-keycloak.yml
@@ -16,7 +16,7 @@ services:
   elasticsearch:
     build:
       context: services/elasticsearch
-    image: ghcr.io/alephdata/aleph-elasticsearch:${ALEPH_TAG:-4.0.0-rc41}
+    image: ghcr.io/alephdata/aleph-elasticsearch:${ALEPH_TAG:-4.0.0-rc43}
     hostname: elasticsearch
     environment:
       - discovery.type=single-node
@@ -55,7 +55,7 @@ services:
   app:
     build:
       context: .
-    image: alephdata/aleph:${ALEPH_TAG:-4.0.0-rc41}
+    image: alephdata/aleph:${ALEPH_TAG:-4.0.0-rc43}
     hostname: aleph
     command: /bin/bash
     links:
@@ -83,7 +83,7 @@ services:
   api:
     build:
       context: .
-    image: alephdata/aleph:${ALEPH_TAG:-4.0.0-rc41}
+    image: alephdata/aleph:${ALEPH_TAG:-4.0.0-rc43}
     command: aleph run -h 0.0.0.0 -p 5000 --with-threads --reload --debugger
     ports:
       - "127.0.0.1:5000:5000"
@@ -117,7 +117,7 @@ services:
   ui:
     build:
       context: ui
-    image: alephdata/aleph-ui:${ALEPH_TAG:-4.0.0-rc41}
+    image: alephdata/aleph-ui:${ALEPH_TAG:-4.0.0-rc43}
     links:
       - api
     command: npm run start

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -46,7 +46,7 @@ services:
       - aleph.env
 
   worker:
-    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc41}
+    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc43}
     command: aleph worker
     restart: on-failure
     depends_on:
@@ -62,7 +62,7 @@ services:
       - aleph.env
 
   shell:
-    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc41}
+    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc43}
     command: /bin/bash
     depends_on:
       - postgres
@@ -80,7 +80,7 @@ services:
       - aleph.env
 
   api:
-    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc41}
+    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-4.0.0-rc43}
     expose:
       - 8000
     depends_on:
@@ -97,7 +97,7 @@ services:
       - aleph.env
 
   ui:
-    image: ghcr.io/alephdata/aleph-ui-production:${ALEPH_TAG:-4.0.0-rc41}
+    image: ghcr.io/alephdata/aleph-ui-production:${ALEPH_TAG:-4.0.0-rc43}
     depends_on:
       - api
     ports:

diff --git a/helm/charts/aleph/Chart.yaml b/helm/charts/aleph/Chart.yaml
@@ -2,5 +2,5 @@ apiVersion: v2
 name: aleph
 description: Helm chart for Aleph
 type: application
-version: 4.0.0-rc41
-appVersion: 4.0.0-rc41
+version: 4.0.0-rc43
+appVersion: 4.0.0-rc43
diff --git a/helm/charts/aleph/README.md b/helm/charts/aleph/README.md
@@ -11,7 +11,7 @@ Helm chart for Aleph
 | global.amazon | bool | `true` | Are we using AWS services like s3? |
 | global.google | bool | `false` | Are we using GCE services like storage, vision api? |
 | global.image.repository | string | `"alephdata/aleph"` | Aleph docker image repo |
-| global.image.tag | string | `"4.0.0-rc41"` | Aleph docker image tag |
+| global.image.tag | string | `"4.0.0-rc43"` | Aleph docker image tag |
 | global.image.tag | string | `"Always"` |  |
 | global.namingPrefix | string | `"aleph"` | Prefix for the names of k8s resources |
 

diff --git a/helm/charts/aleph/templates/worker-index.yaml b/helm/charts/aleph/templates/worker-index.yaml
@@ -0,0 +1,149 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Values.global.namingPrefix }}-worker-index
+  labels:
+    app: {{ .Values.global.namingPrefix }}-worker-index
+spec:
+  replicas: {{ .Values.workerindex.replicas }}
+  selector:
+    matchLabels:
+      app: {{ .Values.global.namingPrefix }}-worker-index
+  strategy: {{- toYaml .Values.workerindex.strategy | nindent 4 }}
+  template:
+    metadata:
+      labels:
+        app: {{ .Values.global.namingPrefix }}-worker-index
+      annotations: {{- toYaml .Values.workerindex.podAnnotations | nindent 8 }}
+    spec:
+      nodeSelector: {{- toYaml .Values.workerindex.nodeSelector | nindent 8 }}
+      restartPolicy: Always
+      securityContext: {{- toYaml .Values.workerindex.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          image: "{{ .Values.global.image.repository }}:{{ .Values.global.image.tag }}"
+          imagePullPolicy: {{ .Values.global.image.pullPolicy }}
+          command: ["aleph", "worker"]
+          volumeMounts:
+            {{ if .Values.global.google }}
+            - mountPath: /var/secrets/google
+              name: service-account-aleph
+              readOnly: true
+            {{ end }}
+            - mountPath: /tmp
+              name: tmp-volume
+            - mountPath: /home/app
+              name: home-volume
+          securityContext:
+            {{- toYaml .Values.workerindex.containerSecurityContext | nindent 12 }}
+          resources:
+            {{- toYaml .Values.workerindex.containerResources | nindent 12 }}
+          env:
+            {{- range $key, $value := .Values.global.commonEnv }}
+            - name: {{ $key }}
+              value: {{ $value | quote }}
+            {{- end }}
+            {{- range $key, $value := .Values.global.env }}
+            - name: {{ $key }}
+              value: {{ $value | quote }}
+            {{- end }}
+            {{- range $key, $value := .Values.workerindex.env }}
+            - name: {{ $key }}
+              value: {{ $value | quote }}
+            {{- end }}
+            - name: ALEPH_DATABASE_URI
+              valueFrom:
+                secretKeyRef:
+                  name: aleph-secrets
+                  key: ALEPH_DATABASE_URI
+            - name: ALEPH_OAUTH_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: aleph-secrets
+                  key: ALEPH_OAUTH_SECRET
+            - name: ALEPH_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: aleph-secrets
+                  key: ALEPH_SECRET_KEY
+            - name: FTM_STORE_URI
+              valueFrom:
+                secretKeyRef:
+                  name: aleph-secrets
+                  key: FTM_STORE_URI
+            - name: RABBITMQ_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: aleph-secrets
+                  key: RABBITMQ_PASSWORD
+                  optional: true
+            - name: SENTRY_DSN
+              valueFrom:
+                secretKeyRef:
+                  name: aleph-secrets
+                  key: SENTRY_DSN
+                  optional: true
+            {{ if .Values.global.google }}
+            - name: GOOGLE_APPLICATION_CREDENTIALS
+              value: /var/secrets/google/service-account.json
+            {{ end }}
+            {{ if .Values.global.amazon }}
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: aleph-secrets
+                  key: AWS_ACCESS_KEY_ID
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: aleph-secrets
+                  key: AWS_SECRET_ACCESS_KEY
+            {{ end }}
+            {{ if .Values.global.prometheus.enabled }}
+            - name: PROMETHEUS_ENABLED
+              value: "true"
+            {{ end }}
+      volumes:
+        {{ if .Values.global.google }}
+        - name: service-account-aleph
+          secret:
+            secretName: service-account-aleph
+        {{ end }}
+        - name: home-volume
+          emptyDir:
+            medium: Memory
+        - name: tmp-volume
+          emptyDir: {}
+---
+{{ if .Values.global.prometheus.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.global.namingPrefix }}-worker-index
+  labels:
+    app: {{ .Values.global.namingPrefix }}-worker-index
+spec:
+  selector:
+    app: {{ .Values.global.namingPrefix }}-worker-index
+  ports:
+    - port: 9100
+      targetPort: 9100
+      protocol: "TCP"
+      name: metrics
+{{ end }}
+---
+{{ if .Values.global.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ .Values.global.namingPrefix }}-worker-index
+  labels:
+    app: {{ .Values.global.namingPrefix }}-worker-index
+spec:
+  selector:
+    matchLabels:
+      app: {{ .Values.global.namingPrefix }}-worker-index
+  endpoints:
+    - interval: "30s"
+      port: metrics
+{{ end }}
diff --git a/helm/charts/aleph/values.yaml b/helm/charts/aleph/values.yaml
@@ -6,7 +6,7 @@ global:
 
   image:
     repository: ghcr.io/alephdata/aleph
-    tag: "4.0.0-rc41"
+    tag: "4.0.0-rc43"
     pullPolicy: Always
 
   commonEnv:
@@ -268,8 +268,48 @@ worker:
       memory: 800Mi
 
   env:
+    ALEPH_WORKER_STAGES: "xref:reingest:reindex:loadmapping:flushmapping:exportsearch:exportxref:updateentity:pruneentity"
     WORKER_THREADS: 0
 
+# Aleph Indexing Worker - templates/worker-index.yaml
+worker-index:
+  strategy:
+    rollingUpdate:
+      maxSurge: 2
+      maxUnavailable: 100%
+    type: RollingUpdate
+
+  podAnnotations: {}
+
+  env:
+    ALEPH_WORKER_STAGES: "index"
+    WORKER_THREADS: 0
+
+  nodeSelector: {}
+
+  podSecurityContext:
+    runAsUser: 1000
+    runAsGroup: 1000
+    fsGroup: 1000
+
+  image: {}
+
+  # Scaling:
+  replicas: 2
+
+  containerSecurityContext:
+    readOnlyRootFilesystem: true
+    allowPrivilegeEscalation: false
+
+  containerResources:
+    requests:
+      cpu: 301m
+      memory: 300Mi
+    limits:
+      # cpu: 1000m
+      memory: 800Mi
+
+
 # Aleph Prometheus Exporter - templates/exporter.yaml
 exporter:
   enabled: false

diff --git a/setup.py b/setup.py
@@ -2,7 +2,7 @@
 
 setup(
     name="aleph",
-    version="4.0.0-rc41",
+    version="4.0.0-rc43",
     description="Document sifting web frontend",
     classifiers=[
         "Intended Audience :: Developers",

diff --git a/ui/package.json b/ui/package.json
@@ -1,6 +1,6 @@
 {
   "name": "aleph-ui",
-  "version": "4.0.0-rc41",
+  "version": "4.0.0-rc43",
   "private": true,
   "dependencies": {
     "@alephdata/followthemoney": "^3.5.5",