From 1008cc3b1216a851bdd22d19a56fa800836afdaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelizaveta=20Leme=C5=A1eva?= Date: Tue, 27 Aug 2024 08:50:39 +0200 Subject: [PATCH 1/4] feat(helm): add OpenSearch deployment (#827) --- helm/configurations/values-dev.yaml | 33 ++++ helm/reana/Chart.yaml | 4 + helm/reana/README.md | 12 ++ .../templates/reana-workflow-controller.yaml | 20 +++ helm/reana/templates/secrets.yaml | 86 +++++++++ helm/reana/values.yaml | 169 ++++++++++++++++++ 6 files changed, 324 insertions(+) diff --git a/helm/configurations/values-dev.yaml b/helm/configurations/values-dev.yaml index 5164f1ac..4f65aa82 100644 --- a/helm/configurations/values-dev.yaml +++ b/helm/configurations/values-dev.yaml @@ -14,6 +14,8 @@ components: image: docker.io/reanahub/reana-workflow-controller environment: REANA_RUNTIME_KUBERNETES_KEEP_ALIVE_JOBS_WITH_STATUSES: failed + REANA_OPENSEARCH_USE_SSL: false + REANA_OPENSEARCH_ENABLED: false # Set to true to enable live logs reana_workflow_engine_cwl: image: docker.io/reanahub/reana-workflow-engine-cwl reana_workflow_engine_yadage: @@ -31,3 +33,34 @@ components: pgbouncer: enabled: true + +# OpenSearch configuration for dev environment +opensearch: + enabled: false # Set to true to enable live logs + tls: + generate: false + singleNode: true + config: + opensearch.yml: | + cluster.name: reana-opensearch + network.host: 0.0.0.0 + plugins.security.disabled: true + securityConfig: + enabled: false + internalUsersSecret: + rolesSecret: + rolesMappingSecret: + resources: + requests: + cpu: "500m" + memory: "2Gi" + extraEnvs: + - name: DISABLE_INSTALL_DEMO_CONFIG + value: "false" + - name: OPENSEARCH_INITIAL_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: reana-opensearch-secrets + key: OPENSEARCH_INITIAL_ADMIN_PASSWORD + secretMounts: [] + customSecurityConfig: diff --git a/helm/reana/Chart.yaml b/helm/reana/Chart.yaml index 6a893f96..b03c7291 100644 --- a/helm/reana/Chart.yaml +++ b/helm/reana/Chart.yaml @@ -35,3 +35,7 @@ dependencies: condition: traefik.enabled tags: - ingress + - name: opensearch + version: 2.22.1 + repository: https://opensearch-project.github.io/helm-charts/ + condition: opensearch.enabled diff --git a/helm/reana/README.md b/helm/reana/README.md index fdfee412..cf75a19e 100644 --- a/helm/reana/README.md +++ b/helm/reana/README.md @@ -98,6 +98,18 @@ This Helm automatically prefixes all names using the release name to avoid colli | `reana_hostname` | REANA hostname (e.g. reana.example.org) | None | | `namespace_runtime` | Namespace in which the REANA runtime pods (workflow engines, jobs etc...) will run | `.Release.Namespace` | | `naming_scheme` | REANA component naming scheme | None | +| `opensearch.*` | Pass any value from [OpenSearch Helm chart values](https://github.com/opensearch-project/helm-charts/tree/main/charts/opensearch#configuration) here | - | +| `opensearch.enabled` | Enable OpenSearch | false | +| `opensearch.tls.generate` | Enable the generation of a self-signed TLS certificates for OpenSearch | true | +| `opensearch.tls.ca.cn` | OpenSearch root CA certificate common name (CN) | reana.io | +| `opensearch.tls.ca.ttl` | OpenSearch root CA certificate TTL in days | 365 | +| `opensearch.tls.cert.cn` | OpenSearch node certificate common name (CN) | reana-opensearch-master.default.svc.cluster.local | +| `opensearch.tls.cert.ttl` | OpenSearch node certificate TTL in days | 180 | +| `opensearch.tls.admin.cn` | OpenSearch admin certificate common name (CN) | opensearch-admin.reana.io | +| `opensearch.tls.admin.ttl` | OpenSearch admin certificate TTL in days | 180 | +| `opensearch.customSecurityConfig.internalUsers` | Provide YAML users configuration for `internal_users.yaml` file; see [documentation](https://opensearch.org/docs/latest/security/configuration/yaml/#internal_usersyml) | None | +| `opensearch.customSecurityConfig.roles` | Provide YAML roles configuration for `roles.yaml` file; see [documentation](https://opensearch.org/docs/latest/security/configuration/yaml/#rolesyml) | None | +| `opensearch.customSecurityConfig.rolesMapping` | Provide YAML roles mapping configuration for `roles_mapping.yaml` file; see [documentation](https://opensearch.org/docs/latest/security/configuration/yaml/#roles_mappingyml) | None | | `pgbouncer.enabled` | Instantiate PgBouncer inside the cluster to pool database connections | false | | `pgbouncer.image` | [PgBouncer image](https://hub.docker.com/r/bitnami/pgbouncer/) to use | `bitnami/pgbouncer:1.23.1` | | `pgbouncer.pool_mode` | Pool mode to use (session, transaction, statement) | transaction | diff --git a/helm/reana/templates/reana-workflow-controller.yaml b/helm/reana/templates/reana-workflow-controller.yaml index 73776964..1a9e52ca 100644 --- a/helm/reana/templates/reana-workflow-controller.yaml +++ b/helm/reana/templates/reana-workflow-controller.yaml @@ -83,6 +83,11 @@ spec: mountPath: {{ $workspace_path._1 }} {{- end }} {{- end }} + {{- if $opensearchTlsEnabled }} + - name: {{ include "reana.prefix" . }}-opensearch-tls-secrets + mountPath: /code/certs + readOnly: true + {{- end }} envFrom: - configMapRef: name: {{ include "reana.prefix" . }}-database-config @@ -216,6 +221,13 @@ spec: secretKeyRef: name: {{ include "reana.prefix" . }}-db-secrets key: password + {{- if $opensearchEnabled }} + - name: REANA_OPENSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "reana.prefix" . }}-opensearch-secrets + key: REANA_OPENSEARCH_PASSWORD + {{- end }} - name: job-status-consumer image: {{ .Values.components.reana_workflow_controller.image }} imagePullPolicy: {{ .Values.components.reana_workflow_controller.imagePullPolicy }} @@ -287,6 +299,14 @@ spec: hostPath: path: {{ .Values.shared_storage.hostpath.root_path }} {{- end }} + {{- if $opensearchTlsEnabled }} + - name: {{ include "reana.prefix" . }}-opensearch-tls-secrets + secret: + secretName: {{ include "reana.prefix" . }}-opensearch-tls-secrets + items: + - key: ca.crt + path: ca.crt + {{- end }} {{- if .Values.debug.enabled }} - name: reana-code hostPath: diff --git a/helm/reana/templates/secrets.yaml b/helm/reana/templates/secrets.yaml index b211ab5c..73a3798d 100644 --- a/helm/reana/templates/secrets.yaml +++ b/helm/reana/templates/secrets.yaml @@ -76,3 +76,89 @@ data: tls.crt: {{ $cert.Cert | b64enc | quote }} tls.key: {{ $cert.Key | b64enc | quote }} {{- end }} +--- +{{- if and .Values.opensearch.enabled .Values.opensearch.tls.generate }} +{{- $prefix := include "reana.prefix" . }} +{{- $tlsSecretName := printf "%s-%s" $prefix "opensearch-tls-secrets" }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $tlsSecretName }} + namespace: {{ .Release.Namespace }} +type: kubernetes.io/tls +data: + {{- $idx := lookup "v1" "Secret" .Release.Namespace $tlsSecretName -}} + {{- if $idx }} + tls.crt: {{ index $idx.data "tls.crt" }} + tls.key: {{ index $idx.data "tls.key" }} + admin.crt: {{ index $idx.data "admin.crt" }} + admin.key: {{ index $idx.data "admin.key" }} + ca.crt: {{ index $idx.data "ca.crt" }} + {{ else }} + {{- $ca := genCA .Values.opensearch.tls.ca.cn (.Values.opensearch.tls.ca.ttl | int) }} + {{- $cert := genSignedCert .Values.opensearch.tls.cert.cn nil nil (.Values.opensearch.tls.cert.ttl | int) $ca }} + {{- $certAdmin := genSignedCert .Values.opensearch.tls.admin.cn nil nil (.Values.opensearch.tls.admin.ttl | int) $ca }} + tls.crt: {{ $cert.Cert | b64enc | quote }} + tls.key: {{ $cert.Key | b64enc | quote }} + admin.crt: {{ $certAdmin.Cert | b64enc | quote }} + admin.key: {{ $certAdmin.Key | b64enc | quote }} + ca.crt: {{ $ca.Cert | b64enc | quote }} + {{- end }} +{{- end }} +--- +{{- if and .Values.opensearch.enabled .Values.opensearch.customSecurityConfig }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "reana.prefix" . }}-opensearch-config-secrets + namespace: {{ .Release.Namespace }} +type: kubernetes.io/opaque +stringData: + {{- if .Values.opensearch.customSecurityConfig.internalUsers }} + internal_users.yml: | + --- + # This is the internal user database + # The hash value is a bcrypt hash and can be generated with plugin/tools/hash.sh + _meta: + type: "internalusers" + config_version: 2 + {{ .Values.opensearch.customSecurityConfig.internalUsers | toYaml | nindent 4 }} + {{- end }} + {{- if .Values.opensearch.customSecurityConfig.rolesMapping }} + roles_mapping.yml: | + --- + # In this file users, backendroles and hosts can be mapped to Security roles. + # Permissions for OpenSearch roles are configured in roles.yml + _meta: + type: "rolesmapping" + config_version: 2 + {{ .Values.opensearch.customSecurityConfig.rolesMapping | toYaml | nindent 4 }} + {{- end }} + {{- if .Values.opensearch.customSecurityConfig.roles }} + roles.yml: | + --- + _meta: + type: "roles" + config_version: 2 + # The security REST API access role is used to assign specific users access to change the security settings through the REST API. + security_rest_api_access: + reserved: true + {{ .Values.opensearch.customSecurityConfig.roles | toYaml | nindent 4 }} + {{- end }} +{{- end }} +--- +{{- if .Values.opensearch.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "reana.prefix" . }}-opensearch-secrets + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/resource-policy": keep +type: Opaque +data: + REANA_OPENSEARCH_PASSWORD: {{ .Values.components.reana_workflow_controller.environment.REANA_OPENSEARCH_PASSWORD | default "reana" | b64enc }} + {{- if not .Values.opensearch.securityConfig.enabled }} + OPENSEARCH_INITIAL_ADMIN_PASSWORD: {{ .Values.opensearch.initialAdminPassword | default "reana" | b64enc }} # dev environment only + {{- end }} +{{- end }} diff --git a/helm/reana/values.yaml b/helm/reana/values.yaml index 4189eea5..23dcaff4 100644 --- a/helm/reana/values.yaml +++ b/helm/reana/values.yaml @@ -112,6 +112,11 @@ components: environment: SHARED_VOLUME_PATH: /var/reana REANA_JOB_STATUS_CONSUMER_PREFETCH_COUNT: 10 + REANA_OPENSEARCH_ENABLED: false + REANA_OPENSEARCH_USE_SSL: true + REANA_OPENSEARCH_CA_CERTS: "/code/certs/ca.crt" + REANA_OPENSEARCH_USER: reana + REANA_OPENSEARCH_PASSWORD: "" # Set this value in the Helm command reana_workflow_engine_cwl: image: docker.io/reanahub/reana-workflow-engine-cwl:0.9.3 environment: {} @@ -189,3 +194,167 @@ quota: # backward compatibility disk_update: "0 3 * * *" # everyday at 3am termination_update_policy: "" + +# OpenSearch chart values.yaml +opensearch: + enabled: false + clusterName: "reana-opensearch" + masterService: "reana-opensearch-master" + tls: + generate: true + ca: + cn: "reana.io" + ttl: 365 + cert: + cn: "reana-opensearch-master.default.svc.cluster.local" + ttl: 180 + admin: + cn: "opensearch-admin.reana.io" + ttl: 180 + singleNode: true # advanced storage configuration needed if set to false + config: + opensearch.yml: | + cluster.name: reana-opensearch + network.host: 0.0.0.0 + plugins: + security: + nodes_dn: + - "CN={{ .Values.tls.cert.cn }}" + authcz: + admin_dn: + - "CN={{ .Values.tls.admin.cn }}" + ssl: + transport: + pemcert_filepath: certs/tls.crt + pemkey_filepath: certs/tls.key + pemtrustedcas_filepath: certs/ca.crt + enforce_hostname_verification: false + http: + enabled: true + pemcert_filepath: certs/tls.crt + pemkey_filepath: certs/tls.key + pemtrustedcas_filepath: certs/ca.crt + allow_default_init_securityindex: true + check_snapshot_restore_write_privileges: true + enable_snapshot_restore_privilege: true + ssl_cert_reload_enabled: true # https://opensearch.org/docs/latest/security/access-control/api/#reload-transport-certificates + restapi: + roles_enabled: + - all_access + - security_rest_api_access + system_indices: + enabled: true + indices: + [ + ".opendistro-alerting-config", + ".opendistro-alerting-alert*", + ".opendistro-anomaly-results*", + ".opendistro-anomaly-detector*", + ".opendistro-anomaly-checkpoints", + ".opendistro-anomaly-detection-state", + ".opendistro-reports-*", + ".opendistro-notifications-*", + ".opendistro-notebooks", + ".opendistro-asynchronous-search-response*", + ] + extraEnvs: + - name: DISABLE_INSTALL_DEMO_CONFIG + value: "true" + secretMounts: + - name: reana-opensearch-tls-secrets + secretName: reana-opensearch-tls-secrets + path: /usr/share/opensearch/config/certs + resources: + requests: + cpu: "1000m" + memory: "4Gi" + persistence: + enabled: false + securityConfig: + enabled: true + internalUsersSecret: "reana-opensearch-config-secrets" + rolesSecret: "reana-opensearch-config-secrets" + rolesMappingSecret: "reana-opensearch-config-secrets" + extraVolumes: + - name: reana-opensearch-volume + hostPath: + path: /var/reana + # You can instead configure infrastructure volume: + # - name: reana-opensearch-volume + # persistentVolumeClaim: + # claimName: reana-infrastructure-persistent-volume + # readOnly: false + # Or shared volume: + # - name: reana-opensearch-volume + # persistentVolumeClaim: + # claimName: reana-shared-persistent-volume + # readOnly: false + extraVolumeMounts: + - mountPath: /usr/share/opensearch/data + subPath: opensearch + name: reana-opensearch-volume + # Configure REANA and FluentBit users and roles for job log collection + customSecurityConfig: + internalUsers: + reana: + hash: "" # Required. To generate hash, run plugins/opensearch-security/tools/hash.sh -p ; supply in Helm command flags + reserved: false + backend_roles: + - readall + description: REANA user + fluentbit: + hash: "" # Required. To generate hash, run plugins/opensearch-security/tools/hash.sh -p ; supply in Helm command flags + reserved: false + backend_roles: + - fluentbit + description: FluentBit user + roles: + fluentbit: + reserved: true + hidden: false + description: Provide the minimum permissions for fluentbit + cluster_permissions: + - cluster_monitor + - cluster_composite_ops + - indices:admin/template/get + - indices:admin/template/put + - cluster:admin/ingest/pipeline/put + - cluster:admin/ingest/pipeline/get + - indices:data/write/bulk* + index_permissions: + - index_patterns: + - fluentbit-* + fls: [] + masked_fields: [] + allowed_actions: + - crud + - create_index + tenant_permissions: [] + static: true + rolesMapping: + fluentbit: + hosts: [] + users: [] + reserved: false + hidden: false + backend_roles: + - fluentbit + and_backend_roles: [] + own_index: + hosts: [] + users: + - "*" + reserved: false + hidden: false + backend_roles: [] + and_backend_roles: [] + description: Allow full access to an index named like the username + readall: + hosts: [] + users: + - "reana" + reserved: false + hidden: false + backend_roles: + - readall + and_backend_roles: [] From 3a5adcc9bfb6f8d072a30513b4e2a877dcf08552 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelizaveta=20Leme=C5=A1eva?= Date: Fri, 30 Aug 2024 09:19:37 +0200 Subject: [PATCH 2/4] feat(helm): add FluentBit DaemonSet (#827) --- helm/configurations/values-dev.yaml | 9 + helm/reana/Chart.yaml | 4 + helm/reana/README.md | 23 +++ helm/reana/templates/priority-classes.yaml | 11 ++ helm/reana/values.yaml | 189 +++++++++++++++++++++ 5 files changed, 236 insertions(+) create mode 100644 helm/reana/templates/priority-classes.yaml diff --git a/helm/configurations/values-dev.yaml b/helm/configurations/values-dev.yaml index 4f65aa82..6ce2343a 100644 --- a/helm/configurations/values-dev.yaml +++ b/helm/configurations/values-dev.yaml @@ -64,3 +64,12 @@ opensearch: key: OPENSEARCH_INITIAL_ADMIN_PASSWORD secretMounts: [] customSecurityConfig: + +# FluentBit configuration for dev environment +fluent-bit: + enabled: false # Set to true to enable live logs + outputConfig: + tls: "Off" + tlsCaFile: "" + extraVolumes: [] + extraVolumeMounts: [] diff --git a/helm/reana/Chart.yaml b/helm/reana/Chart.yaml index b03c7291..6253e1e0 100644 --- a/helm/reana/Chart.yaml +++ b/helm/reana/Chart.yaml @@ -39,3 +39,7 @@ dependencies: version: 2.22.1 repository: https://opensearch-project.github.io/helm-charts/ condition: opensearch.enabled + - name: fluent-bit + version: 0.47.7 + repository: https://fluent.github.io/helm-charts + condition: fluent-bit.enabled diff --git a/helm/reana/README.md b/helm/reana/README.md index cf75a19e..a65cc5b1 100644 --- a/helm/reana/README.md +++ b/helm/reana/README.md @@ -65,6 +65,29 @@ This Helm automatically prefixes all names using the release name to avoid colli | `db_env_config.REANA_DB_PORT` | Environment variable to connect to external databases | "5432" | | `debug.enabled` | Instantiate a [wdb](https://github.com/Kozea/wdb) remote debugger inside the cluster, accessible in port `31984` | false | | `eos.enabled` | **[CERN only]** Enable EOS support inside the cluster | false | +| `fluent-bit.enabled` | Enable FluentBit | false | +| `fluent-bit.inputConfig.*` | Pass certain `tail` input [configuration parameters](https://docs.fluentbit.io/manual/pipeline/inputs/tail#config)| | +| `fluent-bit.inputConfig.refreshInterval` | `tail` input configuration parameter `Refresh_Interval` | 2 | +| `fluent-bit.inputConfig.rotateWait` | `tail` input configuration parameter `Rotate_Wait` | 5 | +| `fluent-bit.inputConfig.skipLongLines` | `tail` input configuration parameter `Skip_Long_Lines` | On | +| `fluent-bit.inputConfig.skipEmptyLines` | `tail` input configuration parameter `Skip_Empty_Lines` | On | +| `fluent-bit.filterConfig.*` | Pass certain `kubernetes` filter [configuration parameters](https://docs.fluentbit.io/manual/pipeline/filters/kubernetes#configuration-parameters)| | +| `fluent-bit.filterConfig.bufferSize` | `kubernetes` filter configuration parameter `Buffer_Size` | 512k | +| `fluent-bit.filterConfig.kubeUrl` | `kubernetes` filter configuration parameter `Kube_URL` | https://kubernetes.default.svc:443 | +| `fluent-bit.filterConfig.kubeCaFile` | `kubernetes` filter configuration parameter `Kube_CA_File` | /var/run/secrets/kubernetes.io/serviceaccount/ca.crt | +| `fluent-bit.filterConfig.kubeTokenFile` | `kubernetes` filter configuration parameter `Kube_Token_File` | /var/run/secrets/kubernetes.io/serviceaccount/token | +| `fluent-bit.outputConfig.*` | Pass certain `opensearch` output [configuration parameters](https://docs.fluentbit.io/manual/pipeline/outputs/opensearch#configuration-parameters)| | +| `fluent-bit.outputConfig.host` | `opensearch` output configuration parameter `Host` | reana-opensearch-master | +| `fluent-bit.outputConfig.httpUser` | `opensearch` output configuration parameter `HTTP_User` | fluentbit | +| `fluent-bit.outputConfig.httpPasswd` | `opensearch` output configuration parameter `HTTP_Passwd` | None | +| `fluent-bit.outputConfig.tls` | `opensearch` output configuration parameter `tls` | "On" | +| `fluent-bit.outputConfig.tlsVerify` | `opensearch` output configuration parameter `tls.verify` | "On" | +| `fluent-bit.outputConfig.tlsVerifyHostname` | `opensearch` output configuration parameter `tls.verify_hostname` | "Off" | +| `fluent-bit.outputConfig.tlsCaFile` | `opensearch` output configuration parameter `tls.ca_file` | /fluent-bit/etc/certs/ca.crt | +| `fluent-bit.outputConfig.tlsCrtFile` | `opensearch` output configuration parameter `tls.crt_file` | "" | +| `fluent-bit.outputConfig.tlsKeyFile` | `opensearch` output configuration parameter `tls.key_file` | "" | +| `fluent-bit.outputConfig.tlsKeyPassword` | `opensearch` output configuration parameter `tls.key_passwd` | "" | +| `fluent-bit.priority` | Priority class value for FluentBit pods | 1000000 | | `fullnameOverride` | Name to override the `reana.prefix` | None | | `infrastructure_storage` | Optional volume used by REANA's infrastructure (i.e. database and message broker). It has the same settings as `shared_storage` | {} | | `ingress.annotations.traefik.ingress.kubernetes.io/router.entrypoints` | Entrypoints allowed by the ingress controller | "web,websecure" | diff --git a/helm/reana/templates/priority-classes.yaml b/helm/reana/templates/priority-classes.yaml new file mode 100644 index 00000000..59d0dbe3 --- /dev/null +++ b/helm/reana/templates/priority-classes.yaml @@ -0,0 +1,11 @@ +{{- if index .Values "fluent-bit" "enabled" }} +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: {{ include "reana.prefix" . }}-fluent-bit-priority-class +value: {{ index .Values "fluent-bit" "priority" | default 1000000 }} +preemptionPolicy: Never +globalDefault: false +description: "PriorityClass for FluentBit DaemonSet pods. This priority class will not cause other pods to be preempted." +{{- end }} diff --git a/helm/reana/values.yaml b/helm/reana/values.yaml index 23dcaff4..0b3471b1 100644 --- a/helm/reana/values.yaml +++ b/helm/reana/values.yaml @@ -358,3 +358,192 @@ opensearch: backend_roles: - readall and_backend_roles: [] + +# FluentBit chart values.yaml +fluent-bit: + enabled: false + inputConfig: + refreshInterval: 2 + rotateWait: 5 + skipLongLines: "On" + skipEmptyLines: "On" + filterConfig: + bufferSize: 512k + kubeUrl: https://kubernetes.default.svc:443 + kubeCaFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + kubeTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + outputConfig: + host: reana-opensearch-master.default.svc.cluster.local + httpUser: fluentbit + httpPasswd: + tls: "On" + tlsVerify: "On" + tlsVerifyHostname: "On" + tlsCaFile: /fluent-bit/etc/certs/ca.crt + tlsCrtFile: "" + tlsKeyFile: "" + tlsKeyPassword: "" + ## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/classic-mode/configuration-file + config: + service: | + [SERVICE] + Daemon Off + Flush {{ .Values.flush }} + Log_Level {{ .Values.logLevel }} + Parsers_File /fluent-bit/etc/parsers.conf + Parsers_File /fluent-bit/etc/conf/custom_parsers.conf + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_Port {{ .Values.metricsPort }} + Health_Check On + + # ## https://docs.fluentbit.io/manual/pipeline/inputs + inputs: | + [INPUT] + Name tail + Path /var/log/containers/reana-run-job-* + multiline.parser docker, cri + Tag kube.* + Skip_Long_Lines {{ .Values.inputConfig.skipLongLines }} + Skip_Empty_Lines {{ .Values.inputConfig.skipEmptyLines }} + Refresh_Interval {{ .Values.inputConfig.refreshInterval }} + Rotate_Wait {{ .Values.inputConfig.rotateWait }} + + [INPUT] + Name tail + Path /var/log/containers/reana-run-batch-* + multiline.parser docker, cri + Tag kube.* + Skip_Long_Lines {{ .Values.inputConfig.skipLongLines }} + Skip_Empty_Lines {{ .Values.inputConfig.skipEmptyLines }} + Refresh_Interval {{ .Values.inputConfig.refreshInterval }} + Rotate_Wait {{ .Values.inputConfig.rotateWait }} + + ## https://docs.fluentbit.io/manual/pipeline/filters + filters: | + [FILTER] + Name kubernetes + Buffer_Size {{ .Values.filterConfig.bufferSize }} + Match kube.* + Annotations Off + Kube_Tag_Prefix kube.var.log.containers. + Kube_URL {{ .Values.filterConfig.kubeUrl }} + Kube_CA_File {{ .Values.filterConfig.kubeCaFile }} + Kube_Token_File {{ .Values.filterConfig.kubeTokenFile }} + + [FILTER] + Name parser + Match kube.var.log.containers.reana-run-batch-*job-controller* + Key_name log + Parser reana-capture + Reserve_Data On + Preserve_Key On + + [FILTER] + Name grep + Match kube.var.log.containers.reana-run-batch-*job-controller* + Logical_Op or + Regex level ERROR + Regex level WARNING + + [FILTER] + Name nest + Match kube.* + Operation lift + Nested_under kubernetes + Add_prefix kubernetes. + + [FILTER] + Name nest + Match kube.* + Operation lift + Nested_under kubernetes.labels + Add_prefix kubernetes.labels. + + [FILTER] + Name record_modifier + Match kube.* + Remove_key time + Remove_key stream + Remove_key kubernetes.pod_name + Remove_key kubernetes.namespace_name + Remove_key kubernetes.pod_id + Remove_key kubernetes.labels.batch.kubernetes.io/controller-uid + Remove_key kubernetes.labels.batch.kubernetes.io/job-name + Remove_key kubernetes.labels.controller-uid + Remove_key kubernetes.labels.reana-run-job-workflow-uuid + Remove_key kubernetes.labels.reana_workflow_mode + Remove_key kubernetes.annotations.* + Remove_key kubernetes.host + Remove_key kubernetes.container_name + Remove_key kubernetes.docker_id + Remove_key kubernetes.container_hash + Remove_key kubernetes.container_image + Remove_key _p + + [FILTER] + Name record_modifier + Match kube.var.log.containers.reana-run-job-* + Remove_key kubernetes.labels.reana-run-batch-workflow-uuid + + [FILTER] + Name record_modifier + Match kube.var.log.containers.reana-run-batch-* + Remove_key kubernetes.labels.job-name + + ## https://docs.fluentbit.io/manual/pipeline/outputs + outputs: | + [OUTPUT] + Name opensearch + Match kube.var.log.containers.reana-run-job-* + Host {{ .Values.outputConfig.host }} + {{- if .Values.outputConfig.httpPasswd }} + HTTP_User {{ .Values.outputConfig.httpUser }} + HTTP_Passwd {{ .Values.outputConfig.httpPasswd }} + {{- end }} + Index fluentbit-job_log + Suppress_Type_Name On + tls {{ .Values.outputConfig.tls }} + tls.verify {{ .Values.outputConfig.tlsVerify }} + tls.verify_hostname {{ .Values.outputConfig.tlsVerifyHostname }} + {{ if .Values.outputConfig.tlsCaFile }}tls.ca_file {{ .Values.outputConfig.tlsCaFile }}{{- end }} + {{ if .Values.outputConfig.tlsCrtFile }}tls.crt_file {{ .Values.outputConfig.tlsCrtFile }}{{- end }} + {{ if .Values.outputConfig.tlsKeyFile }}tls.key_file {{ .Values.outputConfig.tlsKeyFile }}{{- end }} + {{ if .Values.outputConfig.tlsKeyPassword }}tls.key_password {{ .Values.outputConfig.tlsKeyPassword }}{{- end }} + + [OUTPUT] + Name opensearch + Match kube.var.log.containers.reana-run-batch-* + Host {{ .Values.outputConfig.host }} + {{- if .Values.outputConfig.httpPasswd }} + HTTP_User {{ .Values.outputConfig.httpUser }} + HTTP_Passwd {{ .Values.outputConfig.httpPasswd }} + {{- end }} + Index fluentbit-workflow_log + Suppress_Type_Name On + tls {{ .Values.outputConfig.tls }} + tls.verify {{ .Values.outputConfig.tlsVerify }} + tls.verify_hostname {{ .Values.outputConfig.tlsVerifyHostname }} + {{ if .Values.outputConfig.tlsCaFile }}tls.ca_file {{ .Values.outputConfig.tlsCaFile }}{{- end }} + {{ if .Values.outputConfig.tlsCrtFile }}tls.crt_file {{ .Values.outputConfig.tlsCrtFile }}{{- end }} + {{ if .Values.outputConfig.tlsKeyFile }}tls.key_file {{ .Values.outputConfig.tlsKeyFile }}{{- end }} + {{ if .Values.outputConfig.tlsKeyPassword }}tls.key_password {{ .Values.outputConfig.tlsKeyPassword }}{{- end }} + + ## https://docs.fluentbit.io/manual/pipeline/parsers + customParsers: | + [PARSER] + Name reana-capture + Format regex + Regex /^(?[1-9:\-,\w ]+) \| (?[a-zA-Z1-9]+) \| (?[a-zA-Z1-9_\- \w\(\)]+) \| (?[a-zA-Z]+) \| (?.*)/m + extraVolumes: + - name: reana-opensearch-ca + secret: + secretName: reana-opensearch-tls-secrets + items: + - key: ca.crt + path: ca.crt + extraVolumeMounts: + - name: reana-opensearch-ca + mountPath: /fluent-bit/etc/certs + readOnly: true + priorityClassName: "reana-fluent-bit-priority-class" From d020538626ae2ab49cd8cf3a5c65f761cdb8f8d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelizaveta=20Leme=C5=A1eva?= Date: Fri, 30 Aug 2024 09:22:02 +0200 Subject: [PATCH 3/4] feat(reana_dev): prevent Helm values string wrapping by PyYAML (#827) --- reana/reana_dev/cluster.py | 9 +- tests/test_cluster.py | 174 +++++++++++++++++++++++++++++++++++++ 2 files changed, 180 insertions(+), 3 deletions(-) create mode 100644 tests/test_cluster.py diff --git a/reana/reana_dev/cluster.py b/reana/reana_dev/cluster.py index 0a84e769..5c4437de 100644 --- a/reana/reana_dev/cluster.py +++ b/reana/reana_dev/cluster.py @@ -349,7 +349,7 @@ def job_mounts_to_config(job_mounts): values_dict = {} with open(os.path.join(get_srcdir("reana"), values)) as f: - values_dict = yaml.safe_load(f.read()) + values_dict = yaml.safe_load(f.read()) or {} job_mount_config = job_mounts_to_config(job_mounts) if job_mount_config: @@ -365,9 +365,12 @@ def job_mounts_to_config(job_mounts): find_standard_component_name(c) for c in exclude_components.split(",") ] if "reana-ui" in standard_named_exclude_components: - values_dict["components"]["reana_ui"]["enabled"] = False + values_dict.setdefault("components", {}).setdefault("reana_ui", {})[ + "enabled" + ] = False - values_yaml = yaml.dump(values_dict) if values_dict else "" + # set arbitrary big value for `width` to prevent PyYAML from wrapping long lines + values_yaml = yaml.dump(values_dict, width=100000) if values_dict else "" helm_install = f"cat < Date: Tue, 3 Sep 2024 16:29:09 +0200 Subject: [PATCH 4/4] feat(helm): connect r-workflow-controller to OpenSearch (#827) --- helm/reana/README.md | 8 ++++++++ helm/reana/templates/reana-workflow-controller.yaml | 3 +++ helm/reana/templates/secrets.yaml | 4 ++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/helm/reana/README.md b/helm/reana/README.md index a65cc5b1..d07e33f0 100644 --- a/helm/reana/README.md +++ b/helm/reana/README.md @@ -51,6 +51,14 @@ This Helm automatically prefixes all names using the release name to avoid colli | `components.reana_workflow_controller.environment.REANA_JOB_HOSTPATH_MOUNTS` | JSON list of optional hostPath mounts, for all user jobs. Each mount object has a key `name` (name of the mount), `hostPath` (path to the directory to be mounted from the Kubernetes nodes) and `mountPath` (path inside the job containers where the `hostPath` will be mounted) | None | | `components.reana_workflow_controller.environment.REANA_RUNTIME_KUBERNETES_KEEP_ALIVE_JOBS_WITH_STATUSES` | Keep alive Kubernetes user runtime jobs depending on status (`finished` and/or `failed`). | None | | `components.reana_workflow_controller.environment.REANA_JOB_STATUS_CONSUMER_PREFETCH_COUNT` | Define max number of unacknowledged deliveries that are permitted on `jobs-status` queue consumer. | 10 | +| `components.reana_workflow_controller.environment.REANA_OPENSEARCH_ENABLED` | Enable workflow and job log retrieval from OpenSearch. | false | +| `components.reana_workflow_controller.environment.REANA_OPENSEARCH_HOST` | OpenSearch host. | None | +| `components.reana_workflow_controller.environment.REANA_OPENSEARCH_PORT` | OpenSearch port. | None | +| `components.reana_workflow_controller.environment.REANA_OPENSEARCH_URL_PREFIX` | OpenSearch URL prefix. | None | +| `components.reana_workflow_controller.environment.REANA_OPENSEARCH_USE_SSL` | Use SSL when connecting to OpenSearch instance. | true | +| `components.reana_workflow_controller.environment.REANA_OPENSEARCH_CA_CERTS` | Path to a file with OpenSearch root CA certificates. | "/code/certs/ca.crt" | +| `components.reana_workflow_controller.environment.REANA_OPENSEARCH_USER` | OpenSearch user name for Basic Authentication. | reana | +| `components.reana_workflow_controller.environment.REANA_OPENSEARCH_PASSWORD` | OpenSearch password for Basic Authentication. Set this value in the Helm command. | "" | | `components.reana_workflow_engine_cwl.environment` | [REANA-Workflow-Engine-CWL](https://github.com/reanahub/reana-workflow-engine-cwl) environment variables | `{}` | | `components.reana_workflow_engine_cwl.image` | [REANA-Workflow-Engine-CWL image](https://hub.docker.com/r/reanahub/reana-workflow-engine-cwl) to use | `docker.io/reanahub/reana-workflow-engine-cwl:` | | `components.reana_workflow_engine_serial.environment` | [REANA-Workflow-Engine-Serial](https://github.com/reanahub/reana-workflow-engine-serial) environment variables | `{}` | diff --git a/helm/reana/templates/reana-workflow-controller.yaml b/helm/reana/templates/reana-workflow-controller.yaml index 1a9e52ca..a8cdc8d7 100644 --- a/helm/reana/templates/reana-workflow-controller.yaml +++ b/helm/reana/templates/reana-workflow-controller.yaml @@ -1,3 +1,6 @@ +{{- $opensearchEnv := .Values.components.reana_workflow_controller.environment }} +{{- $opensearchEnabled := $opensearchEnv.REANA_OPENSEARCH_ENABLED }} +{{- $opensearchTlsEnabled := and $opensearchEnabled $opensearchEnv.REANA_OPENSEARCH_USE_SSL }} --- apiVersion: v1 kind: Service diff --git a/helm/reana/templates/secrets.yaml b/helm/reana/templates/secrets.yaml index 73a3798d..9db5992a 100644 --- a/helm/reana/templates/secrets.yaml +++ b/helm/reana/templates/secrets.yaml @@ -96,8 +96,8 @@ data: ca.crt: {{ index $idx.data "ca.crt" }} {{ else }} {{- $ca := genCA .Values.opensearch.tls.ca.cn (.Values.opensearch.tls.ca.ttl | int) }} - {{- $cert := genSignedCert .Values.opensearch.tls.cert.cn nil nil (.Values.opensearch.tls.cert.ttl | int) $ca }} - {{- $certAdmin := genSignedCert .Values.opensearch.tls.admin.cn nil nil (.Values.opensearch.tls.admin.ttl | int) $ca }} + {{- $cert := genSignedCert .Values.opensearch.tls.cert.cn nil (list .Values.opensearch.tls.cert.cn) (.Values.opensearch.tls.cert.ttl | int) $ca }} + {{- $certAdmin := genSignedCert .Values.opensearch.tls.admin.cn nil (list .Values.opensearch.tls.cert.cn) (.Values.opensearch.tls.admin.ttl | int) $ca }} tls.crt: {{ $cert.Cert | b64enc | quote }} tls.key: {{ $cert.Key | b64enc | quote }} admin.crt: {{ $certAdmin.Cert | b64enc | quote }}