Add OpenSearch 2.x and 3.x persistence modules with versioned indexing types

docker/README.md

@@ -31,12 +31,12 @@ Refer to the table below for various supported backend and required configuratio
 > | cassandra  | conductor.db.type=cassandra        |    
 >
 
-Conductor is using Elasticsearch or OpenSearch for indexing the workflow data.  
-Currently, Elasticsearch 6, 7 and OpenSearch 2.18 are supported.
+Conductor is using Elasticsearch or OpenSearch for indexing the workflow data.
+Currently, Elasticsearch 7 and OpenSearch 2.x/3.x are supported.
 
 We welcome community contributions for other indexing backends.
 
-**Note:** Docker images use Elasticsearch 7.
+**Note:** Docker images use Elasticsearch 7 by default. Elasticsearch 6 and OpenSearch 1.x are deprecated.
 
 ## Helm Charts
 TODO: Link to the helm charts
@@ -49,7 +49,9 @@ TODO: Link to the helm charts
 | [docker-compose.yaml](docker-compose.yaml)                   | Redis + Elasticsearch 7    |
 | [docker-compose-postgres.yaml](docker-compose-postgres.yaml) | Postgres + Elasticsearch 7 |
 | [docker-compose-mysql.yaml](docker-compose-mysql.yaml)    | Mysql + Elasticsearch 7    |
-| [docker-compose-redis-os.yaml](docker-compose-redis-os.yaml)    | Redis + OpenSearch 2.18    |
+| [docker-compose-redis-os.yaml](docker-compose-redis-os.yaml) | Redis + OpenSearch 2.x (legacy - use os2) |
+| [docker-compose-redis-os2.yaml](docker-compose-redis-os2.yaml) | Redis + OpenSearch 2.x    |
+| [docker-compose-redis-os3.yaml](docker-compose-redis-os3.yaml) | Redis + OpenSearch 3.x    |
 
 ### Network errors during UI build with yarn
 

docker/docker-compose-redis-os2.yaml

@@ -0,0 +1,83 @@
+version: '2.3'
+
+services:
+  conductor-server:
+    environment:
+      - CONFIG_PROP=config-redis-os2.properties
+      - JAVA_OPTS=-Dpolyglot.engine.WarnInterpreterOnly=false
+    image: conductor:server
+    container_name: conductor-server-os2
+    build:
+      context: ../
+      dockerfile: docker/server/Dockerfile
+      args:
+        YARN_OPTS: ${YARN_OPTS}
+        INDEXING_BACKEND: opensearch
+    networks:
+      - internal
+    ports:
+      - 8080:8080
+      - 8127:5000
+    healthcheck:
+      test: ["CMD", "curl","-I" ,"-XGET", "http://localhost:8080/health"]
+      interval: 60s
+      timeout: 30s
+      retries: 12
+    links:
+      - conductor-opensearch:os
+      - conductor-redis:rs
+    depends_on:
+      conductor-opensearch:
+        condition: service_healthy
+      conductor-redis:
+        condition: service_healthy
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "1k"
+        max-file: "3"
+
+  conductor-redis:
+    image: redis:6.2.3-alpine
+    volumes:
+      - ../server/config/redis.conf:/usr/local/etc/redis/redis.conf
+    networks:
+      - internal
+    ports:
+      - 6379:6379
+    healthcheck:
+      test: [ "CMD", "redis-cli","ping" ]
+
+  conductor-opensearch:
+    image: opensearchproject/opensearch:2.18.0
+    container_name: opensearch-2
+    environment:
+      - plugins.security.disabled=true
+      - cluster.name=opensearch-cluster # Name the cluster
+      - node.name=conductor-opensearch # Name the node that will run in this container
+      - discovery.seed_hosts=conductor-opensearch # Nodes to look for when discovering the cluster
+      - cluster.initial_cluster_manager_nodes=conductor-opensearch # Nodes eligible to serve as cluster manager
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=P4zzW)rd>>123_
+    volumes:
+      - osdata2-conductor:/usr/share/opensearch/data
+    networks:
+      - internal
+    ports:
+      - 9201:9200
+    healthcheck:
+      test: curl http://localhost:9200/_cluster/health -o /dev/null
+      interval: 5s
+      timeout: 5s
+      retries: 12
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "1k"
+        max-file: "3"
+
+volumes:
+  osdata2-conductor:
+    driver: local
+
+networks:
+  internal:

docker/docker-compose-redis-os3.yaml

@@ -0,0 +1,83 @@
+version: '2.3'
+
+services:
+  conductor-server:
+    environment:
+      - CONFIG_PROP=config-redis-os3.properties
+      - JAVA_OPTS=-Dpolyglot.engine.WarnInterpreterOnly=false
+    image: conductor:server
+    container_name: conductor-server-os3
+    build:
+      context: ../
+      dockerfile: docker/server/Dockerfile
+      args:
+        YARN_OPTS: ${YARN_OPTS}
+        INDEXING_BACKEND: opensearch
+    networks:
+      - internal
+    ports:
+      - 8081:8080
+      - 8128:5000
+    healthcheck:
+      test: ["CMD", "curl","-I" ,"-XGET", "http://localhost:8080/health"]
+      interval: 60s
+      timeout: 30s
+      retries: 12
+    links:
+      - conductor-opensearch:os
+      - conductor-redis:rs
+    depends_on:
+      conductor-opensearch:
+        condition: service_healthy
+      conductor-redis:
+        condition: service_healthy
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "1k"
+        max-file: "3"
+
+  conductor-redis:
+    image: redis:6.2.3-alpine
+    volumes:
+      - ../server/config/redis.conf:/usr/local/etc/redis/redis.conf
+    networks:
+      - internal
+    ports:
+      - 6380:6379
+    healthcheck:
+      test: [ "CMD", "redis-cli","ping" ]
+
+  conductor-opensearch:
+    image: opensearchproject/opensearch:3.0.0
+    container_name: opensearch-3
+    environment:
+      - plugins.security.disabled=true
+      - cluster.name=opensearch-cluster # Name the cluster
+      - node.name=conductor-opensearch # Name the node that will run in this container
+      - discovery.seed_hosts=conductor-opensearch # Nodes to look for when discovering the cluster
+      - cluster.initial_cluster_manager_nodes=conductor-opensearch # Nodes eligible to serve as cluster manager
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=P4zzW)rd>>123_
+    volumes:
+      - osdata3-conductor:/usr/share/opensearch/data
+    networks:
+      - internal
+    ports:
+      - 9202:9200
+    healthcheck:
+      test: curl http://localhost:9200/_cluster/health -o /dev/null
+      interval: 5s
+      timeout: 5s
+      retries: 12
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "1k"
+        max-file: "3"
+
+volumes:
+  osdata3-conductor:
+    driver: local
+
+networks:
+  internal:

docker/server/Dockerfile

@@ -15,7 +15,7 @@ COPY . /conductor
 WORKDIR /conductor
 
 # Build the server on run
-RUN ./gradlew build -x test -x spotlessCheck -PindexingBackend=${INDEXING_BACKEND}
+RUN ./gradlew build -x test -x spotlessCheck -x shadowJar -x :conductor-os-persistence-v3:build -PindexingBackend=${INDEXING_BACKEND}
 WORKDIR /server/build/libs
 RUN ls -ltr
 

docker/server/config/config-redis-os.properties

@@ -19,15 +19,11 @@ conductor.app.systemTaskMaxPollCount=20
 
 # OpenSearch indexing is enabled.
 conductor.indexing.enabled=true
-conductor.indexing.type=opensearch
-
-# Disable ES7 auto-configuration (required when using OpenSearch)
-conductor.elasticsearch.version=0
+conductor.indexing.type=opensearch2
 
 conductor.opensearch.url=http://os:9200
 conductor.opensearch.indexPrefix=conductor
 conductor.opensearch.indexReplicasCount=0
-conductor.opensearch.version=2
 conductor.opensearch.clusterHealthColor=green
 
 # Additional modules for metrics collection exposed to Prometheus (optional)

docker/server/config/config-redis-os2.properties

@@ -0,0 +1,38 @@
+# Database persistence type.
+# Below are the properties for redis
+conductor.db.type=redis_standalone
+conductor.queue.type=redis_standalone
+
+conductor.redis.hosts=rs:6379:us-east-1c
+conductor.redis-lock.serverAddress=redis://rs:6379
+conductor.redis.taskDefCacheRefreshInterval=1
+conductor.redis.workflowNamespacePrefix=conductor
+conductor.redis.queueNamespacePrefix=conductor_queues
+
+conductor.workflow-execution-lock.type=redis
+conductor.app.workflowExecutionLockEnabled=true
+conductor.app.lockTimeToTry=500
+
+conductor.app.systemTaskWorkerThreadCount=20
+conductor.app.systemTaskMaxPollCount=20
+
+
+# OpenSearch 2.x indexing (NEW configuration from Phase 2)
+conductor.indexing.enabled=true
+conductor.indexing.type=opensearch2
+
+# Shared OpenSearch namespace (from PR #675 - Phase 1)
+conductor.opensearch.url=http://os:9200
+conductor.opensearch.indexPrefix=conductor
+conductor.opensearch.indexReplicasCount=0
+conductor.opensearch.clusterHealthColor=green
+
+# Additional modules for metrics collection exposed to Prometheus (optional)
+conductor.metrics-prometheus.enabled=true
+management.endpoints.web.exposure.include=health,prometheus
+
+# Redis health indicator
+management.health.redis.enabled=true
+
+# Load sample kitchen sink workflow
+loadSample=true

docker/server/config/config-redis-os3.properties

@@ -0,0 +1,38 @@
+# Database persistence type.
+# Below are the properties for redis
+conductor.db.type=redis_standalone
+conductor.queue.type=redis_standalone
+
+conductor.redis.hosts=rs:6379:us-east-1c
+conductor.redis-lock.serverAddress=redis://rs:6379
+conductor.redis.taskDefCacheRefreshInterval=1
+conductor.redis.workflowNamespacePrefix=conductor
+conductor.redis.queueNamespacePrefix=conductor_queues
+
+conductor.workflow-execution-lock.type=redis
+conductor.app.workflowExecutionLockEnabled=true
+conductor.app.lockTimeToTry=500
+
+conductor.app.systemTaskWorkerThreadCount=20
+conductor.app.systemTaskMaxPollCount=20
+
+
+# OpenSearch 3.x indexing (NEW configuration from Phase 2)
+conductor.indexing.enabled=true
+conductor.indexing.type=opensearch3
+
+# Shared OpenSearch namespace (from PR #675 - Phase 1)
+conductor.opensearch.url=http://os:9200
+conductor.opensearch.indexPrefix=conductor
+conductor.opensearch.indexReplicasCount=0
+conductor.opensearch.clusterHealthColor=green
+
+# Additional modules for metrics collection exposed to Prometheus (optional)
+conductor.metrics-prometheus.enabled=true
+management.endpoints.web.exposure.include=health,prometheus
+
+# Redis health indicator
+management.health.redis.enabled=true
+
+# Load sample kitchen sink workflow
+loadSample=true

es6-persistence/README.md

@@ -0,0 +1,47 @@
+# Elasticsearch 6.x Persistence - DEPRECATED
+
+⚠️ **This module is deprecated and provides only a migration error message.**
+
+## What Happened?
+
+Elasticsearch 6.x reached end-of-life in November 2020 and is no longer supported.
+
+The `conductor.indexing.type=elasticsearch_v6` configuration has been deprecated. Users should migrate to Elasticsearch 7.x.
+
+## Migration
+
+Change your configuration from:
+
+```properties
+conductor.indexing.type=elasticsearch_v6
+conductor.elasticsearch.url=http://localhost:9200
+```
+
+To:
+
+```properties
+conductor.indexing.type=elasticsearch
+conductor.elasticsearch.url=http://localhost:9200
+```
+
+All other `conductor.elasticsearch.*` properties remain the same.
+
+## Why the Change?
+
+- Elasticsearch 6.x reached end-of-life in November 2020
+- Security vulnerabilities are no longer patched
+- Elasticsearch 7.x provides better performance and features
+- Reduces maintenance burden of supporting legacy versions
+
+## Legacy Code Reference
+
+If you need the original Elasticsearch 6.x persistence module code for reference, it has been archived at:
+
+https://github.com/conductor-oss/conductor-es6-persistence
+
+**Note:** The archived module is no longer maintained and should not be used in production.
+
+## See Also
+
+- Legacy Elasticsearch 6.x Module: https://github.com/conductor-oss/conductor-es6-persistence
+- Elasticsearch 7.x Support: Use `conductor.indexing.type=elasticsearch` instead

es6-persistence/build.gradle

@@ -1,5 +1,5 @@
 /*
- *  Copyright 2023 Conductor authors
+ *  Copyright 2026 Conductor Authors
  *  <p>
  *  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
  *  the License. You may obtain a copy of the License at
@@ -11,32 +11,13 @@
  *  specific language governing permissions and limitations under the License.
  */
 
-
+// Deprecation stub for Elasticsearch 6.x support
+// This module only contains configuration to throw a helpful error message
+// when users try to use the deprecated elasticsearch_v6 indexing type.
 
 dependencies {
-    implementation project(':conductor-common')
-    implementation project(':conductor-core')
-
     compileOnly 'org.springframework.boot:spring-boot-starter'
-    compileOnly 'org.springframework.retry:spring-retry'
-
-    implementation "commons-io:commons-io:${revCommonsIo}"
-    implementation "org.apache.commons:commons-lang3"
-    // SBMTODO: remove guava dep
-    implementation "com.google.guava:guava:${revGuava}"
-
-    implementation "org.elasticsearch.client:transport:${revElasticSearch6}"
-    implementation "org.elasticsearch.client:elasticsearch-rest-client:${revElasticSearch6}"
-    implementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:${revElasticSearch6}"
-
-    testImplementation 'org.springframework.retry:spring-retry'
-    testImplementation "org.awaitility:awaitility:${revAwaitility}"
-    testImplementation "org.testcontainers:elasticsearch:${revTestContainer}"
-    testImplementation project(':conductor-common').sourceSets.test.output
-}
+    compileOnly 'jakarta.annotation:jakarta.annotation-api'
 
-switch (org.gradle.internal.os.OperatingSystem.current()) {
-    case org.gradle.internal.os.OperatingSystem.MAC_OS:
-        tasks.forEach(task -> task.onlyIf { project.hasProperty('ES6Test') })
-        break;
+    testImplementation 'org.springframework.boot:spring-boot-starter-test'
 }