Implement Kibana/OpenSearch Dashboards dev services

build-parent/pom.xml

@@ -81,6 +81,7 @@
         <elasticsearch.protocol>http</elasticsearch.protocol>
         <opensearch-server.version>3.1.0</opensearch-server.version>
         <opensearch.image>docker.io/opensearchproject/opensearch:${opensearch-server.version}</opensearch.image>
+        <opensearch-dashboards.image>docker.io/opensearchproject/opensearch-dashboards:${opensearch-server.version}</opensearch-dashboards.image>
         <opensearch.protocol>http</opensearch.protocol>
         <junit-pioneer.version>2.2.0</junit-pioneer.version>
 

extensions/elasticsearch-rest-client-common/deployment/src/main/java/io/quarkus/elasticsearch/restclient/common/deployment/DevServicesElasticsearchProcessor.java

@@ -2,6 +2,9 @@
 
 import static io.quarkus.devservices.common.ContainerLocator.locateContainerWithLabels;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
 import java.time.Duration;
 import java.util.Collections;
 import java.util.HashMap;
@@ -10,9 +13,12 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Properties;
 import java.util.Set;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
 
+import org.eclipse.microprofile.config.ConfigProvider;
 import org.jboss.logging.Logger;
 import org.opensearch.testcontainers.OpensearchContainer;
 import org.testcontainers.containers.GenericContainer;
@@ -59,15 +65,21 @@ public class DevServicesElasticsearchProcessor {
     static final String DEV_SERVICE_LABEL = "quarkus-dev-service-elasticsearch";
     static final String NEW_DEV_SERVICE_LABEL = "io.quarkus.devservice.elasticsearch";
     static final int ELASTICSEARCH_PORT = 9200;
+    static final int DASHBOARD_PORT = 5601;
 
     private static final ContainerLocator elasticsearchContainerLocator = locateContainerWithLabels(ELASTICSEARCH_PORT,
             DEV_SERVICE_LABEL, NEW_DEV_SERVICE_LABEL);
+    private static final ContainerLocator dashboardContainerLocator = locateContainerWithLabels(DASHBOARD_PORT,
+            DEV_SERVICE_LABEL, NEW_DEV_SERVICE_LABEL);
 
     private static final Distribution DEFAULT_DISTRIBUTION = Distribution.ELASTIC;
     private static final String DEV_SERVICE_ELASTICSEARCH = "elasticsearch";
     private static final String DEV_SERVICE_OPENSEARCH = "opensearch";
+    private static final String DEV_SERVICE_DASHBOARDS = "opensearch-dashboards";
+    private static final String DEV_SERVICE_KIBANA = "kibana";
 
     static volatile RunningDevService devService;
+    static volatile RunningDevService devDashboardService;
     static volatile ElasticsearchCommonBuildTimeConfig cfg;
     static volatile boolean first = true;
 
@@ -109,6 +121,11 @@ public DevServicesResultBuildItem startElasticsearchDevService(
                     devServicesSharedNetworkBuildItem);
             devService = startElasticsearchDevServices(dockerStatusBuildItem, composeProjectBuildItem,
                     configuration.devservices(), buildItemsConfig, launchMode, useSharedNetwork, devServicesConfig.timeout());
+
+            devDashboardService = startDashboardDevServices(dockerStatusBuildItem, composeProjectBuildItem,
+                    configuration.devservices(),
+                    buildItemsConfig, launchMode, useSharedNetwork, devServicesConfig.timeout());
+
             if (devService == null) {
                 compressor.closeAndDumpCaptured();
             } else {
@@ -130,6 +147,9 @@ public DevServicesResultBuildItem startElasticsearchDevService(
                 if (devService != null) {
                     shutdownElasticsearch();
                 }
+                if (devDashboardService != null) {
+                    shutdownDashboard();
+                }
                 first = true;
                 devService = null;
                 cfg = null;
@@ -165,6 +185,18 @@ private void shutdownElasticsearch() {
         }
     }
 
+    private void shutdownDashboard() {
+        if (devDashboardService != null) {
+            try {
+                devDashboardService.close();
+            } catch (Throwable e) {
+                log.error("Failed to stop the Dashboard", e);
+            } finally {
+                devService = null;
+            }
+        }
+    }
+
     private RunningDevService startElasticsearchDevServices(
             DockerStatusBuildItem dockerStatusBuildItem,
             DevServicesComposeProjectBuildItem composeProjectBuildItem,
@@ -220,11 +252,8 @@ private RunningDevService startElasticsearchDevServices(
                 container.setPortBindings(List.of(config.port().get() + ":" + ELASTICSEARCH_PORT));
             }
             timeout.ifPresent(container::withStartupTimeout);
-
             container.withEnv(config.containerEnv());
-
             container.withReuse(config.reuse());
-
             container.start();
 
             var httpHost = createdContainer.hostName + ":"
@@ -244,6 +273,103 @@ private RunningDevService startElasticsearchDevServices(
                 .orElseGet(defaultElasticsearchSupplier);
     }
 
+    private RunningDevService startDashboardDevServices(
+            DockerStatusBuildItem dockerStatusBuildItem,
+            DevServicesComposeProjectBuildItem composeProjectBuildItem,
+            ElasticsearchDevServicesBuildTimeConfig config,
+            DevservicesElasticsearchBuildItemsConfiguration buildItemConfig,
+            LaunchModeBuildItem launchMode, boolean useSharedNetwork, Optional<Duration> timeout) throws BuildException {
+        if (!config.enabled().orElse(true)) {
+            // explicitly disabled
+            log.debug("Not starting Dashboard DevServices for Elasticsearch, as it has been disabled in the config.");
+            return null;
+        }
+
+        if (!config.dashboard().enabled()) {
+            // Kibana explicitly disabled
+            log.debug("Not starting Kibana Dev Service, as it has been disabled in the config.");
+            return null;
+        }
+
+        if (!dockerStatusBuildItem.isContainerRuntimeAvailable()) {
+            log.warn("Docker is not working, cannot start the Kibana/OpenSearch dashboards dev service.");
+            return null;
+        }
+
+        Distribution resolvedDistribution = resolveDistribution(config, buildItemConfig);
+        DockerImageName resolvedImageName = resolveDashboardImageName(config, resolvedDistribution);
+
+        final Optional<ContainerAddress> maybeContainerAddressSearchBackend = elasticsearchContainerLocator.locateContainer(
+                config.serviceName(),
+                config.shared(),
+                launchMode.getLaunchMode())
+                .or(() -> ComposeLocator.locateContainer(composeProjectBuildItem,
+                        List.of(resolvedImageName.getUnversionedPart(), "elasticsearch", "opensearch"),
+                        ELASTICSEARCH_PORT,
+                        launchMode.getLaunchMode(), useSharedNetwork));
+
+        Set<String> opensearchHosts;
+        if (buildItemConfig.hostsConfigProperties.stream().anyMatch(ConfigUtils::isPropertyNonEmpty)) {
+            opensearchHosts = buildItemConfig.hostsConfigProperties.stream().filter(ConfigUtils::isPropertyNonEmpty)
+                    .flatMap(property -> ConfigProvider.getConfig().getValues(property, String.class).stream())
+                    .map(host -> "http://" + host.replace("localhost", "host.docker.internal"))
+                    .collect(Collectors.toSet());
+        } else {
+            opensearchHosts = maybeContainerAddressSearchBackend.map(containerAddress -> Set
+                    .of(("http://" + containerAddress.getHost() + ":" + containerAddress.getPort())
+                            .replace("localhost", "host.docker.internal")))
+                    .orElseGet(() -> Set.of());
+        }
+
+        final Optional<ContainerAddress> maybeContainerAddress = dashboardContainerLocator.locateContainer(
+                config.serviceName(),
+                config.shared(),
+                launchMode.getLaunchMode())
+                .or(() -> ComposeLocator.locateContainer(composeProjectBuildItem,
+                        List.of(resolvedImageName.getUnversionedPart(), "kibana", "opensearch-dashboards"),
+                        DASHBOARD_PORT,
+                        launchMode.getLaunchMode(), useSharedNetwork));
+
+        // Starting the server
+        final Supplier<RunningDevService> defaultDashboardsSupplier = () -> {
+
+            String defaultNetworkId = composeProjectBuildItem.getDefaultNetworkId();
+            CreatedContainer createdContainer = resolvedDistribution.equals(Distribution.ELASTIC)
+                    ? createKibanaContainer(config, resolvedImageName, defaultNetworkId, useSharedNetwork, launchMode,
+                            composeProjectBuildItem, opensearchHosts)
+                    : createDashboardsContainer(config, resolvedImageName, defaultNetworkId, useSharedNetwork, launchMode,
+                            composeProjectBuildItem, opensearchHosts);
+            GenericContainer<?> container = createdContainer.genericContainer();
+
+            if (config.serviceName() != null) {
+                container.withLabel(DEV_SERVICE_LABEL, config.serviceName());
+                container.withLabel(Labels.QUARKUS_DEV_SERVICE, config.serviceName());
+            }
+            if (config.dashboard().port().isPresent()) {
+                container.setPortBindings(List.of(config.dashboard().port().get() + ":" + DASHBOARD_PORT));
+            }
+            timeout.ifPresent(container::withStartupTimeout);
+            container.withEnv(config.dashboard().containerEnv());
+            container.withReuse(config.reuse());
+            container.start();
+
+            var httpHost = createdContainer.hostName + ":"
+                    + (useSharedNetwork ? DASHBOARD_PORT : container.getMappedPort(DASHBOARD_PORT));
+            return new RunningDevService(Feature.ELASTICSEARCH_REST_CLIENT_COMMON.getName(),
+                    container.getContainerId(),
+                    new ContainerShutdownCloseable(container, "Kibana"),
+                    buildPropertiesMap(buildItemConfig, httpHost));
+        };
+
+        return maybeContainerAddress
+                .map(containerAddress -> new RunningDevService(
+                        Feature.ELASTICSEARCH_REST_CLIENT_COMMON.getName(),
+                        containerAddress.getId(),
+                        null,
+                        buildPropertiesMap(buildItemConfig, containerAddress.getUrl())))
+                .orElseGet(defaultDashboardsSupplier);
+    }
+
     private CreatedContainer createElasticsearchContainer(ElasticsearchDevServicesBuildTimeConfig config,
             DockerImageName resolvedImageName, String defaultNetworkId, boolean useSharedNetwork) {
         ElasticsearchContainer container = new ElasticsearchContainer(
@@ -253,6 +379,9 @@ private CreatedContainer createElasticsearchContainer(ElasticsearchDevServicesBu
 
         // Disable security as else we would need to configure it correctly to avoid tons of WARNING in the log
         container.addEnv("xpack.security.enabled", "false");
+        // disable enrollment token to allow Kibana in a non-interactive automated way
+        container.addEnv("xpack.security.enrollment.enabled", "false");
+        container.addEnv("discovery.type", "single-node");
         // Disable disk-based shard allocation thresholds:
         // in a single-node setup they just don't make sense,
         // and lead to problems on large disks with little space left.
@@ -285,6 +414,46 @@ private CreatedContainer createOpensearchContainer(ElasticsearchDevServicesBuild
         return new CreatedContainer(container, hostName);
     }
 
+    private CreatedContainer createKibanaContainer(ElasticsearchDevServicesBuildTimeConfig config,
+            DockerImageName resolvedImageName, String defaultNetworkId, boolean useSharedNetwork,
+            LaunchModeBuildItem launchMode, DevServicesComposeProjectBuildItem composeProjectBuildItem,
+            Set<String> elasticsearchHosts) {
+        //Create Generic Kibana container
+        GenericContainer<?> container = new GenericContainer<>(
+                resolvedImageName.asCompatibleSubstituteFor("docker.elastic.co/kibana/kibana"));
+
+        String kibanaHostName = ConfigureUtil.configureNetwork(container, defaultNetworkId, useSharedNetwork,
+                DEV_SERVICE_KIBANA);
+        container.setExposedPorts(List.of(DASHBOARD_PORT));
+        if (!elasticsearchHosts.isEmpty()) {
+            container.addEnv("ELASTICSEARCH_HOSTS",
+                    "[" + elasticsearchHosts.stream().map(url -> "\"" + url + "\"").collect(Collectors.joining(",")) + "]");
+        }
+        config.dashboard().nodeOpts().ifPresent(nodeOpts -> container.addEnv("NODE_OPTIONS", nodeOpts));
+        return new CreatedContainer(container, kibanaHostName);
+    }
+
+    private CreatedContainer createDashboardsContainer(ElasticsearchDevServicesBuildTimeConfig config,
+            DockerImageName resolvedImageName, String defaultNetworkId, boolean useSharedNetwork,
+            LaunchModeBuildItem launchMode, DevServicesComposeProjectBuildItem composeProjectBuildItem,
+            Set<String> opensearchHosts) {
+        //Create Generic Kibana container
+        GenericContainer<?> container = new GenericContainer<>(
+                resolvedImageName.asCompatibleSubstituteFor("opensearchproject/opensearch-dashboards"));
+
+        String kibanaHostName = ConfigureUtil.configureNetwork(container, defaultNetworkId, useSharedNetwork,
+                DEV_SERVICE_DASHBOARDS);
+        container.setExposedPorts(List.of(DASHBOARD_PORT));
+        if (!opensearchHosts.isEmpty()) {
+            container.addEnv("OPENSEARCH_HOSTS",
+                    "[" + opensearchHosts.stream().map(url -> "\"" + url + "\"").collect(Collectors.joining(",")) + "]");
+        }
+
+        config.dashboard().nodeOpts().ifPresent(nodeOpts -> container.addEnv("NODE_OPTIONS", nodeOpts));
+        container.addEnv("DISABLE_SECURITY_DASHBOARDS_PLUGIN", "true");
+        return new CreatedContainer(container, kibanaHostName);
+    }
+
     private record CreatedContainer(GenericContainer<?> genericContainer, String hostName) {
     }
 
@@ -296,6 +465,29 @@ private DockerImageName resolveImageName(ElasticsearchDevServicesBuildTimeConfig
                         : DEV_SERVICE_OPENSEARCH)));
     }
 
+    private DockerImageName resolveDashboardImageName(ElasticsearchDevServicesBuildTimeConfig config,
+            Distribution resolvedDistribution) {
+        return DockerImageName.parse(config.dashboard().imageName().orElseGet(() -> loadProperties(
+                Distribution.ELASTIC.equals(resolvedDistribution)
+                        ? DEV_SERVICE_ELASTICSEARCH
+                        : DEV_SERVICE_OPENSEARCH)
+                .getProperty("default.dashboard.image")));
+    }
+
+    private static Properties loadProperties(String devserviceName) {
+        var fileName = devserviceName + "-devservice.properties";
+        try (InputStream in = Thread.currentThread().getContextClassLoader().getResourceAsStream(fileName)) {
+            if (in == null) {
+                throw new IllegalArgumentException(fileName + " not found on classpath");
+            }
+            var properties = new Properties();
+            properties.load(in);
+            return properties;
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
     private Distribution resolveDistribution(ElasticsearchDevServicesBuildTimeConfig config,
             DevservicesElasticsearchBuildItemsConfiguration buildItemConfig) throws BuildException {
         // First, let's see if it was explicitly configured:

extensions/elasticsearch-rest-client-common/deployment/src/main/java/io/quarkus/elasticsearch/restclient/common/deployment/ElasticsearchCommonBuildTimeConfig.java

@@ -133,9 +133,61 @@ interface ElasticsearchDevServicesBuildTimeConfig {
         @WithDefault("true")
         boolean reuse();
 
+        /**
+         * Kibana configuration for Dev Services.
+         */
+        @ConfigDocSection
+        DashboardDevServicesConfig dashboard();
+
         enum Distribution {
             ELASTIC,
             OPENSEARCH
         }
+
+        @ConfigGroup
+        interface DashboardDevServicesConfig {
+            /**
+             * Whether the Dashboard Dev Service should start with the application in dev mode or tests.
+             * <p>
+             * Kibana Dev Services are disabled by default when Elasticsearch Dev Services are enabled.
+             *
+             * @asciidoclet
+             */
+            @WithDefault("false")
+            boolean enabled();
+
+            /**
+             * Optional fixed port the dev service will listen to.
+             * <p>
+             * If not defined, the port will be chosen randomly.
+             */
+            Optional<Integer> port();
+
+            /**
+             * The Elasticsearch container image to use.
+             *
+             * Defaults depend on the configured `distribution`:
+             *
+             * * For the `elastic` distribution: {kibana-image}
+             * * For the `opensearch` distribution: {dashboardimage}
+             *
+             * @asciidoclet
+             */
+            Optional<String> imageName();
+
+            /**
+             * Environment variables that are passed to the container.
+             */
+            @ConfigDocMapKey("environment-variable-name")
+            Map<String, String> containerEnv();
+
+            /**
+             * The value for the NODE_OPTIONS env variable.
+             *
+             * @asciidoclet
+             */
+
+            Optional<String> nodeOpts();
+        }
     }
 }

extensions/elasticsearch-rest-client-common/deployment/src/main/resources/elasticsearch-devservice.properties

@@ -1 +1,2 @@
 default.image=${elasticsearch.image}
+default.dashboard.image=${kibana.image}

extensions/elasticsearch-rest-client-common/deployment/src/main/resources/opensearch-devservice.properties

@@ -1 +1,2 @@
 default.image=${opensearch.image}
+default.dashboard.image=${opensearch-dashboards.image}