Introduce Docker images build (#36246)

This commit introduces the building of the Docker images as bonafide
packaging formats alongside our existing archive and packaging
distributions. This build is migrated from a dedicated repository, and
converted to Gradle in the process.

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -51,6 +51,7 @@ import org.gradle.api.tasks.compile.JavaCompile
 import org.gradle.api.tasks.javadoc.Javadoc
 import org.gradle.internal.jvm.Jvm
 import org.gradle.process.ExecResult
+import org.gradle.process.ExecSpec
 import org.gradle.util.GradleVersion
 
 import java.nio.charset.StandardCharsets
@@ -232,6 +233,95 @@ class BuildPlugin implements Plugin<Project> {
         project.ext.java9Home = project.rootProject.ext.java9Home
     }
 
+    static void requireDocker(final Task task) {
+        final Project rootProject = task.project.rootProject
+        if (rootProject.hasProperty('requiresDocker') == false) {
+            /*
+             * This is our first time encountering a task that requires Docker. We will add an extension that will let us track the tasks
+             * that register as requiring Docker. We will add a delayed execution that when the task graph is ready if any such tasks are
+             * in the task graph, then we check two things:
+             *  - the Docker binary is available
+             *  - we can execute a Docker command that requires privileges
+             *
+             *  If either of these fail, we fail the build.
+             */
+            final boolean buildDocker
+            final String buildDockerProperty = System.getProperty("build.docker")
+            if (buildDockerProperty == null || buildDockerProperty == "true") {
+                buildDocker = true
+            } else if (buildDockerProperty == "false") {
+                buildDocker = false
+            } else {
+                throw new IllegalArgumentException(
+                        "expected build.docker to be unset or one of \"true\" or \"false\" but was [" + buildDockerProperty + "]")
+            }
+            rootProject.rootProject.ext.buildDocker = buildDocker
+            rootProject.rootProject.ext.requiresDocker = []
+            rootProject.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph ->
+                // check if the Docker binary exists and record its path
+                final List<String> maybeDockerBinaries = ['/usr/bin/docker2', '/usr/local/bin/docker2']
+                final String dockerBinary = maybeDockerBinaries.find { it -> new File(it).exists() }
+
+                int exitCode
+                String dockerErrorOutput
+                if (dockerBinary == null) {
+                    exitCode = -1
+                    dockerErrorOutput = null
+                } else {
+                    // the Docker binary executes, check that we can execute a privileged command
+                    final ByteArrayOutputStream output = new ByteArrayOutputStream()
+                    final ExecResult result = LoggedExec.exec(rootProject, { ExecSpec it ->
+                        it.commandLine dockerBinary, "images"
+                        it.errorOutput = output
+                        it.ignoreExitValue = true
+                    })
+                    if (result.exitValue == 0) {
+                        return
+                    }
+                    exitCode = result.exitValue
+                    dockerErrorOutput = output.toString()
+                }
+                final List<String> tasks =
+                        ((List<Task>)rootProject.requiresDocker).findAll { taskGraph.hasTask(it) }.collect { "  ${it.path}".toString()}
+                if (tasks.isEmpty() == false) {
+                    /*
+                     * There are tasks in the task graph that require Docker. Now we are failing because either the Docker binary does not
+                     * exist or because execution of a privileged Docker command failed.
+                     */
+                    String message
+                    if (dockerBinary == null) {
+                        message = String.format(
+                                Locale.ROOT,
+                                "Docker (checked [%s]) is required to run the following task%s: \n%s",
+                                maybeDockerBinaries.join(","),
+                                tasks.size() > 1 ? "s" : "",
+                                tasks.join('\n'))
+                    } else {
+                        assert exitCode > 0 && dockerErrorOutput != null
+                        message = String.format(
+                                Locale.ROOT,
+                                "a problem occurred running Docker from [%s] yet it is required to run the following task%s: \n%s\n" +
+                                        "the problem is that Docker exited with exit code [%d] with standard error output [%s]",
+                                dockerBinary,
+                                tasks.size() > 1 ? "s" : "",
+                                tasks.join('\n'),
+                                exitCode,
+                                dockerErrorOutput.trim())
+                    }
+                    throw new GradleException(
+                            message + "\nyou can address this by attending to the reported issue, "
+                                    + "removing the offending tasks from being executed, "
+                                    + "or by passing -Dbuild.docker=false")
+                }
+            }
+        }
+        if (rootProject.buildDocker) {
+            rootProject.requiresDocker.add(task)
+        } else {
+            task.enabled = false
+        }
+    }
+
     private static String findCompilerJavaHome() {
         String compilerJavaHome = System.getenv('JAVA_HOME')
         final String compilerJavaProperty = System.getProperty('compiler.java')

distribution/docker/build.gradle

@@ -0,0 +1,106 @@
+import org.elasticsearch.gradle.BuildPlugin
+import org.elasticsearch.gradle.LoggedExec
+import org.elasticsearch.gradle.MavenFilteringHack
+import org.elasticsearch.gradle.VersionProperties
+
+apply plugin: 'base'
+
+configurations {
+  dockerPlugins
+  dockerSource
+  ossDockerSource
+}
+
+dependencies {
+  dockerPlugins project(path: ":plugins:ingest-geoip", configuration: 'zip')
+  dockerPlugins project(path: ":plugins:ingest-user-agent", configuration: 'zip')
+  dockerSource project(path: ":distribution:archives:tar")
+  ossDockerSource project(path: ":distribution:archives:oss-tar")
+}
+
+ext.expansions = { oss ->
+  return [
+    'elasticsearch' : oss ? "elasticsearch-oss-${VersionProperties.elasticsearch}.tar.gz" : "elasticsearch-${VersionProperties.elasticsearch}.tar.gz",
+    'jdkUrl' : 'https://download.java.net/java/GA/jdk11/13/GPL/openjdk-11.0.1_linux-x64_bin.tar.gz',
+    'jdkVersion' : '11.0.1',
+    'license': oss ? 'Apache-2.0' : 'Elastic License',
+    'ingest-geoip' : "ingest-geoip-${VersionProperties.elasticsearch}.zip",
+    'ingest-user-agent' : "ingest-user-agent-${VersionProperties.elasticsearch}.zip",
+    'version' : VersionProperties.elasticsearch
+  ]
+}
+
+private static String files(final boolean oss) {
+  return "build/${ oss ? 'oss-' : ''}docker"
+}
+
+private static String taskName(final String prefix, final boolean oss, final String suffix) {
+  return "${prefix}${oss ? 'Oss' : ''}${suffix}"
+}
+
+void addCopyDockerContextTask(final boolean oss) {
+  task(taskName("copy", oss, "DockerContext"), type: Sync) {
+    into files(oss)
+
+    into('bin') {
+      from 'src/docker/bin'
+    }
+
+    into('config') {
+      from 'src/docker/config'
+    }
+
+    if (oss) {
+      from configurations.ossDockerSource
+    } else {
+      from configurations.dockerSource
+    }
+
+    from configurations.dockerPlugins
+  }
+}
+
+void addCopyDockerfileTask(final boolean oss) {
+  task(taskName("copy", oss, "Dockerfile"), type: Copy) {
+    mustRunAfter(taskName("copy", oss, "DockerContext"))
+    into files(oss)
+
+    from('src/docker/Dockerfile') {
+      MavenFilteringHack.filter(it, expansions(oss))
+    }
+  }
+}
+
+void addBuildDockerImage(final boolean oss) {
+  final Task buildDockerImageTask = task(taskName("build", oss, "DockerImage"), type: LoggedExec) {
+    dependsOn taskName("copy", oss, "DockerContext")
+    dependsOn taskName("copy", oss, "Dockerfile")
+    List<String> tags
+    if (oss) {
+      tags = [ "docker.elastic.co/elasticsearch/elasticsearch-oss:${VersionProperties.elasticsearch}" ]
+    } else {
+      tags = [
+        "elasticsearch:${VersionProperties.elasticsearch}",
+        "docker.elastic.co/elasticsearch/elasticsearch:${VersionProperties.elasticsearch}",
+        "docker.elastic.co/elasticsearch/elasticsearch-full:${VersionProperties.elasticsearch}"
+      ]
+    }
+    executable 'docker'
+    final List<String> dockerArgs = ['build', files(oss), '--pull']
+    for (final String tag : tags) {
+      dockerArgs.add('--tag')
+      dockerArgs.add(tag)
+    }
+    args dockerArgs.toArray()
+  }
+  BuildPlugin.requireDocker(buildDockerImageTask)
+}
+
+for (final boolean oss : [false, true]) {
+  addCopyDockerContextTask(oss)
+  addCopyDockerfileTask(oss)
+  addBuildDockerImage(oss)
+}
+
+assemble.dependsOn "buildOssDockerImage"
+assemble.dependsOn "buildDockerImage"

distribution/docker/src/docker/Dockerfile

@@ -0,0 +1,92 @@
+################################################################################
+# This Dockerfile was generated from the template at distribution/src/docker/Dockerfile
+#
+# Beginning of multi stage Dockerfile
+################################################################################
+
+################################################################################
+# Build stage 0 `builder`:
+# Extract elasticsearch artifact
+# Install required plugins
+# Set gid=0 and make group perms==owner perms
+################################################################################
+
+FROM centos:7 AS builder
+
+ENV PATH /usr/share/elasticsearch/bin:$PATH
+ENV JAVA_HOME /opt/jdk-${jdkVersion}
+
+RUN curl -s ${jdkUrl} | tar -C /opt -zxf -
+
+# Replace OpenJDK's built-in CA certificate keystore with the one from the OS
+# vendor. The latter is superior in several ways.
+# REF: https://github.com/elastic/elasticsearch-docker/issues/171
+RUN ln -sf /etc/pki/ca-trust/extracted/java/cacerts /opt/jdk-${jdkVersion}/lib/security/cacerts
+
+RUN yum install -y unzip which
+
+RUN groupadd -g 1000 elasticsearch && \
+    adduser -u 1000 -g 1000 -d /usr/share/elasticsearch elasticsearch
+
+WORKDIR /usr/share/elasticsearch
+
+COPY ${elasticsearch} ${ingest-geoip} ${ingest-user-agent} /opt/
+RUN tar zxf /opt/${elasticsearch} --strip-components=1
+RUN elasticsearch-plugin install --batch file:///opt/${ingest-geoip}
+RUN elasticsearch-plugin install --batch file:///opt/${ingest-user-agent}
+RUN mkdir -p config data logs
+RUN chmod 0775 config data logs
+COPY config/elasticsearch.yml config/log4j2.properties config/
+
+
+################################################################################
+# Build stage 1 (the actual elasticsearch image):
+# Copy elasticsearch from stage 0
+# Add entrypoint
+################################################################################
+
+FROM centos:7
+
+ENV ELASTIC_CONTAINER true
+ENV JAVA_HOME /opt/jdk-${jdkVersion}
+
+COPY --from=builder /opt/jdk-${jdkVersion} /opt/jdk-${jdkVersion}
+
+RUN yum update -y && \
+    yum install -y nc unzip wget which && \
+    yum clean all
+
+RUN groupadd -g 1000 elasticsearch && \
+    adduser -u 1000 -g 1000 -G 0 -d /usr/share/elasticsearch elasticsearch && \
+    chmod 0775 /usr/share/elasticsearch && \
+    chgrp 0 /usr/share/elasticsearch
+
+WORKDIR /usr/share/elasticsearch
+COPY --from=builder --chown=1000:0 /usr/share/elasticsearch /usr/share/elasticsearch
+ENV PATH /usr/share/elasticsearch/bin:$PATH
+
+COPY --chown=1000:0 bin/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+
+# Openshift overrides USER and uses ones with randomly uid>1024 and gid=0
+# Allow ENTRYPOINT (and ES) to run even with a different user
+RUN chgrp 0 /usr/local/bin/docker-entrypoint.sh && \
+    chmod g=u /etc/passwd && \
+    chmod 0775 /usr/local/bin/docker-entrypoint.sh
+
+EXPOSE 9200 9300
+
+LABEL org.label-schema.schema-version="1.0" \
+  org.label-schema.vendor="Elastic" \
+  org.label-schema.name="elasticsearch" \
+  org.label-schema.version="${version}" \
+  org.label-schema.url="https://www.elastic.co/products/elasticsearch" \
+  org.label-schema.vcs-url="https://github.com/elastic/elasticsearch" \
+  license="${license}"
+
+ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
+# Dummy overridable parameter parsed by entrypoint
+CMD ["eswrapper"]
+
+################################################################################
+# End of multi-stage Dockerfile
+################################################################################