build.gradle

import org.apache.tools.ant.util.LazyFileOutputStream
import org.apache.tools.ant.util.OutputStreamFunneler

import java.nio.file.Files
import java.nio.file.Paths
import java.util.regex.Pattern

import static java.nio.file.StandardCopyOption.REPLACE_EXISTING

buildscript {
    repositories {
        mavenCentral()
        maven {
            url "https://plugins.gradle.org/m2/"
        }
    }
}

plugins {
    id "java"
    id "idea"
    id "com.adarshr.test-logger" version "3.2.0"
    id "de.undercouch.download" version "5.3.1"
    id "com.github.johnrengelman.shadow" version "8.1.1"
}

// ===========================================================================
// plugin info
// ===========================================================================
group                      'co.elastic.logstash.plugins.filter.elasticintegration' // must match the package of the main plugin class
version                    "${file("VERSION").text.trim()}" // read from required VERSION file
description                = "Elastic Integration filter"
// ===========================================================================

java {
    sourceCompatibility = 17
    targetCompatibility = 17
}
tasks.withType(JavaCompile) {
    options.encoding = 'UTF-8'
}

repositories {
    mavenCentral()
}

configurations {
    logstashCore { canBeResolved = false; canBeConsumed = false }
    elasticsearchMinimalCore { canBeConsumed = false }
    elasticsearchClient { canBeConsumed = false }

    geolite2 { canBeConsumed = false }

    implementation.extendsFrom(logstashCore, elasticsearchMinimalCore, elasticsearchClient)
}

wrapper {
    gradleVersion = '8.7'
    distributionUrl = distributionUrl.replace("bin", "all")
}

shadowJar {
    configurations = [project.configurations.elasticsearchMinimalCore,
                      project.configurations.elasticsearchClient]

    zip64 true
}

dependencies {
    elasticsearchClient('co.elastic.clients:elasticsearch-java') {
        // brings latest version available for 8.x branch
        version {
            strictly('[8.0, 9.0[')
        }
    }
    elasticsearchMinimalCore fileTree(dir: { importMinimalElasticsearch.jars }, include: ["*.jar"])

    // Logstash core and its known-provided and required dependencies
    // must NEVER be included in the gem-vendored jars
    logstashCore files(objects.fileCollection(), {
        from requiredLogstashCoreJar("logstash-core")
        from requiredLogstashCoreJar("log4j-api")
        from requiredLogstashVendorJar("jruby/lib", "jruby")
    })

    // Logstash core-provided dependencies that are also used directly
    // by this plugin's implementation and SHOULD be gem-vendored
    // once they can be safely shaded
    logstashCore files(objects.fileCollection(), {
        from requiredLogstashCoreJar("jackson-core")
        from requiredLogstashCoreJar("jackson-databind")
        from requiredLogstashCoreJar("jackson-annotations")
        from requiredLogstashCoreJar("guava", "jre")
    })

    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.1.0'
    testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.1.0'
    testImplementation 'org.mockito:mockito-junit-jupiter:5.1.1'
    testImplementation 'org.hamcrest:hamcrest-library:2.2'
    testImplementation "com.github.seregamorph:hamcrest-more-matchers:0.1"
    testImplementation "com.github.tomakehurst:wiremock-jre8:2.35.0"
    testRuntimeOnly requiredLogstashCoreJar("log4j-core")

    geolite2('org.elasticsearch:geolite2-databases:20191119') {
        transitive = false
    }
}

/**
 * @param packageNameSpec e.g., "package-name" or "nesting/path/package-name"
 * @return a {@code Closure<Boolean>} determines if the provided {@code FileTreeElement}
 *         is a package matching the {@code packageNameSpec} or a nesting that _could_ include it.
 */
def jarPackageNamed(String packageNameSpec, String packageFlavorSpec = null) {
    def packageNamePath = Paths.get(packageNameSpec)
    def packageName = packageNamePath.fileName.toString()
    def packageNesting = packageNamePath.parent
    def packagePattern = namedPackageWithAnyVersionPattern(packageName, packageFlavorSpec, "jar")

    return { FileTreeElement element ->
        if (packageNesting != null && packageNesting.startsWith(element.relativePath.toString())) { return true }
        if (packageNesting == null && element.relativePath.toString() != element.name) { return false }

        return element.name.matches(packagePattern)
    }
}

def namedPackageWithAnyVersionPattern(String packageName, String packageFlavorSpec = null, String suffix = null) {
    def escapedPackagePart = Pattern.quote(packageName)
    def optionalAnyVersionPart = "(?:-\\d+(?:\\.\\d+)*(?i:-SNAPSHOT(?:-[0-9A-F]+)?)?)?"
    def escapedFlavorPart = packageFlavorSpec == null ? "" : "(?:-${Pattern.quote(packageFlavorSpec)})"
    def escapedSuffixPart = suffix == null ? "" : Pattern.quote(".${suffix}")
    return Pattern.compile("${escapedPackagePart}${optionalAnyVersionPart}${escapedFlavorPart}${escapedSuffixPart}");
}

def envOrPropertyValue(String name) {
    return System.getenv(name) ? System.getenv(name) : project.findProperty(name)
}

def requiredLogstashCoreJar(jarSpec, flavorSpec = null) {
    _requiredLogstashJar("logstash-core/lib/jars", jarSpec, flavorSpec)
}

def requiredLogstashVendorJar(path, jarSpec) {
    _requiredLogstashJar("vendor/${path}", jarSpec)
}

def _requiredLogstashJar(pathPrefix, jarSpec, flavorSpec = null) {
    def logstashPath = envOrPropertyValue("LOGSTASH_PATH")
    assert logstashPath != null : "The property LOGSTASH_PATH must be defined, and be path to logstash directory"
    file("${logstashPath}").with { logstashDir ->
        assert logstashDir.exists() : "LOGSTASH_PATH does not exist: ${logstashDir}"
        assert logstashDir.directory : "LOGSTASH_PATH is not a direcrtory: ${logstashDir}"
        file("${logstashDir.path}/${pathPrefix}/").with { jarSourceDir ->
            fileTree(jarSourceDir) { include jarPackageNamed(jarSpec, flavorSpec) }.tap { jarSource ->
                assert !jarSource.empty : "LOGSTASH_PATH's ${pathPrefix} does not provide any ${jarSpec} jars: ${jarSourceDir}"
                assert jarSource.files.size() == 1 : "LOGSTASH_PATH's ${pathPrefix} provides multiple ${jarSpec} jars: ${jarSource.files}"
            }
        }
    }
}

// https://docs.github.com/en/repositories/working-with-files/using-files/downloading-source-code-archives#source-code-archive-urls
String githubArchivePath(org, repo, treeish="main", archiveFormat="zip") {
    def pathFragment = {
        switch(treeish) {
            case ~'^refs/(?:heads|tags)/'       : return "${treeish}" // unambiguous ref
            case ~'^[0-9a-f]{8,40}$'            : return "${treeish}" // possibly-abbreviated commit sha
            case ~'^v[0-9]+[.][0-9]+[.][0-9]+$' : return "refs/tags/${treeish}"  // guess: version tag
            default                             : return "refs/heads/${treeish}" // guess: branch name
        }
    }()

    "https://github.com/${org}/${repo}/archive/${pathFragment}.${archiveFormat}"
}

task downloadElasticsearchSourceZip(type: Download) {
    def esTreeish = envOrPropertyValue("ELASTICSEARCH_TREEISH")
    inputs.property("ELASTICSEARCH_TREEISH", esTreeish)

    if (!esTreeish) {
        throw new StopActionException("""
            The property ELASTICSEARCH_TREEISH must be defined, and be a tree-ish (branch, commit hash, or tag)
            available on the github repository for elastic/elasticsearch (e.g., `v8.7.0`, `8.7`, `main`,
            `ff1caebad1dea`, `refs/heads/branch-name`, `refs/tags/tag-name`)
        """)
    }

    src githubArchivePath("elastic", "elasticsearch", esTreeish, "zip")
    dest "${buildDir}/elasticsearch-source.zip"
    onlyIfModified true
    useETag "all"
}

task unzipDownloadedElasticsearchSourceZip(dependsOn: downloadElasticsearchSourceZip, type: Copy) {
    description "extracts Elasticsearch source from a downloaded zip file"

    from zipTree(downloadElasticsearchSourceZip.dest)
    into "${buildDir}/elasticsearch-source/"
    eachFile {
        // strip top-level directory
        path = path.replaceFirst(/^.+?\//, "")
    }
}

task buildElasticsearchLocalDistro(dependsOn: unzipDownloadedElasticsearchSourceZip, type: Exec) {
    description "builds Elasticsearch localDistro"

    def logFile = project.file("${buildDir}/elasticsearch-build.log")
    doFirst {
        def funneler = new OutputStreamFunneler(new LazyFileOutputStream(logFile))
        standardOutput = funneler.funnelInstance
        errorOutput = funneler.funnelInstance
    }

    def esSource = "${buildDir}/elasticsearch-source/"
    def esBuildDir = "${esSource}/build"

    inputs.dir esSource
    outputs.dir esBuildDir

    ext.buildRoot = esBuildDir
    ext.localDistroParent = "${esBuildDir}/distribution/local"
    ext.localDistroResult = provider({
        project.fileTree(dir: ext.localDistroParent, include: "*/lib/elasticsearch-core-*.jar").with {
            it && !it.isEmpty() ? it.singleFile.parentFile.parentFile : null
        }
    })
    ext.lib = localDistroResult.map { "${it}/lib" }
    ext.module = { moduleName -> localDistroResult.map { "${it}/modules/${moduleName}"} }

    workingDir esSource
    commandLine "./gradlew", "localDistro"

    ignoreExitValue true // handled in doLast
    doLast {
        def exitValue = executionResult.get().exitValue
        assert exitValue == 0 : "Elasticsearch build failed; logs available in ${logFile.path}"
        assert ext.localDistroResult.isPresent() : "Elasticsearch did not produce exactly one localdistro build"
    }
}

task shadeElasticsearchIngestGeoIpModule(type: com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
    description "Shades embedded dependencies of the Elasticsearch Ingest GeoIP module"

    dependsOn buildElasticsearchLocalDistro

    from(buildElasticsearchLocalDistro.module("ingest-geoip").orElse(objects.fileCollection()))

    archiveFileName = 'ingest-geoip-shaded.jar'
    destinationDirectory = file("${buildDir}/shaded")

    mergeServiceFiles()

    String shadeNamespace = "org.elasticsearch.ingest.geoip.shaded"
    relocate('com.fasterxml.jackson', "${shadeNamespace}.com.fasterxml.jackson")
    relocate('com.maxmind',           "${shadeNamespace}.com.maxmind")

    exclude '**/module-info.class'
}

task shadeElasticsearchGrokImplementation(type: com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
    description "Shades embedded dependencies of the Elasticsearch Grok implementation"

    dependsOn buildElasticsearchLocalDistro

    from(buildElasticsearchLocalDistro.lib.orElse(objects.fileCollection())) {
        include jarPackageNamed("elasticsearch-grok")
        include jarPackageNamed("joni")
        include jarPackageNamed("jcodings")
    }

    archiveFileName = "elasticsearch-grok-uber.jar"
    destinationDirectory = file("${buildDir}/shaded")

    mergeServiceFiles()
    String shadeNamespace = "org.elasticsearch.grok.shaded"
    relocate('org.joni', "${shadeNamespace}.org.joni")
    relocate('org.jcodings', "${shadeNamespace}.org.jcodings")

    exclude '**/module-info.class'
}

task shadeElasticsearchRedactPlugin(type: com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
    description "Shades Elasticsearch Redact plugin to reference Grok's shaded dependencies"
    dependsOn buildElasticsearchLocalDistro

    from(buildElasticsearchLocalDistro.module("x-pack-redact").orElse(objects.fileCollection())) {
        include jarPackageNamed("x-pack-redact")
    }
    archiveFileName = "elasticsearch-x-pack-redact-shaded.jar"
    destinationDirectory = file("${buildDir}/shaded")

    // relocate elasticsearch-grok's dependencies to match
    String shadeNamespace = "org.elasticsearch.grok.shaded"
    relocate('org.joni', "${shadeNamespace}.org.joni")
    relocate('org.jcodings', "${shadeNamespace}.org.jcodings")

    exclude '**/module-info.class'
}

task importMinimalElasticsearch() {
    description "Imports minimal portions of Elasticsearch localDistro"

    dependsOn buildElasticsearchLocalDistro
    dependsOn shadeElasticsearchIngestGeoIpModule
    dependsOn shadeElasticsearchGrokImplementation
    dependsOn shadeElasticsearchRedactPlugin

    ext.jars = "${buildDir}/elasticsearch-minimal-jars"

    inputs.dir buildElasticsearchLocalDistro.buildRoot
    outputs.dir ext.jars

    doLast {
        delete(project.file(ext.jars))

        copy {
            from(buildElasticsearchLocalDistro.lib) {
                include jarPackageNamed("elasticsearch")
                include jarPackageNamed("elasticsearch-core")
                include jarPackageNamed("elasticsearch-x-content")
                include jarPackageNamed("elasticsearch-geo")
                include jarPackageNamed("lucene-core")
                include jarPackageNamed("lucene-analysis-common")
            }
            from(shadeElasticsearchGrokImplementation)
            from(buildElasticsearchLocalDistro.module("x-pack-core"))

            from(buildElasticsearchLocalDistro.module("ingest-common")) {
                include jarPackageNamed("ingest-common")
                include jarPackageNamed("elasticsearch-dissect")
            }

            from(buildElasticsearchLocalDistro.module("ingest-user-agent")) {
                include jarPackageNamed("ingest-user-agent")
            }

            from(shadeElasticsearchIngestGeoIpModule)
            from(shadeElasticsearchRedactPlugin)

            from(buildElasticsearchLocalDistro.module("lang-mustache")) {
                include jarPackageNamed("lang-mustache")
                include jarPackageNamed("compiler")
            }

            from(buildElasticsearchLocalDistro.module("lang-painless")) {
                include jarPackageNamed("lang-painless")
                include jarPackageNamed("antlr4-runtime")
                include jarPackageNamed("asm-util")
                include jarPackageNamed("asm-commons")
                include jarPackageNamed("asm-tree")
                include jarPackageNamed("asm")
                include jarPackageNamed("spi/elasticsearch-scripting-painless-spi")
            }

            into ext.jars

            includeEmptyDirs(false)
            eachFile { path = name } // flatten
            eachFile { logger.info "copying ${it} to ${ext.jars}" }
        }
    }
}
compileJava.dependsOn(importMinimalElasticsearch)
shadowJar.dependsOn(importMinimalElasticsearch)

tasks.withType(JavaCompile) {
    options.encoding = 'UTF-8'
}

test {
    useJUnitPlatform()
    testLogging {
        exceptionFormat "full"
    }
}

task generateTestCertificates(type: Exec) {
    def sslTestCertsDir = "${projectDir}/src/test/resources/co/elastic/logstash/filters/elasticintegration/ssl-test-certs"

    workingDir sslTestCertsDir
    commandLine './generate.sh'

    def commonOutputFile = Paths.get("${buildDir}/generateTestCertificates.log")
    file(commonOutputFile.parent).mkdirs()
    def commonOutputStream = Files.newOutputStream(commonOutputFile)

    standardOutput = commonOutputStream
    errorOutput = commonOutputStream

    inputs.file("${sslTestCertsDir}/generate.sh")
    inputs.file("${sslTestCertsDir}/openssl.cnf")

    outputs.dir("${sslTestCertsDir}/generated")
    outputs.file(commonOutputFile)
}
processTestResources.dependsOn(generateTestCertificates)

task geoipTestResources {
    def output = "${project.rootDir}/src/test/resources/co/elastic/logstash/filters/elasticintegration/geoip/databases"

    outputs.dir output
    inputs.files configurations.geolite2

    doLast {
        copy {
            from(zipTree(configurations.geolite2.singleFile)) {
                include "*.mmdb"
            }
            into(output)
        }
    }
}
processTestResources.dependsOn(geoipTestResources)

tasks.withType(Test) {
    // Add Exports to enable tests to run in JDK17
    jvmArgs = [
            "--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED",
            "--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED",
            "--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED",
            "--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED",
            "--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED",
            "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED",
            "--add-opens=java.base/java.io=ALL-UNNAMED",
            "--add-opens=java.base/java.lang=ALL-UNNAMED",
            "--add-opens=java.base/java.util=ALL-UNNAMED"
    ]
}

task vendorShadowJar(dependsOn: shadowJar) {
    description "vendors jar dependencies"

    String vendorPathPrefix = "vendor/jar-dependencies/co/elastic"
    File projectJarFile = file("${vendorPathPrefix}/${project.name}/${project.version}/${project.name}-${project.version}.jar")
    File shadowJar = project.file("${buildDir}/libs/${project.name}-${project.version}-all.jar")

    inputs.file shadowJar
    outputs.dir vendorPathPrefix

    doLast {
        projectJarFile.mkdirs()
        Files.copy(shadowJar.toPath(), projectJarFile.toPath(), REPLACE_EXISTING)
    }
}

task generateGemJarRequiresFile() {
    description "Generates a ruby script for runtime-requiring vendored jars"
    dependsOn vendorShadowJar

    def jarRequiresFile = file("${projectDir}/lib/logstash/filters/elastic_integration/jar_dependencies.rb")

    outputs.file jarRequiresFile
    inputs.property 'project.group', project.group
    inputs.property 'project.name', project.name
    inputs.property 'project.version', project.version

    doLast {
        jarRequiresFile.parentFile.mkdirs()
        jarRequiresFile.withWriter { w ->
            w << "# AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.\n\n"
            w << '########################################################################\n' +
                 '# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V.\n' +
                 '# under one or more contributor license agreements. Licensed under the\n' +
                 '# Elastic License 2.0; you may not use this file except in compliance\n' +
                 '# with the Elastic License 2.0.\n' +
                 '########################################################################\n'
            w << '\n'
            w << "require \'jar_dependencies\'\n"
            w << "require_jar('co/elastic', \'${project.name}\', \'${project.version}\')\n"
        }
    }
}
task generateGemVersionFile() {
    description "Generates a ruby script including version information"

    def versionFile = file("${projectDir}/lib/logstash/filters/elastic_integration/version.rb")
    outputs.file versionFile
    inputs.property 'project.version', project.version

    doLast {
        versionFile.parentFile.mkdirs();
        versionFile.withWriter { w ->
            w << "# AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.\n\n"
            w << '########################################################################\n' +
                 '# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V.\n' +
                 '# under one or more contributor license agreements. Licensed under the\n' +
                 '# Elastic License 2.0; you may not use this file except in compliance\n' +
                 '# with the Elastic License 2.0.\n' +
                 '########################################################################\n'
            w << '\n'
            w << "LogStash::Filters::ElasticIntegration::VERSION='${project.version}'\n"
        }
    }
}

task rubyAssemble {
    description "vendors dependencies and generates required ruby code"

    dependsOn vendorShadowJar
    dependsOn generateGemJarRequiresFile

    dependsOn generateGemVersionFile
}

task vendor {
    description "vendors dependencies and generates the ruby code to require them"
    dependsOn rubyAssemble
}

task publish(dependsOn: vendor) {
    doLast {
        exec {
            commandLine "bundle", "exec", "rake", "publish_gem"
        }
    }
}

task localGem(dependsOn: vendor) {
    doLast {
        exec {
            commandLine "gem", "build", "logstash-filter-elastic_integration.gemspec"
        }
    }
}

task bundleInstall(dependsOn: vendor) {
    def logstashPath = envOrPropertyValue("LOGSTASH_PATH")
    assert logstashPath != null : "The property LOGSTASH_PATH must be defined, and be path to logstash directory"
    inputs.file("logstash-filter-elastic_integration.gemspec")
    inputs.file("Gemfile")
    inputs.property("LOGSTASH_PATH", logstashPath)

    outputs.file("Gemfile.lock")

    doLast {
        exec {
            environment(Map.of("LOGSTASH_PATH", logstashPath, "LOGSTASH_SOURCE", "1"))
            commandLine "bundle", "install"
        }
    }
}

task unitSpecs(dependsOn: bundleInstall) {

    def unitSpecFiles = project.fileTree("spec/unit") { include "**/*_spec.rb" }.files*.path
    assert unitSpecFiles : "No unit spec files could be found"

    inputs.files unitSpecFiles
    inputs.files project.fileTree("lib/**/*.rb")
    inputs.files tasks.named('compileJava').map { it.outputs.files }

    doLast {
        exec {
            commandLine "bundle", "exec", "rspec"
            args "--format=documentation"
            if (project.gradle.startParameter.consoleOutput != ConsoleOutput.Plain) {
                args "--color"
                args "--tty"
            }

            args unitSpecFiles
        }
    }
}

clean {
    delete "${projectDir}/*.gem"
}