Avoid packaging / unpacking cycle when using local current distributions (#61592) (#62176)

- Extract distribution archives defaults into plugin
- Added basic test coverage
- Avoid packaging/unpackaging cycle when relying on locally build distributions
- Provide DSL for setting up distribution archives
- Cleanup archives build script

Author: breskeby

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPluginFuncTest.groovy

@@ -0,0 +1,208 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.internal
+
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream
+import org.apache.tools.zip.ZipEntry
+import org.apache.tools.zip.ZipFile
+import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
+import org.gradle.testkit.runner.BuildResult
+import org.gradle.testkit.runner.TaskOutcome
+
+class InternalDistributionArchiveSetupPluginFuncTest extends AbstractGradleFuncTest {
+
+    def setup() {
+        buildFile << """
+        import org.elasticsearch.gradle.tar.SymbolicLinkPreservingTar
+        
+        plugins {
+            id 'elasticsearch.internal-distribution-archive-setup'
+        }
+        """
+        file('someFile.txt') << "some content"
+    }
+
+    def "applies defaults to tar tasks"() {
+        given:
+        file('someFile.txt') << "some content"
+        buildFile << """
+        tasks.register('${buildTaskName}', SymbolicLinkPreservingTar) {
+            from 'someFile.txt'
+        }
+        """
+
+        when:
+        def result = gradleRunner(buildTaskName).build()
+
+        then:
+        file(expectedOutputArchivePath).exists()
+        assertTarPermissionDefaults(file(expectedOutputArchivePath))
+        assertEmptyDirTasksTriggered(result)
+
+        where:
+        buildTaskName       | expectedOutputArchivePath
+        "buildDarwinTar"    | "darwin-tar/build/distributions/elasticsearch.tar.gz"
+        "buildOssDarwinTar" | "oss-darwin-tar/build/distributions/elasticsearch-oss.tar.gz"
+    }
+
+    def "applies defaults to zip tasks"() {
+        given:
+        buildFile << """
+        tasks.register('${buildTaskName}', Zip) {
+            from 'someFile.txt'
+        }
+        """
+
+        when:
+        def result = gradleRunner(buildTaskName).build()
+
+        then:
+        file(expectedOutputArchivePath).exists()
+        assertZipPermissionDefaults(file(expectedOutputArchivePath))
+        assertEmptyDirTasksTriggered(result)
+
+        where:
+        buildTaskName       | expectedOutputArchivePath
+        "buildDarwinZip"    | "darwin-zip/build/distributions/elasticsearch.zip"
+        "buildOssDarwinZip" | "oss-darwin-zip/build/distributions/elasticsearch-oss.zip"
+    }
+
+    def "registered distribution provides archives and directory variant"() {
+        given:
+        file('someFile.txt') << "some content"
+
+        settingsFile << """
+            include ':consumer'
+            include ':producer-tar'
+        """
+
+        buildFile << """
+        import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
+        import org.gradle.api.internal.artifacts.ArtifactAttributes;
+
+        distribution_archives {
+            producerTar {
+                content {
+                    project.copySpec {
+                        from 'someFile.txt'
+                    }
+                }
+            }
+        }
+        
+        project('consumer') { p ->
+            configurations {
+                consumeArchive {}
+                consumeDir {}
+            }
+            
+            dependencies {
+                consumeDir project(path: ':producer-tar', configuration:'extracted')
+                consumeArchive project(path: ':producer-tar', configuration:'default' )
+            }
+            
+            tasks.register("copyDir", Copy) {
+                from(configurations.consumeDir)
+                into('build/dir')
+            }
+            
+            tasks.register("copyArchive", Copy) {
+                from(configurations.consumeArchive)
+                into('build/archives')
+            }
+        }
+        """
+        when:
+        def result = gradleRunner("copyArchive").build()
+
+        then:"tar task executed and target folder contains plain tar"
+        result.task(':buildProducerTar').outcome == TaskOutcome.SUCCESS
+        result.task(':consumer:copyArchive').outcome == TaskOutcome.SUCCESS
+        file("producer-tar/build/distributions/elasticsearch.tar.gz").exists()
+        file("consumer/build/archives/elasticsearch.tar.gz").exists()
+
+        when:
+        result = gradleRunner("copyDir").build()
+        then:"plain copy task executed and target folder contains plain content"
+        result.task(':buildProducer').outcome == TaskOutcome.SUCCESS
+        result.task(':consumer:copyDir').outcome == TaskOutcome.SUCCESS
+        file("producer-tar/build/install/someFile.txt").exists()
+        file("consumer/build/dir/someFile.txt").exists()
+    }
+
+    private static boolean assertTarPermissionDefaults(File tarArchive) {
+        TarArchiveInputStream tarInput = new TarArchiveInputStream(new GzipCompressorInputStream(new FileInputStream(tarArchive)))
+        try {
+            TarArchiveEntry currentEntry = tarInput.getNextTarEntry()
+            while (currentEntry != null) {
+                if (currentEntry.isDirectory()) {
+                    assertDefaultDirPermissions(currentEntry.getMode())
+                } else {
+                    assertDefaultFilePermissions(currentEntry.getMode())
+                }
+                currentEntry = tarInput.getNextTarEntry()
+            }
+            return true
+        } finally {
+            tarInput.close()
+        }
+    }
+
+    private static boolean assertZipPermissionDefaults(File archive) {
+        ZipFile zip = new ZipFile(archive)
+        try {
+            Enumeration<ZipEntry> entries = zip.getEntries()
+            while (entries.hasMoreElements()) {
+                ZipEntry zipEntry = entries.nextElement()
+                if (zipEntry.isDirectory()) {
+                    assertDefaultDirPermissions(zipEntry.getUnixMode())
+                } else {
+                    assertDefaultFilePermissions(zipEntry.getUnixMode())
+                }
+            }
+        } finally {
+            zip.close()
+        }
+        true
+    }
+
+    private static boolean assertDefaultDirPermissions(int mode) {
+        assert ((mode >> 6) & 07) == 7
+        assert ((mode >> 3) & 07) == 5
+        assert ((mode >> 0) & 07) == 5
+        true
+    }
+
+    private static boolean assertDefaultFilePermissions(int mode) {
+        assert ((mode >> 6) & 07) == 6
+        assert ((mode >> 3) & 07) == 4
+        assert ((mode >> 0) & 07) == 4
+        true
+    }
+
+    private static boolean assertEmptyDirTasksTriggered(BuildResult result) {
+        result.task(":createJvmOptionsDir").outcome == TaskOutcome.SUCCESS
+        result.task(":createLogsDir").outcome == TaskOutcome.SUCCESS
+        result.task(":createPluginsDir").outcome == TaskOutcome.SUCCESS
+        true
+    }
+}

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy

@@ -72,7 +72,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
         def result = gradleRunner("setupDistro", '-g', testProjectDir.newFolder('GUH').path).build()
 
         then:
-        result.task(":distribution:archives:linux-tar:buildTar").outcome == TaskOutcome.SUCCESS
+        result.task(":distribution:archives:linux-tar:buildExploded").outcome == TaskOutcome.SUCCESS
         result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
         assertExtractedDistroIsCreated(distroVersion, "build/distro", 'current-marker.txt')
     }
@@ -183,14 +183,29 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
         def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "archives", "linux-tar")
         new File(bwcSubProjectFolder, 'current-marker.txt') << "current"
         new File(bwcSubProjectFolder, 'build.gradle') << """
+            import org.gradle.api.internal.artifacts.ArtifactAttributes;
+
             apply plugin:'distribution'
-            tasks.register("buildTar", Tar) {
+
+            def buildTar = tasks.register("buildTar", Tar) {
                 from('current-marker.txt')
                 archiveExtension = "tar.gz"
                 compression = Compression.GZIP
             }
+            def buildExploded = tasks.register("buildExploded", Copy) {
+                from('current-marker.txt')
+                into("build/local")
+            }
+            configurations {
+                extracted {
+                    attributes {
+                          attribute(ArtifactAttributes.ARTIFACT_FORMAT, "directory")
+                    }
+                }
+            }
             artifacts {
                 it.add("default", buildTar)
+                it.add("extracted", buildExploded)
             }
         """
         buildFile << """

buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDependency.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle;
+
+public interface DistributionDependency {
+    static DistributionDependency of(String dependencyNotation) {
+        return new StringBasedDistributionDependency(dependencyNotation);
+    }
+
+    Object getDefaultNotation();
+
+    Object getExtractedNotation();
+
+    class StringBasedDistributionDependency implements DistributionDependency {
+        private final String notation;
+
+        public StringBasedDistributionDependency(String notation) {
+            this.notation = notation;
+        }
+
+        @Override
+        public Object getDefaultNotation() {
+            return notation;
+        }
+
+        @Override
+        public Object getExtractedNotation() {
+            return notation;
+        }
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java

@@ -44,7 +44,7 @@
 
 /**
  * A plugin to manage getting and extracting distributions of Elasticsearch.
- *
+ * <p>
  * The plugin provides hooks to register custom distribution resolutions.
  * This plugin resolves distributions from the Elastic downloads service if
  * no registered resolution strategy can resolve to a distribution.
@@ -122,24 +122,24 @@ void setupDistributions(Project project) {
             distribution.finalizeValues();
             DependencyHandler dependencies = project.getDependencies();
             // for the distribution as a file, just depend on the artifact directly
-            Object resolvedDependency = resolveDependencyNotation(project, distribution);
-            dependencies.add(distribution.configuration.getName(), resolvedDependency);
+            DistributionDependency distributionDependency = resolveDependencyNotation(project, distribution);
+            dependencies.add(distribution.configuration.getName(), distributionDependency.getDefaultNotation());
             // no extraction allowed for rpm, deb or docker
             if (distribution.getType().shouldExtract()) {
                 // The extracted configuration depends on the artifact directly but has
                 // an artifact transform registered to resolve it as an unpacked folder.
-                dependencies.add(distribution.getExtracted().getName(), resolvedDependency);
+                dependencies.add(distribution.getExtracted().getName(), distributionDependency.getExtractedNotation());
             }
         }
     }
 
-    private Object resolveDependencyNotation(Project p, ElasticsearchDistribution distribution) {
+    private DistributionDependency resolveDependencyNotation(Project p, ElasticsearchDistribution distribution) {
         return distributionsResolutionStrategiesContainer.stream()
             .sorted(Comparator.comparingInt(DistributionResolution::getPriority))
             .map(r -> r.getResolver().resolve(p, distribution))
             .filter(d -> d != null)
             .findFirst()
-            .orElseGet(() -> dependencyNotation(distribution));
+            .orElseGet(() -> DistributionDependency.of(dependencyNotation(distribution)));
     }
 
     private static void addIvyRepo(Project project, String name, String url, String group) {
@@ -177,7 +177,7 @@ private static void setupDownloadServiceRepo(Project project) {
      * Maven coordinates point to either the integ-test-zip coordinates on maven central, or a set of artificial
      * coordinates that resolve to the Elastic download service through an ivy repository.
      */
-    private Object dependencyNotation(ElasticsearchDistribution distribution) {
+    private String dependencyNotation(ElasticsearchDistribution distribution) {
         if (distribution.getType() == Type.INTEG_TEST_ZIP) {
             return "org.elasticsearch.distribution.integ-test-zip:elasticsearch:" + distribution.getVersion() + "@zip";
         }

buildSrc/src/main/java/org/elasticsearch/gradle/DistributionResolution.java

@@ -51,6 +51,6 @@ public int getPriority() {
     }
 
     public interface Resolver {
-        Object resolve(Project project, ElasticsearchDistribution distribution);
+        DistributionDependency resolve(Project project, ElasticsearchDistribution distribution);
     }
 }

buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java

@@ -193,6 +193,10 @@ public Architecture getArchitecture() {
 
     @Override
     public String toString() {
+        return getName() + "_" + getType() + "_" + getVersion();
+    }
+
+    public String getFilepath() {
         return configuration.getSingleFile().toString();
     }