Fix unknown licenses

build.gradle

@@ -543,7 +543,7 @@ subprojects { project ->
   }
 }
 
-/* Remove assemble on all qa projects because we don't need to publish
+/* Remove assemble/dependenciesInfo on all qa projects because we don't need to publish
  * artifacts for them. */
 gradle.projectsEvaluated {
   subprojects {
@@ -553,6 +553,11 @@ gradle.projectsEvaluated {
         project.tasks.remove(assemble)
         project.build.dependsOn.remove('assemble')
       }
+      Task dependenciesInfo = project.tasks.findByName('dependenciesInfo')
+      if (dependenciesInfo) {
+        project.tasks.remove(dependenciesInfo)
+        project.precommit.dependsOn.remove('dependenciesInfo')
+      }
     }
   }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -762,6 +762,10 @@ class BuildPlugin implements Plugin<Project> {
 
     private static configureDependenciesInfo(Project project) {
         Task deps = project.tasks.create("dependenciesInfo", DependenciesInfoTask.class)
-        deps.dependencies = project.configurations.compile.allDependencies
+        deps.runtimeConfiguration = project.configurations.runtime
+        deps.compileOnlyConfiguration = project.configurations.compileOnly
+        project.afterEvaluate {
+            deps.mappings = project.dependencyLicenses.mappings
+        }
     }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/DependenciesInfoTask.groovy

@@ -19,14 +19,19 @@
 
 package org.elasticsearch.gradle
 
+import org.elasticsearch.gradle.precommit.DependencyLicensesTask
 import org.gradle.api.DefaultTask
+import org.gradle.api.artifacts.Configuration
 import org.gradle.api.artifacts.Dependency
+import org.gradle.api.artifacts.DependencyResolutionListener
 import org.gradle.api.artifacts.DependencySet
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.InputDirectory
 import org.gradle.api.tasks.OutputFile
 import org.gradle.api.tasks.TaskAction
 
+import java.util.regex.Matcher
+import java.util.regex.Pattern
 
 /**
  * A task to gather information about the dependencies and export them into a csv file.
@@ -44,7 +49,14 @@ public class DependenciesInfoTask extends DefaultTask {
 
     /** Dependencies to gather information from. */
     @Input
-    public DependencySet dependencies
+    public Configuration runtimeConfiguration
+
+    /** We subtract compile-only dependencies. */
+    @Input
+    public Configuration compileOnlyConfiguration
+
+    @Input
+    public LinkedHashMap<String, String> mappings
 
     /** Directory to read license files */
     @InputDirectory
@@ -59,15 +71,34 @@ public class DependenciesInfoTask extends DefaultTask {
 
     @TaskAction
     public void generateDependenciesInfo() {
+
+        final DependencySet runtimeDependencies = runtimeConfiguration.getAllDependencies()
+        // we have to resolve the transitive dependencies and create a group:artifactId:version map
+        final Set<String> compileOnlyArtifacts =
+                compileOnlyConfiguration
+                        .getResolvedConfiguration()
+                        .resolvedArtifacts
+                        .collect { it -> "${it.moduleVersion.id.group}:${it.moduleVersion.id.name}:${it.moduleVersion.id.version}" }
+
         final StringBuilder output = new StringBuilder()
 
-        for (Dependency dependency : dependencies) {
-            // Only external dependencies are checked
-            if (dependency.group != null && dependency.group.contains("elasticsearch") == false) {
-                final String url = createURL(dependency.group, dependency.name, dependency.version)
-                final String licenseType = getLicenseType(dependency.group, dependency.name)
-                output.append("${dependency.group}:${dependency.name},${dependency.version},${url},${licenseType}\n")
+        for (final Dependency dependency : runtimeDependencies) {
+            // we do not need compile-only dependencies here
+            if (compileOnlyArtifacts.contains("${dependency.group}:${dependency.name}:${dependency.version}")) {
+                continue
+            }
+            // only external dependencies are checked
+            if (dependency.group != null && dependency.group.contains("org.elasticsearch")) {
+                continue
             }
+
+            final String url = createURL(dependency.group, dependency.name, dependency.version)
+            final String dependencyName = DependencyLicensesTask.getDependencyName(mappings, dependency.name)
+            logger.info("mapped dependency ${dependency.group}:${dependency.name} to ${dependencyName} for license info")
+
+            final String licenseType = getLicenseType(dependency.group, dependencyName)
+            output.append("${dependency.group}:${dependency.name},${dependency.version},${url},${licenseType}\n")
+
         }
         outputFile.setText(output.toString(), 'UTF-8')
     }
@@ -173,7 +204,7 @@ are met:
     derived from this software without specific prior written permission\\.|
   (3\\.)? Neither the name of .+ nor the names of its
      contributors may be used to endorse or promote products derived from
-     this software without specific prior written permission\\.)  
+     this software without specific prior written permission\\.)
 
 THIS SOFTWARE IS PROVIDED BY .+ (``|''|")AS IS(''|") AND ANY EXPRESS OR
 IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/DependencyLicensesTask.groovy

@@ -109,6 +109,10 @@ public class DependencyLicensesTask extends DefaultTask {
         mappings.put(from, to)
     }
 
+    public LinkedHashMap<String, String> getMappings() {
+        return new LinkedHashMap<>(mappings)
+    }
+
     /**
      * Add a rule which will skip SHA checking for the given dependency name. This should be used for
      * locally build dependencies, which cause the sha to change constantly.
@@ -129,10 +133,6 @@ public class DependencyLicensesTask extends DefaultTask {
             throw new GradleException("Licences dir ${licensesDir} does not exist, but there are dependencies")
         }
 
-
-        // order is the same for keys and values iteration since we use a linked hashmap
-        List<String> mapped = new ArrayList<>(mappings.values())
-        Pattern mappingsPattern = Pattern.compile('(' + mappings.keySet().join(')|(') + ')')
         Map<String, Integer> licenses = new HashMap<>()
         Map<String, Integer> notices = new HashMap<>()
         Set<File> shaFiles = new HashSet<File>()
@@ -162,16 +162,10 @@ public class DependencyLicensesTask extends DefaultTask {
                 checkSha(dependency, jarName, shaFiles)
             }
 
-            logger.info("Checking license/notice for " + depName)
-            Matcher match = mappingsPattern.matcher(depName)
-            if (match.matches()) {
-                int i = 0
-                while (i < match.groupCount() && match.group(i + 1) == null) ++i;
-                logger.info("Mapped dependency name ${depName} to ${mapped.get(i)} for license check")
-                depName = mapped.get(i)
-            }
-            checkFile(depName, jarName, licenses, 'LICENSE')
-            checkFile(depName, jarName, notices, 'NOTICE')
+            final String dependencyName = getDependencyName(mappings, depName)
+            logger.info("mapped dependency name ${depName} to ${dependencyName} for license/notice check")
+            checkFile(dependencyName, jarName, licenses, 'LICENSE')
+            checkFile(dependencyName, jarName, notices, 'NOTICE')
         }
 
         licenses.each { license, count ->
@@ -189,6 +183,19 @@ public class DependencyLicensesTask extends DefaultTask {
         }
     }
 
+    public static String getDependencyName(final LinkedHashMap<String, String> mappings, final String dependencyName) {
+        // order is the same for keys and values iteration since we use a linked hashmap
+        List<String> mapped = new ArrayList<>(mappings.values())
+        Pattern mappingsPattern = Pattern.compile('(' + mappings.keySet().join(')|(') + ')')
+        Matcher match = mappingsPattern.matcher(dependencyName)
+        if (match.matches()) {
+            int i = 0
+            while (i < match.groupCount() && match.group(i + 1) == null) ++i;
+            return mapped.get(i)
+        }
+        return dependencyName
+    }
+
     private File getShaFile(String jarName) {
         return new File(licensesDir, jarName + SHA_EXTENSION)
     }

client/client-benchmark-noop-api-plugin/build.gradle

@@ -31,6 +31,9 @@ esplugin {
 tasks.remove(assemble)
 build.dependsOn.remove('assemble')
 
+dependencyLicenses.enabled = false
+dependenciesInfo.enabled = false
+
 compileJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-try,-unchecked"
 
 // no unit tests

distribution/build.gradle

@@ -32,6 +32,11 @@ Collection distributions = project('archives').subprojects + project('packages')
 
 // Concatenates the dependencies CSV files into a single file
 task generateDependenciesReport(type: ConcatFilesTask) {
+  project.rootProject.allprojects {
+    afterEvaluate {
+      if (it.tasks.findByName("dependenciesInfo")) dependsOn it.tasks.dependenciesInfo
+    }
+  }
   files = fileTree(dir: project.rootDir,  include: '**/dependencies.csv' )
   headerLine = "name,version,url,license"
   target = new File(System.getProperty('csv')?: "${project.buildDir}/dependencies/es-dependencies.csv")

test/fixtures/example-fixture/build.gradle

@@ -22,3 +22,5 @@ test.enabled = false
 // Not published so no need to assemble
 tasks.remove(assemble)
 build.dependsOn.remove('assemble')
+
+dependenciesInfo.enabled = false

x-pack/qa/build.gradle

@@ -28,5 +28,9 @@ gradle.projectsEvaluated {
       project.tasks.remove(assemble)
       project.build.dependsOn.remove('assemble')
     }
+    Task dependenciesInfo = project.tasks.findByName('dependenciesInfo')
+    if (dependenciesInfo) {
+      project.precommit.dependsOn.remove('dependenciesInfo')
+    }
   }
 }

x-pack/qa/sql/build.gradle

@@ -22,6 +22,7 @@ dependencies {
 test.enabled = false
 
 dependencyLicenses.enabled = false
+dependenciesInfo.enabled = false
 
 // the main files are actually test files, so use the appropriate forbidden api sigs
 forbiddenApisMain {

x-pack/test/feature-aware/build.gradle

@@ -10,6 +10,7 @@ dependencies {
 forbiddenApisMain.enabled = true
 
 dependencyLicenses.enabled = false
+dependenciesInfo.enabled = false
 
 jarHell.enabled = false