Merge pull request #156 from DataDog/jb/merge_upstream

Merging changes from the upstream repo

Author: jbachorik

.circleci/config.yml

@@ -129,6 +129,8 @@ jobs:
       - image: memcached
         # This is used by rabbitmq instrumentation tests
       - image: rabbitmq
+        # This is used by aerospike instrumentation tests
+      - image: aerospike
 
     parameters:
       testTask:
@@ -239,10 +241,13 @@ jobs:
           command: >-
             SKIP_BUILDSCAN="true"
             GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx4G -Xms64M' -Ddatadog.forkedMaxHeapSize=4G -Ddatadog.forkedMinHeapSize=64M"
-            ./gradlew `circleci tests split workspace/build/muzzleTasks | xargs`
+            ./gradlew `circleci tests split --split-by=timings workspace/build/muzzleTasks | xargs`
             << pipeline.parameters.gradle_flags >>
             --max-workers=16
 
+      - store_test_results:
+          path: workspace/build/muzzle-test-results
+
 workflows:
   build_test_deploy:
     jobs:

buildSrc/src/main/groovy/MuzzlePlugin.groovy

@@ -109,34 +109,60 @@ class MuzzlePlugin implements Plugin<Project> {
       return
     }
 
+    // We only get here if we are running muzzle, so let's start timing things
+    long startTime = System.currentTimeMillis()
+
     final RepositorySystem system = newRepositorySystem()
     final RepositorySystemSession session = newRepositorySystemSession(system)
 
     project.afterEvaluate {
       // use runAfter to set up task finalizers in version order
       Task runAfter = project.tasks.muzzle
+      // runLast is the last task to finish, so we can time the execution
+      Task runLast = runAfter
 
       for (MuzzleDirective muzzleDirective : project.muzzle.directives) {
         project.getLogger().info("configured $muzzleDirective")
 
         if (muzzleDirective.coreJdk) {
-          runAfter = addMuzzleTask(muzzleDirective, null, project, runAfter, bootstrapProject, toolingProject)
+          runLast = runAfter = addMuzzleTask(muzzleDirective, null, project, runAfter, bootstrapProject, toolingProject)
         } else {
-          muzzleDirectiveToArtifacts(muzzleDirective, system, session).collect() { Artifact singleVersion ->
+          runLast = muzzleDirectiveToArtifacts(muzzleDirective, system, session).inject(runLast) { last, Artifact singleVersion ->
             runAfter = addMuzzleTask(muzzleDirective, singleVersion, project, runAfter, bootstrapProject, toolingProject)
           }
           if (muzzleDirective.assertInverse) {
-            inverseOf(muzzleDirective, system, session).collect() { MuzzleDirective inverseDirective ->
-              muzzleDirectiveToArtifacts(inverseDirective, system, session).collect() { Artifact singleVersion ->
+            runLast = inverseOf(muzzleDirective, system, session).inject(runLast) { last1, MuzzleDirective inverseDirective ->
+              muzzleDirectiveToArtifacts(inverseDirective, system, session).inject(last1) { last2, Artifact singleVersion ->
                 runAfter = addMuzzleTask(inverseDirective, singleVersion, project, runAfter, bootstrapProject, toolingProject)
               }
             }
           }
         }
       }
+      def timingTask = project.task("muzzle-end") {
+        doLast {
+          long endTime = System.currentTimeMillis()
+          generateResultsXML(project, endTime - startTime)
+        }
+      }
+      runLast.finalizedBy(timingTask)
     }
   }
 
+  private static void generateResultsXML(Project project, long millis) {
+    def seconds = (millis * 1.0) / 1000
+    def name = "${project.path}:muzzle"
+    def dirname = name.replaceFirst('^:', '').replace(':', '_')
+    def dir = project.file("${project.rootProject.buildDir}/muzzle-test-results/$dirname")
+    dir.mkdirs()
+    def file = project.file("$dir/results.xml")
+    file.text = """|<?xml version="1.0" encoding="UTF-8"?>
+                   |<testsuite name="${name}" tests="1" id="0" time="${seconds}">
+                   |  <testcase name="${name}" time="${seconds}">
+                   |  </testcase>
+                   |</testsuite>\n""".stripMargin()
+  }
+
   private static ClassLoader getOrCreateToolingLoader(Project toolingProject) {
     synchronized (TOOLING_LOADER) {
       final ClassLoader toolingLoader = TOOLING_LOADER.get()
@@ -216,14 +242,12 @@ class MuzzlePlugin implements Plugin<Project> {
     rangeRequest.setRepositories(MUZZLE_REPOS)
     rangeRequest.setArtifact(directiveArtifact)
     final VersionRangeResult rangeResult = system.resolveVersionRange(session, rangeRequest)
-    final Set<Version> versions = rangeResult.versions.toSet()
-
-    limitLargeRanges(rangeResult, versions, muzzleDirective.skipVersions)
+    final Set<Version> versions = filterAndLimitVersions(rangeResult, muzzleDirective.skipVersions)
 
 //    println "Range Request: " + rangeRequest
 //    println "Range Result: " + rangeResult
 
-    final Set<Artifact> allVersionArtifacts = filterVersion(versions, muzzleDirective.skipVersions).collect { version ->
+    final Set<Artifact> allVersionArtifacts = versions.collect { version ->
       new DefaultArtifact(muzzleDirective.group, muzzleDirective.module, "jar", version.toString())
     }.toSet()
 
@@ -238,50 +262,51 @@ class MuzzlePlugin implements Plugin<Project> {
    * Create a list of muzzle directives which assert the opposite of the given MuzzleDirective.
    */
   private static Set<MuzzleDirective> inverseOf(MuzzleDirective muzzleDirective, RepositorySystem system, RepositorySystemSession session) {
-    Set<MuzzleDirective> inverseDirectives = new HashSet<>()
-
-    final Artifact allVerisonsArtifact = new DefaultArtifact(muzzleDirective.group, muzzleDirective.module, "jar", "[,)")
+    final Artifact allVersionsArtifact = new DefaultArtifact(muzzleDirective.group, muzzleDirective.module, "jar", "[,)")
     final Artifact directiveArtifact = new DefaultArtifact(muzzleDirective.group, muzzleDirective.module, "jar", muzzleDirective.versions)
 
-
     final VersionRangeRequest allRangeRequest = new VersionRangeRequest()
     allRangeRequest.setRepositories(MUZZLE_REPOS)
-    allRangeRequest.setArtifact(allVerisonsArtifact)
+    allRangeRequest.setArtifact(allVersionsArtifact)
     final VersionRangeResult allRangeResult = system.resolveVersionRange(session, allRangeRequest)
 
     final VersionRangeRequest rangeRequest = new VersionRangeRequest()
     rangeRequest.setRepositories(MUZZLE_REPOS)
     rangeRequest.setArtifact(directiveArtifact)
     final VersionRangeResult rangeResult = system.resolveVersionRange(session, rangeRequest)
     final Set<Version> versions = rangeResult.versions.toSet()
+    allRangeResult.versions.removeAll(versions)
+
+    return filterAndLimitVersions(allRangeResult, muzzleDirective.skipVersions).collect { version ->
+      final MuzzleDirective inverseDirective = new MuzzleDirective()
+      inverseDirective.group = muzzleDirective.group
+      inverseDirective.module = muzzleDirective.module
+      inverseDirective.versions = "$version"
+      inverseDirective.assertPass = !muzzleDirective.assertPass
+      inverseDirective
+    }.toSet()
+  }
 
-    filterVersion(allRangeResult.versions.toSet(), muzzleDirective.skipVersions).collect { version ->
-      if (!versions.contains(version)) {
-        final MuzzleDirective inverseDirective = new MuzzleDirective()
-        inverseDirective.group = muzzleDirective.group
-        inverseDirective.module = muzzleDirective.module
-        inverseDirective.versions = "$version"
-        inverseDirective.assertPass = !muzzleDirective.assertPass
-        inverseDirectives.add(inverseDirective)
-      }
-    }
-
-    return inverseDirectives
+  private static Set<Version> filterAndLimitVersions(VersionRangeResult result, Set<String> skipVersions) {
+    return limitLargeRanges(result, filterVersion(result.versions.toSet(), skipVersions), skipVersions)
   }
 
-  private static void limitLargeRanges(VersionRangeResult result, Set<Version> versions, Set<String> skipVersions) {
-    List<Version> copy = new ArrayList<>(versions)
-    Collections.shuffle(copy)
-    copy.removeAll(skipVersions)
-    while (RANGE_COUNT_LIMIT <= copy.size()) {
-      Version version = copy.pop()
+  private static Set<Version> limitLargeRanges(VersionRangeResult result, Set<Version> versions, Set<String> skipVersions) {
+    List<Version> versionsCopy = new ArrayList<>(versions)
+    Set<String> skipCopy = new HashSet<>(skipVersions)
+    versionsCopy.removeAll(skipVersions)
+    Collections.shuffle(versionsCopy)
+    while (RANGE_COUNT_LIMIT <= versionsCopy.size()) {
+      Version version = versionsCopy.pop()
       if (!(version.equals(result.lowestVersion) || version.equals(result.highestVersion))) {
-        skipVersions.add(version.toString())
+        skipCopy.add(version.toString())
       }
     }
-    if (skipVersions.size() > 0) {
-      println "Muzzle skipping " + skipVersions.size() + " versions"
+    if (skipCopy.size() > 0) {
+      println "Muzzle skipping " + skipCopy.size() + " versions"
     }
+
+    return versionsCopy.toSet()
   }
 
   /**
@@ -380,6 +405,7 @@ class MuzzlePlugin implements Plugin<Project> {
   }
 
   private static final Pattern GIT_SHA_PATTERN = Pattern.compile('^.*-[0-9a-f]{7,}$')
+  private static final Pattern END_NMN_PATTERN = Pattern.compile('^.*\\.[0-9]+[mM][0-9]+$')
 
   /**
    * Filter out snapshot-type builds from versions list.
@@ -398,7 +424,9 @@ class MuzzlePlugin implements Plugin<Project> {
         version.contains("-ea") ||
         version.contains("-atlassian-") ||
         version.contains("public_draft") ||
+        version.contains("-cr") ||
         skipVersions.contains(version) ||
+        version.matches(END_NMN_PATTERN) ||
         version.matches(GIT_SHA_PATTERN)
     }
     return list

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java

@@ -1,5 +1,6 @@
 package datadog.trace.bootstrap;
 
+import static datadog.trace.api.Platform.isJavaVersionAtLeast;
 import static datadog.trace.util.AgentThreadFactory.AgentThread.JMX_STARTUP;
 import static datadog.trace.util.AgentThreadFactory.AgentThread.PROFILER_STARTUP;
 import static datadog.trace.util.AgentThreadFactory.AgentThread.TRACE_STARTUP;
@@ -111,7 +112,7 @@ public static void start(final Instrumentation inst, final URL bootstrapURL) {
      * events which in turn loads LogManager. This is not a problem on newer JDKs because there JFR uses different
      * logging facility.
      */
-    if (isJavaBefore9() && appUsingCustomLogManager) {
+    if (!isJavaVersionAtLeast(9) && appUsingCustomLogManager) {
       log.debug("Custom logger detected. Delaying Profiling Agent startup.");
       registerLogManagerCallback(new StartProfilingAgentCallback(inst, bootstrapURL));
     } else {
@@ -237,7 +238,7 @@ private static synchronized void createParentClassloader(final URL bootstrapURL)
         BOOTSTRAP_PROXY = (ClassLoader) constructor.newInstance(bootstrapURL);
 
         final ClassLoader grandParent;
-        if (isJavaBefore9()) {
+        if (!isJavaVersionAtLeast(9)) {
           grandParent = null; // bootstrap
         } else {
           // platform classloader is parent of system in java 9+
@@ -298,11 +299,6 @@ private static synchronized void startJmx(final URL bootstrapURL) {
     }
     initializeJmxSystemAccessProvider(AGENT_CLASSLOADER);
 
-    if (PROFILING_CLASSLOADER == null) {
-      throw new IllegalStateException("Datadog profiling agent should have been started already");
-    }
-    initializeJmxSystemAccessProvider(PROFILING_CLASSLOADER);
-
     registerDeadlockDetectionEvent(bootstrapURL);
   }
 
@@ -606,12 +602,8 @@ private static boolean isAppUsingCustomJMXBuilder() {
     return false;
   }
 
-  private static boolean isJavaBefore9() {
-    return System.getProperty("java.version").startsWith("1.");
-  }
-
   private static boolean isJavaBefore9WithJFR() {
-    if (!isJavaBefore9()) {
+    if (isJavaVersionAtLeast(9)) {
       return false;
     }
     // FIXME: this is quite a hack because there maybe jfr classes on classpath somehow that have

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/InternalJarURLHandler.java

@@ -28,14 +28,6 @@ public class InternalJarURLHandler extends URLStreamHandler {
   private final Map<String, Lock> packages = new HashMap<>();
   private final JarFile bootstrapJarFile;
 
-  private static final ThreadLocal<StringBuilder> JAR_ENTRY_QUERY =
-      new ThreadLocal<StringBuilder>() {
-        @Override
-        protected StringBuilder initialValue() {
-          return new StringBuilder(128);
-        }
-      };
-
   private WeakReference<Pair<String, JarEntry>> cache = NULL;
 
   InternalJarURLHandler(final String internalJarFileName, final URL bootstrapJarLocation) {
@@ -94,13 +86,7 @@ protected URLConnection openConnection(final URL url) throws IOException {
     // and the key will be a new object each time.
     Pair<String, JarEntry> pair = cache.get();
     if (null == pair || !filename.equals(pair.getLeft())) {
-      StringBuilder sb = JAR_ENTRY_QUERY.get();
-      sb.append(this.name).append(filename);
-      if (filename.endsWith(".class")) {
-        sb.append("data");
-      }
-      String classFileName = sb.toString();
-      sb.setLength(0);
+      String classFileName = this.name + filename + (filename.endsWith(".class") ? "data" : "");
       JarEntry entry = bootstrapJarFile.getJarEntry(classFileName);
       if (null != entry) {
         pair = Pair.of(filename, entry);

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/WeakCache.java

@@ -1,19 +1,15 @@
 package datadog.trace.bootstrap;
 
-import java.util.concurrent.Callable;
+import datadog.trace.api.Function;
 
 public interface WeakCache<K, V> {
-  interface Provider<K, V> {
-    WeakCache<K, V> newWeakCache();
-
-    WeakCache<K, V> newWeakCache(final long maxSize);
+  interface Provider {
+    <K, V> WeakCache<K, V> newWeakCache(long maxSize);
   }
 
   V getIfPresent(final K key);
 
-  V getIfPresentOrCompute(final K key, final Callable<? extends V> loader);
-
-  V get(final K key, final Callable<? extends V> loader);
+  V computeIfAbsent(K key, Function<? super K, ? extends V> mappingFunction);
 
   void put(final K key, final V value);
 }

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/WeakMap.java

@@ -1,5 +1,6 @@
 package datadog.trace.bootstrap;
 
+import datadog.trace.api.Function;
 import java.util.Collections;
 import java.util.Map;
 import java.util.WeakHashMap;
@@ -18,7 +19,7 @@ public interface WeakMap<K, V> {
 
   void putIfAbsent(K key, V value);
 
-  V computeIfAbsent(K key, ValueSupplier<? super K, ? extends V> supplier);
+  V computeIfAbsent(K key, Function<? super K, ? extends V> supplier);
 
   @Slf4j
   class Provider {
@@ -58,14 +59,6 @@ public <K, V> WeakMap<K, V> get() {
     }
   }
 
-  /**
-   * Supplies the value to be stored and it is called only when a value does not exists yet in the
-   * registry.
-   */
-  interface ValueSupplier<K, V> {
-    V get(K key);
-  }
-
   class MapAdapter<K, V> implements WeakMap<K, V> {
     private final Object[] locks = new Object[16];
     private final Map<K, V> map;
@@ -111,14 +104,14 @@ public void putIfAbsent(final K key, final V value) {
     }
 
     @Override
-    public V computeIfAbsent(final K key, final ValueSupplier<? super K, ? extends V> supplier) {
+    public V computeIfAbsent(final K key, final Function<? super K, ? extends V> supplier) {
       // We can't use computeIfAbsent since it was added in 1.8.
       V value = map.get(key);
       if (null == value) {
         synchronized (locks[key.hashCode() & (locks.length - 1)]) {
           value = map.get(key);
           if (null == value) {
-            value = supplier.get(key);
+            value = supplier.apply(key);
             map.put(key, value);
           }
         }

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/decorator/BaseDecorator.java

@@ -37,7 +37,7 @@ public String apply(Class<?> clazz) {
 
   protected final boolean endToEndDurationsEnabled;
   protected final boolean traceAnalyticsEnabled;
-  protected final float traceAnalyticsSampleRate;
+  protected final Double traceAnalyticsSampleRate;
 
   protected BaseDecorator() {
     final Config config = Config.get();
@@ -47,7 +47,7 @@ protected BaseDecorator() {
             && config.isTraceAnalyticsIntegrationEnabled(
                 traceAnalyticsDefault(), instrumentationNames);
     this.traceAnalyticsSampleRate =
-        config.getInstrumentationAnalyticsSampleRate(instrumentationNames);
+        (double) config.getInstrumentationAnalyticsSampleRate(instrumentationNames);
     this.endToEndDurationsEnabled =
         instrumentationNames.length > 0
             && config.isEndToEndDurationEnabled(endToEndDurationsDefault(), instrumentationNames);
@@ -74,7 +74,7 @@ public AgentSpan afterStart(final AgentSpan span) {
     }
     span.setTag(Tags.COMPONENT, component());
     if (traceAnalyticsEnabled) {
-      span.setTag(DDTags.ANALYTICS_SAMPLE_RATE, traceAnalyticsSampleRate);
+      span.setMetric(DDTags.ANALYTICS_SAMPLE_RATE, traceAnalyticsSampleRate);
     }
     if (endToEndDurationsEnabled) {
       if (null == span.getBaggageItem(DDTags.TRACE_START_TIME)) {