Update Spark to version 3.1.1

.sbtopts

@@ -1 +1,7 @@
--J-XX:MaxMetaspaceSize=512m
+# SBT laucher evicts most memory-related JVM options and attempts to determine
+# their values itself based on the amount of memory passed to the -mem option.
+# The code cache size is set to 1/8 of the -mem value, resulting in 128M code
+# cache for the default -mem value of 1024. This is not enough for a clean
+# build, in which the code cache size grows to slightly over 200M. Setting the
+# -mem option to 2048 gives us 256M of code cache.
+-mem 2048

.travis.yml

@@ -3,127 +3,128 @@ dist: trusty
 
 language: java
 
-env:
-  global:
-    - RENAISSANCE_VERSION="0.11.0"
-
 refs:
-  - &bundle
-    stage: bundle
+  - &build_ref
+    stage: Build
     script:
       # Fail the script if any command fails
-      - set -e
-      - 'java -version'
-      - 'javac -version'
-      - ./tools/sbt/bin/sbt assembly
-      - ./tools/sbt/bin/sbt renaissanceJmh/jmh:assembly
-      # Ensure that the JMH bundle contains non-empty BenchmarkList
-      - 'test $(unzip -p renaissance-jmh/target/scala-2.12/renaissance-jmh-assembly-$RENAISSANCE_VERSION.jar META-INF/BenchmarkList | wc -l) -gt 0'
-      # Clear cache directory
-      - 'mkdir -p "$HOME/.prebuilt"'
-      - 'ls "$HOME/.prebuilt"'
-      - 'rm -f "$HOME/.prebuilt"/*.jar'
-      # Store bundles in cache directory
-      - 'cp target/renaissance-gpl-$RENAISSANCE_VERSION.jar "$HOME/.prebuilt/$(git rev-parse HEAD).jar"'
-      - 'cp renaissance-jmh/target/scala-2.12/renaissance-jmh-assembly-$RENAISSANCE_VERSION.jar "$HOME/.prebuilt/$(git rev-parse HEAD)-jmh-assembly.jar"'
-
-  - &bench
-    stage: benchmark
+      - set -o errexit
+      # Build the base and JMH bundles
+      - tools/ci/cache-get.sh
+      - tools/ci/build-base.sh
+      - tools/ci/build-jmh.sh
+      - tools/ci/stop-sbt.sh
+      - tools/ci/check-jmh.sh
+      - tools/ci/cache-put.sh
+
+  - &bench_ref
+    stage: Bench
     script:
       # Fail the script if any command fails
-      - set -e
-      # Get bundles from cache or build them
-      - mkdir -p target
-      - CACHED_JAR_NAME="$HOME/.prebuilt/$(git rev-parse HEAD).jar"; if [ -e "$CACHED_JAR_NAME" ]; then cp "$CACHED_JAR_NAME" target/renaissance-gpl-$RENAISSANCE_VERSION.jar; else ./tools/sbt/bin/sbt assembly; fi
-      - mkdir -p renaissance-jmh/target/scala-2.12
-      - CACHED_JMH_JAR_NAME="$HOME/.prebuilt/$(git rev-parse HEAD)-jmh-assembly.jar"; if [ -e "$CACHED_JMH_JAR_NAME" ]; then cp "$CACHED_JMH_JAR_NAME" renaissance-jmh/target/scala-2.12/renaissance-jmh-assembly-$RENAISSANCE_VERSION.jar; else ./tools/sbt/bin/sbt renaissanceJmh/jmh:assembly; fi
-      # Ensure that the JMH bundle contains non-empty BenchmarkList
-      - 'test $(unzip -p renaissance-jmh/target/scala-2.12/renaissance-jmh-assembly-$RENAISSANCE_VERSION.jar META-INF/BenchmarkList | wc -l) -gt 0'
-      - 'java -version'
-      # Test benchmarks under Renaissance harness
-      - 'java -jar ./target/renaissance-gpl-$RENAISSANCE_VERSION.jar --raw-list >list.txt'
-      - 'for BENCH in $(cat list.txt); do echo "====> $BENCH"; java -Xms2500M -Xmx2500M -jar ./target/renaissance-gpl-$RENAISSANCE_VERSION.jar -c test -r 1 --csv output.csv --json output.json "$BENCH" || exit 1; done'
-      # Test benchmarks under JMH harness, replacing incompatible benchmarks with 'dummy-empty'
-      - 'java -Xms2500M -Xmx2500M -Dorg.renaissance.jmh.fakeIncompatible=true -jar ./renaissance-jmh/target/scala-2.12/renaissance-jmh-assembly-$RENAISSANCE_VERSION.jar -wi 0 -i 1 -f 1 -foe true'
+      - set -o errexit
+      # Test the base and JMH bundles
+      - tools/ci/cache-get.sh
+      - tools/ci/build-base.sh
+      - tools/ci/build-jmh.sh
+      - tools/ci/stop-sbt.sh
+      - tools/ci/check-jmh.sh
+      - tools/ci/bench-base.sh
+      - tools/ci/bench-jmh.sh
 
 jobs:
   include:
-    - stage: "Basic checks"
-      name: "Check style"
-      script:
-        - ./tools/sbt/bin/sbt renaissanceFormatCheck
-
-    - stage: "Basic checks"
-      name: "README.md is up to date"
-      script:
-        - ./tools/sbt/bin/sbt assembly
-        - java -jar ./target/renaissance-gpl-$RENAISSANCE_VERSION.jar --readme && git diff --exit-code -- README.md CONTRIBUTION.md
-
-    - stage: "Basic checks"
-      name: "Content encoding of Java sources"
+    # Pre-flight check on OS X (most fragile)
+    - stage: Check
+      name: "OS X, JDK9, Oracle"
+      os: osx
+      osx_image: xcode9.3
+      jdk: oraclejdk9
       script:
-        - ./tools/check-encoding.sh
+        # Fail the script if any command fails
+        - set -o errexit
+        - tools/ci/check-encoding.sh
+        - tools/ci/check-sbt.sh
+        - tools/ci/check-formatting.sh
+        - tools/ci/build-base.sh
+        - tools/ci/stop-sbt.sh
+        - tools/ci/check-markdown.sh
+        - tools/ci/cache-put.sh
 
-    - <<: *bundle
+    # Build and cache with different JDKs
+    - <<: *build_ref
+      name: "OS X, JDK9, Oracle"
       os: osx
       osx_image: xcode9.3
       jdk: oraclejdk9
-    - <<: *bundle
+
+    - <<: *build_ref
+      name: "Linux, JDK8, OpenJDK"
       jdk: openjdk8
-    - <<: *bundle
+
+    - <<: *build_ref
+      name: "Linux, JDK8, Oracle"
       jdk: oraclejdk8
-    - <<: *bundle
-      jdk: oraclejdk9
-    - <<: *bundle
-      env:
-       - USE_JDK=openjdk11
-    - <<: *bundle
+
+    - <<: *build_ref
+      name: "Linux, JDK11, OpenJDK"
+      jdk: openjdk11
+
+    - <<: *build_ref
+      name: "Linux, JDK11, Oracle"
+      jdk: oraclejdk11
+
+    - <<: *build_ref
+      name: "Linux, JDK11, OpenJ9"
       env:
-        - USE_JDK=OpenJ9
+        - USE_JDK=openj9jdk11
 
-    - <<: *bench
+    # Test benchmarks with different JDKs
+    - <<: *bench_ref
+      name: "OS X, JDK9, Oracle"
       os: osx
       osx_image: xcode9.3
       jdk: oraclejdk9
-    - <<: *bench
+
+    - <<: *bench_ref
+      name: "Linux, JDK8, OpenJDK"
       jdk: openjdk8
-    - <<: *bench
+
+    - <<: *bench_ref
+      name: "Linux, JDK8, Oracle"
       jdk: oraclejdk8
-    - <<: *bench
-      jdk: oraclejdk9
-    - <<: *bench
+
+    - <<: *bench_ref
+      name: "Linux, JDK11, OpenJDK"
+      jdk: openjdk11
+
+    - <<: *bench_ref
+      name: "Linux, JDK11, Oracle"
+      jdk: oraclejdk11
+
+    - <<: *bench_ref
+      name: "Linux, JDK11, OpenJ9"
       env:
-       - USE_JDK=openjdk11
-    # Disabled because of #131
-    #- <<: *bench
-    #  env:
-    #    - USE_JDK=OpenJ9
+        - USE_JDK=openj9jdk11
 
 stages:
-  - "Basic checks"
-  - bundle
-  - benchmark
+  - Check
+  - Build
+  - Bench
 
 before_script:
-  # For us, the default options are broken as they pass "-v"
-  # to the JVM instead after the sbt JAR.
-  - unset SBT_OPTS
-  - "# Install custom JDK if needed"
-  - 'if [ -n "$USE_JDK" ]; then wget "https://github.com/sormuras/bach/raw/master/install-jdk.sh" && chmod +x install-jdk.sh; fi'
-  - 'if [ -n "$USE_JDK" ]; then export JAVA_HOME="$HOME/$USE_JDK"; fi'
-  - 'if [ "$USE_JDK" = "OpenJ9" ]; then ./install-jdk.sh --url "https://api.adoptopenjdk.net/v2/binary/releases/openjdk11?openjdk_impl=openj9&os=linux&arch=x64&release=latest&heap_size=normal&type=jdk" --target "$JAVA_HOME"; fi'
-  - 'if [ "$USE_JDK" = "openjdk11" ]; then ./install-jdk.sh -f 11 --target "$JAVA_HOME"; fi'
-  - 'if [ -n "$USE_JDK" ]; then export PATH="$JAVA_HOME/bin:$PATH"; fi'
+  # Setup JDK indicated in USE_JDK.
+  - source tools/ci/setup-jdk.sh
 
 cache:
   directories:
+    - $HOME/.cache/coursier
     - $HOME/.ivy2/cache
-    - $HOME/.sbt
     - $HOME/.prebuilt
+    - $HOME/.sbt
 
 before_cache:
   # Cleanup the cached directories to avoid unnecessary cache updates
   # (see https://www.scala-sbt.org/1.0/docs/Travis-CI-with-sbt.html#Caching)
-  - rm -fv $HOME/.ivy2/.sbt.ivy.lock
+  - find $HOME/{.ivy2,.sbt} -type f -name "*.lock" -print -delete
+  - find $HOME/.sbt -type f -name "*-libswoval-*" -print -delete
   - find $HOME/.ivy2/cache -name "ivydata-*.properties" -print -delete
-  - find $HOME/.sbt -name "*.lock" -print -delete

benchmarks/actors-reactors/reactors/build.sbt

@@ -62,24 +62,6 @@ def jvmProjectSettings(suffix: String) =
   )
 
 
-def gitPropsContents(dir: File, baseDir: File): Seq[File] = {
-  def run(cmd: String*): String = scala.sys.process.Process(cmd, Some(baseDir)).!!
-  val branch = run("git", "rev-parse", "--abbrev-ref", "HEAD").trim
-  val commitTs = run("git", "--no-pager", "show", "-s", "--format=%ct", "HEAD")
-  val sha = run("git", "rev-parse", "HEAD").trim
-  val contents = s"""
-  {
-    "branch": "$branch",
-    "commit-timestamp": $commitTs,
-    "sha": "$sha"
-  }
-  """
-  val file = dir / "reactors-io" / ".gitprops"
-  IO.write(file, contents)
-  Seq(file)
-}
-
-
 // Produces reactorsCommonJVM
 
 lazy val reactorsCommon = crossProject(JVMPlatform)
@@ -114,9 +96,6 @@ lazy val reactorsCore = crossProject(JVMPlatform)
   .in(file("reactors-core"))
   .settings(
     projectSettings("-core") ++ Seq(
-      Compile / resourceGenerators += Def.task {
-        gitPropsContents((Compile / resourceManaged).value, baseDirectory.value)
-      },
       libraryDependencies ++= Seq(
         "org.scalatest" %%% "scalatest" % scalaTestVersion % "test",
         "org.scalacheck" %%% "scalacheck" % scalaCheckVersion % "test"

benchmarks/actors-reactors/reactors/version.sbt

@@ -1 +1 @@
-version in ThisBuild := "0.9-renaissance-83d194"
+version := "0.9-e605e145-60-g9838414"

benchmarks/apache-spark/build.sbt

@@ -1,19 +1,13 @@
 lazy val renaissanceCore = RootProject(uri("../../renaissance-core"))
 
-// Note: these versions should not be changed here.
-// If we want to target a different Spark version, we should instead clone this subproject.
-// That way, we make it convenient to run the old versions,
-// and avoid the benchmarking of a moving target.
-val sparkScalaVersion = "2.12.13"
-
-val sparkVersion = "3.0.1"
+val sparkVersion = "3.1.1"
 
 lazy val apacheSpark = (project in file("."))
   .settings(
     name := "apache-spark",
     version := (version in renaissanceCore).value,
     organization := (organization in renaissanceCore).value,
-    scalaVersion := sparkScalaVersion,
+    scalaVersion := "2.12.13",
     libraryDependencies ++= Seq(
       "org.apache.spark" %% "spark-core" % sparkVersion,
       "org.apache.spark" %% "spark-sql" % sparkVersion,

benchmarks/apache-spark/src/main/scala/org/renaissance/apache/spark/SparkUtil.scala

@@ -54,10 +54,10 @@ trait SparkUtil {
   }
 
   /**
-   * Spark version 2.4.7 uses Hadoop version 2.6.5. The dependencies
+   * Spark version 3.1.1 uses Hadoop version 3.2.0. The dependencies
    * do not include a binary zip for Hadoop on Windows. Instead,
    * Renaissance includes winutils.exe as a resource, downloaded from
-   * https://github.com/cdarlint/winutils/tree/master/hadoop-2.6.5/bin
+   * https://github.com/cdarlint/winutils/tree/master/hadoop-3.2.0/bin
    *
    * When updating Spark in Renaissance, the file must be upgraded to the
    * corresponding Hadoop version from https://github.com/cdarlint/winutils

build.sbt

@@ -7,6 +7,8 @@ import org.renaissance.core.Launcher
 
 import scala.collection._
 
+enablePlugins(GitBranchPrompt)
+
 lazy val renaissanceCore = RootProject(uri("renaissance-core"))
 
 lazy val renaissanceHarness = RootProject(uri("renaissance-harness"))

renaissance-core/build.sbt

@@ -1,3 +1,6 @@
+enablePlugins(GitVersioning)
+git.useGitDescribe := true
+
 lazy val renaissanceCore = (project in file("."))
   .settings(
     name := "renaissance-core",

renaissance-core/project/plugins.sbt

@@ -1 +1,2 @@
 addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.2")
+addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.0")

renaissance-core/src/main/java/org/renaissance/core/Launcher.java

@@ -25,9 +25,10 @@ public static void main(String[] args) {
     try {
       if (args.length == 1 && "--readme".equalsIgnoreCase(args[0])) {
         final Package benchmarkPkg = Benchmark.class.getPackage();
+        final String fullVersion = benchmarkPkg.getImplementationVersion();
         final String[] generatorArgs = new String[] {
           "--title", benchmarkPkg.getSpecificationTitle(),
-          "--version", benchmarkPkg.getImplementationVersion()
+          "--version", fullVersion.split("-")[0]
         };
 
         // TODO Launch the generator from the build system.

renaissance-core/version.sbt

@@ -1 +1 @@
-version := "0.11.0" // also requires a readme update and a .travis.yml change
+git.baseVersion := "0.12.0"

tools/ci/bench-base.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+source "$(dirname "$0")/common.sh"
+
+# Test benchmarks under Renaissance harness
+java -jar "$RENAISSANCE_JAR" --raw-list > list.txt
+
+for BENCH in $(cat list.txt); do
+	echo "====> $BENCH"
+	java -Xms2500M -Xmx2500M -jar "$RENAISSANCE_JAR" -c test -r 1 --csv output.csv --json output.json "$BENCH" || exit 1
+done

tools/ci/bench-jmh.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+source "$(dirname "$0")/common.sh"
+
+# Test benchmarks under JMH harness, replacing incompatible benchmarks with 'dummy-empty'
+java -Xms2500M -Xmx2500M -Dorg.renaissance.jmh.fakeIncompatible=true -jar "$RENAISSANCE_JMH_JAR" -wi 0 -i 1 -f 1 -foe true

tools/ci/build-base.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+source "$(dirname "$0")/common.sh"
+
+# Build the base bundle if missing
+[ -s "$RENAISSANCE_JAR" ] || ci_sbt --client --batch assembly

tools/ci/build-jmh.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+source "$(dirname "$0")/common.sh"
+
+# Build the JMH bundle if missing
+[ -s "$RENAISSANCE_JMH_JAR" ] || ci_sbt --client --batch renaissanceJmh/jmh:assembly

tools/ci/cache-get.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+source "$(dirname "$0")/common.sh"
+
+# Show the bundle cache directory
+ls -1 "$CACHE_DIR"
+
+# Get the base bundle from cache
+if [ -s "$RENAISSANCE_JAR_CACHED" ]; then
+	mkdir -p "$RENAISSANCE_DIR"
+	cp_reflink "$RENAISSANCE_JAR_CACHED" "$RENAISSANCE_DIR"
+fi
+
+# Get the JMH bundle from cache
+if [ -s "$RENAISSANCE_JMH_JAR_CACHED" ]; then
+	mkdir -p "$RENAISSANCE_JMH_DIR"
+	cp_reflink "$RENAISSANCE_JMH_JAR_CACHED" "$RENAISSANCE_JMH_DIR"
+fi