Initial support for the SPECjbb perf testing benchmark (#4363)

* initial commit * complete build config for specjbb * add run scripts * move build script and update locations in build file * add playlist file for specjbb * remove comment * remove quotes from JAVA env var * move test env make file into run dir * update perf readme to include specjbb * add a readme for the specjbb benchmark * fix formatting for xml files * clean up shell script comments * remove unnessasary step of checking current dst since the build dir is always new * change rsync command to cp since we always start with a clean build dir * change SPEC_OPTS to SPECJBB_OPTS * reword comments to match the action that they are describing * add comments to describe shell scripts * update SpecJBB job from initial TRSS testing (#5) * update SpecJBB job from initial TRSS testing * fix bug * fixup * rewrite getSpecjbbSuite.sh using antfile * elaborate on the reasoning for the jvm options that were used * Update perf/specjbb/README.md * Update perf/specjbb/README.md * Update perf/specjbb/README.md * Update perf/specjbb/run/run_multi.sh * Update perf/specjbb/run/run_multi.sh Co-authored-by: George Adams <george.adams@microsoft.com> * update perf/specjbb/README.md Co-authored-by: Shelley Lambert <slambert@gmail.com> * add test to the dev level so it will never get included as a necessary test to run as part of AQAvit verification Co-authored-by: Shelley Lambert <slambert@gmail.com> * update perf/specjbb/README.md Co-authored-by: Shelley Lambert <slambert@gmail.com> * specify microsoft as the vendor that this test should run for Co-authored-by: Shelley Lambert <slambert@gmail.com> * fix playlist.xml formatting --------- Co-authored-by: George Adams <georgeadams1995@gmail.com> Co-authored-by: Martijn Verburg <martijnverburg@gmail.com> Co-authored-by: George Adams <george.adams@microsoft.com> Co-authored-by: Shelley Lambert <slambert@gmail.com>
adoptium · Mar 7, 2023 · 43354d1 · 43354d1
1 parent a996bea
commit 43354d1
Show file tree

Hide file tree

Showing 7 changed files with 298 additions and 0 deletions.
diff --git a/perf/README.md b/perf/README.md
@@ -49,4 +49,7 @@ Liberty benchmarks from https://github.com/OpenLiberty - including liberty-dt7-s
 #### renaissance
 Renaissance benchmarks from https://github.com/renaissance-benchmarks/renaissance - including renaissance-akka-uct, renaissance-als, renaissance-chi-square, renaissance-db-shootout, renaissance-dec-tree, renaissance-finagle-chirper, renaissance-finagle-http, renaissance-fj-kmeans, renaissance-future-genetic, renaissance-gauss-mix, renaissance-log-regression, renaissance-mnemonics, renaissance-movie-lens, renaissance-naive-bayes, renaissance-par-mnemonics, renaissance-philosophers and renaissance-scala-kmeans
 
+#### specjbb [licence required]
+SPECjbb 2015 benchmark from https://www.spec.org/jbb2015/  
+
 Additional benchmarks are being reviewed for addition and if you wish to include more, please comment in the open performance benchmarks [issue 1112](https://github.com/adoptium/aqa-tests/issues/1112).
diff --git a/perf/specjbb/README.md b/perf/specjbb/README.md
@@ -0,0 +1,34 @@
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+[1]https://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+-->
+
+
+## SPECjbb 2015
+
+SPECjbb is a Java Client/Server performance testing benchmark. 
+Please visit its product page for more information, including a user guide: https://www.spec.org/jbb2015/
+
+## Prerequisites
+Since SPECjbb requires a license to run, this aqa-test config requires that a licensed copy of the benchmark already exists on the machine that you are running on.
+
+## Setup
+
+Set the `SPECJBB_SRC` environment variable to the absolute path of your SPECjbb benchmark directory
+
+```sh
+export SPECJBB_SRC=/Users/aqa-tester/SPECjbb2015-1.03
+```
+
+## Run
+
+This test target is part of the `dev` level, and the test target name is, `SPECjbb2015-multijvm-simple`. You can use the following commands to run it:
+```
+make _SPECjbb2015-multijvm-simple
+```
diff --git a/perf/specjbb/build.xml b/perf/specjbb/build.xml
@@ -0,0 +1,112 @@
+<?xml version="1.0"?>
+
+<!--
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+
+<project name="Specjbb suite" default="build" basedir=".">
+    <taskdef resource="net/sf/antcontrib/antlib.xml" />
+    <description>Specjbb suite</description>
+
+    <!-- set global properties for this build -->
+    <property name="DEST" value="${BUILD_ROOT}/perf/specjbb" />
+    <property name="SRC" location="." />
+    <property environment="env"/>
+    <property name="SPECJBB_DEST" value="${DEST}/suite" />
+    <property name="SPECJBB_SRC" value="${env.SPECJBB_SRC}" />
+    <property name="JAVA_BIN" value="${env.TEST_JDK_HOME}/bin/java"/>
+    <property name="SPECJBB_JAR" value="specjbb2015.jar"/>
+
+    <target name="init">
+        <mkdir dir="${DEST}" />
+    </target>
+
+    <target name="exists">
+        <property name="specjbb_dir" value="${dir}"/>
+            <condition property="exists.exists">
+                <and>
+                    <available file="${specjbb_dir}/${SPECJBB_JAR}" type="file"/>
+                </and>
+            </condition>
+    </target>
+
+    <target name="isRunnable">
+        <exec executable="${JAVA_BIN}" resultproperty="isRunnable.result" >
+            <arg line="-jar ${specjbb_dir}/${SPECJBB_JAR} -m COMPOSITE -h"/>
+        </exec>
+        <fail message="Failed to run SPECjbb suite.">
+            <condition>
+                <not>
+                    <equals arg1="${isRunnable.result}" arg2="0"/>
+                </not>
+            </condition>
+        </fail>
+    </target>
+
+    <target name="dirIsValid">
+        <antcall target="exists">
+            <param name="specjbb_dir" value="${dir}"/>
+        </antcall>
+        <antcall target="isRunnable">
+            <param name="specjbb_dir" value="${dir}"/>
+        </antcall>
+    </target>
+
+    <target name="dstIsValid">
+        <echo message="##### Validating destination SPECjbb dir"/>
+        <antcall target="dirIsValid">
+            <param name="dir" value="${SPECJBB_DEST}"/>
+        </antcall>
+            <condition property="dstIsValid.result">
+                <isset property="dirIsValid.result"/>
+            </condition>
+    </target>
+
+    <target name="srcIsValid">
+        <echo message="##### Validating source SPECjbb dir"/>
+        <antcall target="dirIsValid">
+            <param name="dir" value="${SPECJBB_SRC}"/>
+        </antcall>
+    </target>
+
+    <target name="copySrcToDst">
+        <echo message="##### Copying contents of source SPECjbb dir to destination SPECjbb dir"/>
+        <copy todir="${SPECJBB_DEST}">
+            <fileset dir="${SPECJBB_SRC}"/>
+        </copy>
+    </target>
+
+    <target name="printInput">
+        <echo message="SPECJBB_SRC ${SPECJBB_SRC}"/>
+        <echo message="SPECJBB_DEST ${SPECJBB_DEST}"/>
+        <echo message="SPECJBB_JAR ${SPECJBB_JAR}"/>
+        <echo message="JAVA_BIN ${JAVA_BIN}"/>
+    </target>
+
+    <target name="getSpecjbbSuite" depends="init">
+        <antcall target="printInput"/>
+        <antcall target="srcIsValid"/>
+        <antcall target="copySrcToDst"/>
+        <antcall target="dstIsValid"/>
+    </target>
+
+    <target name="dist" depends="getSpecjbbSuite" description="generate the distribution">
+        <copy todir="${DEST}/run">
+            <fileset dir="${SRC}/run"/>
+        </copy>
+    </target>
+
+    <target name="build">
+        <antcall target="dist" inheritall="true" />
+    </target>
+</project>
diff --git a/perf/specjbb/playlist.xml b/perf/specjbb/playlist.xml
@@ -0,0 +1,33 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<!--
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+<playlist xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../../TKG/playlist.xsd">
+    <include>run/testenv.mk</include>
+    <test>
+        <!-- The minimum hardware requirement to run this test is 8GB of memory and 8 CPU cores -->
+        <!-- see specjbb/run/options/specjbb-multijvm-simple.sh for more details -->
+
+        <testCaseName>SPECjbb2015-multijvm-simple</testCaseName>
+        <command>. $(RUN_OPTIONS)/specjbb-multijvm-simple.sh; bash $(RUN_SCRIPTS)/run_multi.sh; $(TEST_STATUS)</command>
+        <levels>
+            <level>dev</level>
+        </levels>
+        <groups>
+            <group>perf</group>
+        </groups>
+        <vendors>
+            <vendor>microsoft</vendor>
+        </vendors>
+    </test>
+</playlist>
diff --git a/perf/specjbb/run/options/specjbb-multijvm-simple.sh b/perf/specjbb/run/options/specjbb-multijvm-simple.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# This will configure a basic run of SPECjbb in multi-jvm mode
+# Therefore, the topography of the run includes 1 group consisting of
+# 1 TransactionInjector and 1 Backend
+
+export GROUP_COUNT=1
+export TI_JVM_COUNT=1 
+export NUM_OF_RUNS=1
+
+export SPECJBB_OPTS_C="-Dspecjbb.group.count=$GROUP_COUNT -Dspecjbb.txi.pergroup.count=$TI_JVM_COUNT"
+export SPECJBB_OPTS_TI=""
+export SPECJBB_OPTS_BE=""
+
+# The Controller, TransactionInjector, and Backend (aka System Under Test; SUT) are configured to meet the minimum 
+# hardware requirements suggested by SPECjbb 
+# (see the SPECjbb 2015 user guide, section 2.3 'Minimum hardware requirements', for more details)
+#
+# This implies that a machine should have atleast 8GB of memory and 8 CPU cores to run this test
+export JAVA_OPTS_C="-Xms2g -Xmx2g -Xmn1536m -XX:+UseParallelGC -XX:ParallelGCThreads=2 -XX:CICompilerCount=4"
+export JAVA_OPTS_TI="${JAVA_OPTS_C}" 
+export JAVA_OPTS_BE="-Xms4g -Xmx4g -Xmn3g -XX:+UseParallelGC -XX:ParallelGCThreads=4 -XX:-UseAdaptiveSizePolicy"
+
+export MODE_ARGS_C=""
+export MODE_ARGS_TI=""
+export MODE_ARGS_BE=""
+
diff --git a/perf/specjbb/run/run_multi.sh b/perf/specjbb/run/run_multi.sh
@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+
+
+echo "Backend opts: $JAVA_OPTS_BE"
+echo "Number of runs: ${NUM_OF_RUNS}"
+echo "Results dir: ${RESULTS_DIR}"
+
+for ((n=1; $n<=$NUM_OF_RUNS; n=$n+1)); do
+
+  # Create result directory                
+  temp="${RESULTS_DIR%\"}"
+  temp="${temp#\"}"
+  timestamp=$(date '+%y-%m-%d_%H%M%S')
+  result="${temp}/${timestamp}"
+  mkdir -pv $result
+
+  # Copy current config to the result directory
+  cp -r $SPECJBB_CONFIG $result
+
+  cd $result
+
+  echo "Run $n: $timestamp"
+  echo "Launching SPECjbb2015 in MultiJVM mode..."
+  echo
+
+  JAVA="${JAVA%\"}"
+  JAVA="${JAVA#\"}"
+  echo "Start Controller JVM"
+  $JAVA $JAVA_OPTS_C $SPECJBB_OPTS_C -jar $SPECJBB_JAR -m MULTICONTROLLER $MODE_ARGS_C 2>controller.log 2>&1 | tee controller.out &
+
+  CTRL_PID=$!
+  echo "Controller PID = $CTRL_PID"
+
+  sleep 3
+
+  for ((gnum=1; $gnum<$GROUP_COUNT+1; gnum=$gnum+1)); do
+
+    GROUPID=Group$gnum
+    echo -e "\nStarting JVMs from $GROUPID:"
+
+    for ((jnum=1; $jnum<$TI_JVM_COUNT+1; jnum=$jnum+1)); do
+
+        JVMID=txiJVM$jnum
+        TI_NAME=$GROUPID.TxInjector.$JVMID
+
+        echo "    Start $TI_NAME"
+        $JAVA $JAVA_OPTS_TI $SPECJBB_OPTS_TI -jar $SPECJBB_JAR -m TXINJECTOR -G=$GROUPID -J=$JVMID $MODE_ARGS_TI > $TI_NAME.log 2>&1 &
+        echo -e "\t$TI_NAME PID = $!"
+        sleep 1
+    done
+
+    JVMID=beJVM
+    BE_NAME=$GROUPID.Backend.$JVMID
+
+    # Add GC logging to the backend's JVM options 
+    JAVA_OPTS_BE_WITH_GC_LOG="$JAVA_OPTS_BE -Xlog:gc*:file=${BE_NAME}_gc.log"
+    echo "    Start $BE_NAME"
+    $JAVA $JAVA_OPTS_BE_WITH_GC_LOG $SPECJBB_OPTS_BE -jar $SPECJBB_JAR -m BACKEND -G=$GROUPID -J=$JVMID $MODE_ARGS_BE > $BE_NAME.log 2>&1 &
+    echo -e "\t$BE_NAME PID = $!"
+    sleep 1
+
+  done
+
+  echo
+  echo "SPECjbb2015 is running..."
+  echo "Please monitor $result/controller.out for progress"
+
+  wait $CTRL_PID
+  echo
+  echo "Controller has stopped"
+
+  echo "SPECjbb2015 has finished"
+  echo
+
+  cd $WORKING_DIR
+
+done
+
+exit 0
diff --git a/perf/specjbb/run/testenv.mk b/perf/specjbb/run/testenv.mk
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+SPECJBB_BUILD=$(JVM_TEST_ROOT)/perf/specjbb
+
+export JAVA=$(JAVA_COMMAND)
+export SPECJBB_JAR=$(SPECJBB_BUILD)/suite/specjbb2015.jar
+export SPECJBB_CONFIG=$(SPECJBB_BUILD)/suite/config
+export RUN_SCRIPTS=$(SPECJBB_BUILD)/run
+export RUN_OPTIONS=$(SPECJBB_BUILD)/run/options
+export RESULTS_DIR=$(REPORTDIR)