419 benchmarking spmv spmspv spmspm (#75)

This MR adds three kernel-level performance tests to ALP/GraphBLAS: sparse matrix--dense vector multiplication, sparse matrix--sparse vector multiplication, and sparse matrix--sparse matrix multiplication. These three tests run on arbitrary input data sets, and employ the usual benchmarking methodology by default: run at least 10 experiments while computing the min, max, avg, and stddev aggregates and while sleeping for one second between experiments.
Individual timings, again when used in default mode, will take at least 100 ms.-- if a single computation is shorter than that, then an "individual" timing in fact corresponds to n such timings, where n is the minimum number of times for which the computation takes 100 milliseconds or more. Both the resulting "inner" (100 ms.) and "outer" (10) repetitions may of course also be manually set.

The performance test script is updated to run these three new performance tests on a fixed series of datasets. If one of the test sets is not available, then the experiment is skipped. For the sparse matrix--sparse matrix multiplication, square matrices are multiplied with itself; this test is, by default, only enabled for the smaller matrices in the dataset. The dataset series can easily be extended in the script should need be.

The new performance tests are only run for the smallest datasets for the hyperdags backend, in-line with the current behaviour of performance tests for that backend already present in the main branch.

The MR also includes minor code style fixes, as well as a workaround for CIs that run out of storage capacity due to the -g generating large binary files, while, related, not archiving some additional files that are not required to execute tests after building them.

Thanks to Anders Hansson and Aristeidis Mastoras for contributing this MR!

Author: aristeidis-mastoras

.gitlab-ci.yml

@@ -122,12 +122,15 @@ build_test:
     - apt update && apt -y install make cmake libnuma-dev coreutils
   script:
     - mkdir -p install build && cd ./build && ../bootstrap.sh --prefix=../install && make -j$(nproc) build_tests_all
+    - strip -s $(find tests/unit/ -type f -executable -print) $(find tests/smoke/ -type f -executable -print) $(find tests/performance/ -type f -executable -print)
   artifacts:
     paths:
       - build/
     exclude:
       - build/**/*.o
       - build/**/*.o.d
+      - build/**/CMakeFiles
+      - build/**/*.dir
     expire_in: 30 minutes
 
 

include/graphblas/algorithms/conjugate_gradient.hpp

@@ -384,7 +384,7 @@ namespace grb {
 
 				sigma = beta;
 
-			} while( iter++ < max_iterations && ret == SUCCESS );
+			} while( ++iter < max_iterations && ret == SUCCESS );
 
 			// output
 			iterations = iter;

tests/performance/CMakeLists.txt

@@ -78,6 +78,21 @@ add_grb_executables( driver_label ../smoke/label.cpp
 	ADDITIONAL_LINK_LIBRARIES test_utils_headers
 )
 
+add_grb_executables( driver_spmv spmv.cpp
+	BACKENDS reference reference_omp bsp1d hybrid hyperdags
+	ADDITIONAL_LINK_LIBRARIES test_utils_headers
+)
+
+add_grb_executables( driver_spmspv spmspv.cpp
+	BACKENDS reference reference_omp bsp1d hybrid hyperdags
+	ADDITIONAL_LINK_LIBRARIES test_utils_headers
+)
+
+add_grb_executables( driver_spmspm spmspm.cpp
+	BACKENDS reference reference_omp bsp1d hybrid hyperdags
+	ADDITIONAL_LINK_LIBRARIES test_utils_headers
+)
+
 # targets to list and build the test for this category
 get_property( performance_tests_list GLOBAL PROPERTY tests_category_performance )
 add_custom_target( "list_tests_category_performance"

tests/performance/performancetests.sh

@@ -73,6 +73,9 @@ DATASET_SIZES=(497 4039 27770 334863 1134890 3774768 3072441)
 KNN4SOLS=(118 499 2805 1 64 1 1048907)
 KNN6SOLS=(357 547 5176 1 246 1 1453447)
 
+#the following datasets are used for benchmarking SpMV, SpMSpV, and SpMSpM
+MULTIPLICATION_DATASETS=(west0497.mtx fidap037.mtx cavity17.mtx s3rmt3m3.mtx bloweybq.mtx bcsstk17.mtx Pres_Poisson.mtx gyro_m.mtx memplus.mtx lhr34.mtx bcsstk32.mtx vanbody.mtx s3dkt3m2.mtx G2_circuit.mtx Stanford.mtx coPapersCiteseer.mtx bundle_adj.mtx Stanford_Berkeley.mtx apache2.mtx Emilia_923.mtx ldoor.mtx ecology2.mtx Serena.mtx cage14.mtx G3_circuit.mtx wikipedia-20051105.mtx wikipedia-20061104.mtx Freescale1.mtx wikipedia-20070206.mtx Queen_4147.mtx cage15.mtx adaptive.mtx rgg_n_2_24_s0.mtx uk-2002.mtx road_usa.mtx MOLIERE_2016.mtx europe_osm.mtx twitter.mtx com-Friendster.mtx)
+
 #which command to use to run a GraphBLAS program
 LPF=yes
 if [ -z "${LPFRUN}" ]; then
@@ -277,6 +280,81 @@ runOtherBenchMarkTests()
 	echo >> ${TEST_OUT_DIR}/benchmarks
 }
 
+runMultiplicationKernels()
+{
+	local runner=$1
+	local backend=$2
+	local dataSet=$3
+	local parseMode=$4
+	local i=$5
+
+	# the check for the matrices existence is assumed to have already passed
+
+	if [ -z "$EXPTYPE" ] || [ "$EXPTYPE" == "SPMV" ]; then
+
+		# ---------------------------------------------------------------------
+		# spmv
+		echo ">>>      [ ]           [x]       Testing spmv using ${dataSet} dataset, $backend backend."
+		echo
+		$runner ${TEST_BIN_DIR}/driver_spmv_${backend} ${INPUT_DIR}/${dataSet} ${parseMode} &> ${TEST_OUT_DIR}/driver_spmv_${backend}_${dataSet}
+		head -1 ${TEST_OUT_DIR}/driver_spmv_${backend}_${dataSet}
+		if grep -q "Test OK" ${TEST_OUT_DIR}/driver_spmv_${backend}_${dataSet}; then
+			printf "Test OK\n\n"
+		else
+			printf "Test FAILED\n\n"
+		fi
+		echo "$backend spmv using the ${dataSet} dataset" >> ${TEST_OUT_DIR}/benchmarks
+		egrep 'Avg|Std' ${TEST_OUT_DIR}/driver_spmv_${backend}_${dataSet} >> ${TEST_OUT_DIR}/benchmarks
+		echo >> ${TEST_OUT_DIR}/benchmarks
+
+	fi
+
+	if [ -z "$EXPTYPE" ] || [ "$EXPTYPE" == "SPMSPV" ]; then
+
+		# ---------------------------------------------------------------------
+		# spmspv
+		echo ">>>      [ ]           [x]       Testing spmspv using ${dataSet} dataset, $backend backend."
+		echo
+		$runner ${TEST_BIN_DIR}/driver_spmspv_${backend} ${INPUT_DIR}/${dataSet} ${parseMode} &> ${TEST_OUT_DIR}/driver_spmspv_${backend}_${dataSet}
+		head -1 ${TEST_OUT_DIR}/driver_spmspv_${backend}_${dataSet}
+		if grep -q "Test OK" ${TEST_OUT_DIR}/driver_spmspv_${backend}_${dataSet}; then
+			printf "Test OK\n\n"
+		else
+			printf "Test FAILED\n\n"
+		fi
+		echo "$backend spmspv using the ${dataSet} dataset" >> ${TEST_OUT_DIR}/benchmarks
+		egrep 'Avg|Std' ${TEST_OUT_DIR}/driver_spmspv_${backend}_${dataSet} >> ${TEST_OUT_DIR}/benchmarks
+		echo >> ${TEST_OUT_DIR}/benchmarks
+
+	fi
+
+	if [ -z "$EXPTYPE" ] || [ "$EXPTYPE" == "SPMSPM" ]; then
+
+		# ---------------------------------------------------------------------
+		# spmspm
+		echo ">>>      [ ]           [x]       Testing spmspm using ${dataSet} dataset, $backend backend."
+		echo
+		if [ "$BACKEND" = "bsp1d" ] || [ "$BACKEND" = "hybrid" ]; then
+			echo "Test DISABLED: no sparse level-3 operations recommended for 1D distributions."
+			echo " "
+		elif [ "$i" -gt "14" ]; then
+			echo "Tests DISABLED: by default, long-running sparse matrix--sparse matrix multiplications are disabled (skipping dataset ${dataSet})."
+			echo " "
+		else
+			$runner ${TEST_BIN_DIR}/driver_spmspm_${backend} ${INPUT_DIR}/${dataSet} ${INPUT_DIR}/${dataSet} ${parseMode} &> ${TEST_OUT_DIR}/driver_spmspm_${backend}_${dataSet}
+			head -1 ${TEST_OUT_DIR}/driver_spmspm_${backend}_${dataSet}
+			if grep -q "Test OK" ${TEST_OUT_DIR}/driver_spmspm_${backend}_${dataSet}; then
+				printf "Test OK\n\n"
+			else
+				printf "Test FAILED\n\n"
+			fi
+			echo "$backend spmspm using the ${dataSet} dataset" >> ${TEST_OUT_DIR}/benchmarks
+			egrep 'Avg|Std' ${TEST_OUT_DIR}/driver_spmspm_${backend}_${dataSet} >> ${TEST_OUT_DIR}/benchmarks
+			echo >> ${TEST_OUT_DIR}/benchmarks
+		fi
+	fi
+}
+
 # end helper functions
 
 if [ -z "$EXPTYPE" ] || ! [ "$EXPTYPE" == "KERNEL" ]; then
@@ -360,7 +438,8 @@ if [ -z "$EXPTYPE" ] || ! [ "$EXPTYPE" == "KERNEL" ]; then
 
 			# test for file
 			if [ ! -f ${INPUT_DIR}/${DATASET} ]; then
-				echo "Warning: dataset/${DATASET} not found. Provide the dataset to enable performance tests with it."
+				echo ">>>      [x]           [x]       Test algorithms using ${DATASET} dataset, ${BACKEND} backend."
+				echo "Tests DISABLED: dataset/${DATASET} not found. Provide the dataset to enable performance tests with it."
 				echo " "
 				continue
 			fi
@@ -396,6 +475,35 @@ if [ -z "$EXPTYPE" ] || ! [ "$EXPTYPE" == "KERNEL" ]; then
 			fi
 		done
 
+		for ((i=0;i<${#MULTIPLICATION_DATASETS[@]};++i));
+		do
+			if [ ! -z "$DATASETTORUN" ] && [ "$DATASETTORUN" != "${MULTIPLICATION_DATASETS[i]}" ]; then
+				continue
+			fi
+
+			if [ "$BACKEND" = "hyperdags" ] && [ "$i" -gt "0" ]; then
+				echo "Info: hyperdags performance tests run only on the smallest dataset"
+				echo " "
+				break
+			fi
+
+			# initialise parameters
+			DATASET=${MULTIPLICATION_DATASETS[i]}
+			PARSE_MODE=direct
+			
+			# test for file
+			if [ ! -f ${INPUT_DIR}/${DATASET} ]; then
+				echo ">>>      [ ]           [x]       Test multiplication kernels using ${DATASET} dataset,"
+				echo "                                 ${BACKEND} backend."
+				echo "Tests DISABLED: dataset/${DATASET} not found. Provide the dataset to enable performance tests with it."
+				echo " "
+				continue
+			fi
+
+			runMultiplicationKernels "$runner" "$BACKEND" "$DATASET" "$PARSE_MODE" "$i"
+
+		done
+
 	done
 
 fi