From 6ae2eed39275713343c08562fc03de8bf66efe28 Mon Sep 17 00:00:00 2001 From: Dennis Huo Date: Tue, 30 Apr 2024 02:18:36 +0000 Subject: [PATCH] Add handy message for how to diff manually instead of using meld since (#18) it's hard to install meld on mac. Also, wget may be missing on mac so auto-install that during automatic spark-download. Fix the first-regtest-setup case by updating SPARK_HOME after download. --- regtests/run.sh | 3 ++- regtests/setup.sh | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/regtests/run.sh b/regtests/run.sh index 4672fa416..64d5fddde 100755 --- a/regtests/run.sh +++ b/regtests/run.sh @@ -58,7 +58,8 @@ for TEST_FILE in ${TEST_LIST}; do echo '#!/bin/bash' > ${TEST_TMPDIR}/${TEST_SHORTNAME}.fixdiffs.sh echo "meld ${TEST_STDOUT} ${TEST_REF}" >> ${TEST_TMPDIR}/${TEST_SHORTNAME}.fixdiffs.sh chmod 750 ${TEST_TMPDIR}/${TEST_SHORTNAME}.fixdiffs.sh - logred "To compare and fix diffs: ${TEST_TMPDIR}/${TEST_SHORTNAME}.fixdiffs.sh" + logred "To compare and fix diffs (if 'meld' installed): ${TEST_TMPDIR}/${TEST_SHORTNAME}.fixdiffs.sh" + logred "Or manually diff: diff ${TEST_STDOUT} ${TEST_REF}" logred "See stderr from test run for additional diagnostics: ${TEST_STDERR}" NUM_FAILURES=$(( NUM_FAILURES + 1 )) fi diff --git a/regtests/setup.sh b/regtests/setup.sh index f000fce7c..6b35143f8 100755 --- a/regtests/setup.sh +++ b/regtests/setup.sh @@ -10,6 +10,7 @@ set -x SPARK_HOME=$(realpath ~/spark-3.5.1-bin-hadoop3-scala2.13) SPARK_CONF="${SPARK_HOME}/conf/spark-defaults.conf" + # Ensure binaries are downloaded locally echo 'Verifying Spark binaries...' if ! [ -f ${SPARK_HOME}/bin/spark-sql ]; then @@ -17,11 +18,20 @@ if ! [ -f ${SPARK_HOME}/bin/spark-sql ]; then if ! [ -f ~/spark-3.5.1-bin-hadoop3-scala2.13.tgz ]; then echo 'Downloading spark distro...' wget -O ~/spark-3.5.1-bin-hadoop3-scala2.13.tgz https://dlcdn.apache.org/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3-scala2.13.tgz + if ! [ -f ~/spark-3.5.1-bin-hadoop3-scala2.13.tgz ]; then + if [[ "${OSTYPE}" == "darwin"* ]]; then + echo "Detected OS: mac. Running 'brew install wget' to try again." + brew install wget + wget -O ~/spark-3.5.1-bin-hadoop3-scala2.13.tgz https://dlcdn.apache.org/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3-scala2.13.tgz + fi + fi else echo 'Found existing Spark tarball' fi tar xzvf ~/spark-3.5.1-bin-hadoop3-scala2.13.tgz -C ~ echo 'Done!' + SPARK_HOME=$(realpath ~/spark-3.5.1-bin-hadoop3-scala2.13) + SPARK_CONF="${SPARK_HOME}/conf/spark-defaults.conf" else echo 'Verified Spark distro already installed.' fi