Skip to content

Commit 2a87421

Browse files
committed
Fix SPARK_VERSION check as suggested on review
1 parent 72089d5 commit 2a87421

File tree

1 file changed

+9
-12
lines changed

1 file changed

+9
-12
lines changed

ci/scripts/integration_spark.sh

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ spark_version=${SPARK_VERSION:-master}
3030
# Use old behavior that always dropped tiemzones.
3131
export PYARROW_IGNORE_TIMEZONE=1
3232

33-
if [ "${SPARK_VERSION:0:2}" == "2." ]; then
33+
if [ "${SPARK_VERSION:1:2}" == "2." ]; then
3434
# https://github.com/apache/spark/blob/master/docs/sql-pyspark-pandas-with-arrow.md#compatibility-setting-for-pyarrow--0150-and-spark-23x-24x
3535
export ARROW_PRE_0_15_IPC_FORMAT=1
3636
fi
@@ -75,17 +75,14 @@ pushd ${spark_dir}
7575
spark_python_tests=(
7676
"pyspark.sql.tests.test_arrow")
7777

78-
old_test_modules=false
79-
# Spark <= 2.x or <= 3.3.x
80-
if [ "${SPARK_VERSION:0:2}" == "2." ]; then
81-
old_test_modules=true
82-
elif [ "${SPARK_VERSION:0:2}" == "3." ]; then
83-
if [ "${SPARK_VERSION:2:2}" == "1." ] || \
84-
[ "${SPARK_VERSION:2:2}" == "2." ] || \
85-
[ "${SPARK_VERSION:2:2}" == "3." ]; then
86-
old_test_modules=true
87-
fi
88-
fi
78+
case "${SPARK_VERSION}" in
79+
v1.*|v2.*|v3.0.*|v3.1.*|v3.2.*|v3.3.*)
80+
old_test_modules=true
81+
;;
82+
*)
83+
old_test_modules=false
84+
;;
85+
esac
8986
if [ "${old_test_modules}" == "true" ]; then
9087
spark_python_tests+=(
9188
"pyspark.sql.tests.test_pandas_grouped_map"

0 commit comments

Comments
 (0)