Skip to content

ARROW-7760: [Release] Fix verify-release-candidate.sh since pip3 seems to no longer be in miniconda, install miniconda unconditionally #6348

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

ARROW-7760: [Release] Fix verify-release-candidate.sh since pip3 seems to no longer be in miniconda, install miniconda unconditionally #6348

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
490ceac
pip3 no longer in miniconda
wesm Feb 3, 2020
26da759
always set up miniconda
kszucs Feb 4, 2020
fcf3ae6
conda environment must be activated for other steps to work if C++ is…
wesm Feb 4, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions dev/archery/archery/integration/runner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,23 +227,23 @@ def get_static_json_files():
]


def run_all_tests(enable_cpp=True, enable_java=True, enable_js=True,
enable_go=True, run_flight=False,
def run_all_tests(with_cpp=True, with_java=True, with_js=True,
with_go=True, run_flight=False,
tempdir=None, **kwargs):
tempdir = tempdir or tempfile.mkdtemp()

testers = []

if enable_cpp:
if with_cpp:
testers.append(CPPTester(kwargs))

if enable_java:
if with_java:
testers.append(JavaTester(kwargs))

if enable_js:
if with_js:
testers.append(JSTester(kwargs))

if enable_go:
if with_go:
testers.append(GoTester(kwargs))

static_json_files = get_static_json_files()
Expand Down
44 changes: 26 additions & 18 deletions dev/release/verify-release-candidate.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@
# LD_LIBRARY_PATH.
#
# To reuse build artifacts between runs set TMPDIR environment variable to
# a directory where the temporary files should be placed to.
# a directory where the temporary files should be placed to, note that this
# directory is not cleaned up automatically.

case $# in
3) ARTIFACT="$1"
Expand Down Expand Up @@ -126,7 +127,7 @@ test_binary() {
local download_dir=binaries
mkdir -p ${download_dir}

python3 $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER --dest=${download_dir}
python $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER --dest=${download_dir}
verify_dir_artifact_signatures ${download_dir}
}

Expand Down Expand Up @@ -207,11 +208,13 @@ setup_tempdir() {
echo "Failed to verify release candidate. See ${TMPDIR} for details."
fi
}
trap cleanup EXIT

if [ -z "${TMPDIR}" ]; then
# clean up automatically if TMPDIR is not defined
TMPDIR=$(mktemp -d -t "$1.XXXXX")
trap cleanup EXIT
else
# don't clean up automatically
mkdir -p "${TMPDIR}"
fi
}
Expand All @@ -235,6 +238,15 @@ setup_miniconda() {
fi

. $MINICONDA/etc/profile.d/conda.sh

conda create -n arrow-test -y -q -c conda-forge \
python=3.6 \
nomkl \
numpy \
pandas \
six \
cython
conda activate arrow-test
}

# Build and test Java (Requires newer Maven -- I used 3.3.9)
Expand All @@ -251,15 +263,6 @@ test_package_java() {
# Build and test C++

test_and_install_cpp() {
conda create -n arrow-test -y -q -c conda-forge \
python=3.6 \
nomkl \
numpy \
pandas \
six \
cython
conda activate arrow-test

mkdir -p cpp/build
pushd cpp/build

Expand Down Expand Up @@ -503,7 +506,7 @@ test_integration() {
export ARROW_JAVA_INTEGRATION_JAR=$JAVA_DIR/tools/target/arrow-tools-$VERSION-jar-with-dependencies.jar
export ARROW_CPP_EXE_PATH=$CPP_BUILD_DIR/release

pip3 install -e dev/archery
pip install -e dev/archery
Copy link
Member

@pitrou pitrou Feb 3, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you are sure you're using Python 3, I would say python3 -m pip install ... is the surest way.

Copy link
Member Author

@wesm wesm Feb 4, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This step is invoked from within a conda environment that is set up in the script. I guess recently (January 2020-related?) the pip3 alias disappeared

Copy link
Member

@kszucs kszucs Feb 4, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The conda environment wouldn't have been installed in certain flag configuration:

TEST_DEFAULT=0 TEST_INTEGRATION_GO=1 dev/release/verify-release-candidate.sh source 0.16.0 2

Copy link
Member

@kszucs kszucs Feb 4, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Changed the script to always use miniconda.


INTEGRATION_TEST_ARGS=""

Expand Down Expand Up @@ -552,7 +555,6 @@ test_source_distribution() {
test_package_java
fi
if [ ${TEST_CPP} -gt 0 ]; then
setup_miniconda
test_and_install_cpp
fi
if [ ${TEST_CSHARP} -gt 0 ]; then
Expand Down Expand Up @@ -686,9 +688,9 @@ test_wheels() {
conda create -yq -n py3-base python=3.7
conda activate py3-base

python3 $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER \
--regex=${filter_regex} \
--dest=${download_dir}
python $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER \
--regex=${filter_regex} \
--dest=${download_dir}

verify_dir_artifact_signatures ${download_dir}

Expand Down Expand Up @@ -751,6 +753,9 @@ setup_tempdir "arrow-${VERSION}"
echo "Working in sandbox ${TMPDIR}"
cd ${TMPDIR}

setup_miniconda
echo "Using miniconda environment ${MINICONDA}"

if [ "${ARTIFACT}" == "source" ]; then
dist_name="apache-arrow-${VERSION}"
if [ ${TEST_SOURCE} -gt 0 ]; then
Expand All @@ -759,14 +764,17 @@ if [ "${ARTIFACT}" == "source" ]; then
tar xf ${dist_name}.tar.gz
else
mkdir -p ${dist_name}
if [ ! -f ${TEST_ARCHIVE} ]; then
echo "${TEST_ARCHIVE} not found, did you mean to pass TEST_SOURCE=1?"
exit 1
fi
tar xf ${TEST_ARCHIVE} -C ${dist_name} --strip-components=1
fi
pushd ${dist_name}
test_source_distribution
popd
elif [ "${ARTIFACT}" == "wheels" ]; then
import_gpg_keys
setup_miniconda
test_wheels
else
import_gpg_keys
Expand Down