Skip to content

Commit

Permalink
[opt](ci) start dockers in parallel (apache#41338)
Browse files Browse the repository at this point in the history
Start docker in parallel to reduce external pipeline time
  • Loading branch information
suxiaogang223 committed Oct 16, 2024
1 parent 34ef067 commit fa24bf2
Show file tree
Hide file tree
Showing 2 changed files with 206 additions and 41 deletions.
38 changes: 30 additions & 8 deletions docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,15 @@ sleep 10s
# new cases should use separate dir
hadoop fs -mkdir -p /user/doris/suites/

lockfile1 = "mnt/scripts/run-data.lock"

# wait lockfile
while [ -f "$lockfile1" ]; do
sleep 10
done

touch "$lockfile1"

DATA_DIR="/mnt/scripts/data/"
find "${DATA_DIR}" -type f -name "run.sh" -print0 | xargs -0 -n 1 -P 10 -I {} sh -c '
START_TIME=$(date +%s)
Expand All @@ -36,6 +45,17 @@ find "${DATA_DIR}" -type f -name "run.sh" -print0 | xargs -0 -n 1 -P 10 -I {} sh
echo "Script: {} executed in $EXECUTION_TIME seconds"
'

rm -f "$lockfile1"

lockfile2 = "mnt/scripts/download-data.lock"

# wait lockfile
while [ -f "$lockfile2" ]; do
sleep 10
done

touch "$lockfile2"

# if you test in your local,better use # to annotation section about tpch1.db
if [[ ! -d "/mnt/scripts/tpch1.db" ]]; then
echo "/mnt/scripts/tpch1.db does not exist"
Expand All @@ -48,11 +68,6 @@ else
echo "/mnt/scripts/tpch1.db exist, continue !"
fi

# put data file
## put tpch1
hadoop fs -mkdir -p /user/doris/
hadoop fs -put /mnt/scripts/tpch1.db /user/doris/

# paimon data file is small and update frequently, so we download it every time
rm -rf "/mnt/scripts/paimon1"
echo "/mnt/scripts/paimon1 does not exist"
Expand All @@ -62,9 +77,6 @@ tar -zxf paimon1.tar.gz
rm -rf paimon1.tar.gz
cd -

## put paimon1
hadoop fs -put /mnt/scripts/paimon1 /user/doris/

# download tvf_data
if [[ ! -d "/mnt/scripts/tvf_data" ]]; then
echo "/mnt/scripts/tvf_data does not exist"
Expand All @@ -77,6 +89,16 @@ else
echo "/mnt/scripts/tvf_data exist, continue !"
fi

rm -f "$lockfile2"

# put data file
## put tpch1
hadoop fs -mkdir -p /user/doris/
hadoop fs -put /mnt/scripts/tpch1.db /user/doris/

## put paimon1
hadoop fs -put /mnt/scripts/paimon1 /user/doris/

## put tvf_data
hadoop fs -put /mnt/scripts/tvf_data /user/doris/

Expand Down
Loading

0 comments on commit fa24bf2

Please sign in to comment.