vector-db-benchmark/tools/run_remote_benchmark.sh at master · qdrant/vector-db-benchmark · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
#!/bin/bash

PS4='ts=$(date "+%Y-%m-%dT%H:%M:%SZ") level=DEBUG line=$LINENO file=$BASH_SOURCE '
set -euo pipefail

# Setup 2 machines in Hetzner Cloud
# One machine will be used as a server, another one as a client

cleanup() {
  echo "cleaning up file=$BASH_SOURCE"
  #  bash -x "${SCRIPT_PATH}/tear_down.sh"
}

trap 'echo signal received!; kill $(jobs -p); wait; cleanup' SIGINT SIGTERM

CLOUD_NAME=${CLOUD_NAME:-"hetzner"}

SCRIPT=$(realpath "$0")
SCRIPT_PATH=$(dirname "$SCRIPT")


BENCH_SERVER_NAME=${SERVER_NAME:-"benchmark-server-1"}
BENCH_CLIENT_NAME=${CLIENT_NAME:-"benchmark-client-1"}

trap 'cleanup' EXIT

# Uncomment this to dynamically create servers

#SERVER_NAME=$BENCH_SERVER_NAME SERVER_TYPE='ccx13' bash -x "${SCRIPT_PATH}/${CLOUD_NAME}/create_and_install.sh" &
#SERVER_CREATION_PID=$!
#SERVER_NAME=$BENCH_CLIENT_NAME SERVER_TYPE='cpx11' bash -x "${SCRIPT_PATH}/${CLOUD_NAME}/create_and_install.sh"
#wait $SERVER_CREATION_PID

BENCHMARK_STRATEGY=${BENCHMARK_STRATEGY:-"default"}

SERVER_NAME=$BENCH_SERVER_NAME bash -x "${SCRIPT_PATH}/${CLOUD_NAME}/check_ssh_connection.sh"
SERVER_NAME=$BENCH_CLIENT_NAME bash -x "${SCRIPT_PATH}/${CLOUD_NAME}/check_ssh_connection.sh"

case "$BENCHMARK_STRATEGY" in
  "default")
  echo "Default benchmark, no volume, no memory limit"

  SERVER_CONTAINER_NAME=${SERVER_CONTAINER_NAME:-"qdrant-continuous-benchmarks"}

  bash -x "${SCRIPT_PATH}/run_server_container.sh" "$SERVER_CONTAINER_NAME"

  bash -x "${SCRIPT_PATH}/qdrant_collect_cpu_usage.sh" "start"

  bash -x "${SCRIPT_PATH}/run_client_script.sh"

  bash -x "${SCRIPT_PATH}/qdrant_collect_cpu_usage.sh" "end"

  bash -x "${SCRIPT_PATH}/qdrant_collect_stats.sh" "$SERVER_CONTAINER_NAME"

  bash -x "${SCRIPT_PATH}/qdrant_collect_storage_size.sh" "$SERVER_CONTAINER_NAME"
  ;;
  "tenants")
  if [[ -z "${CONTAINER_MEM_LIMIT:-}" ]]; then
    echo "Tenants benchmark, but CONTAINER_MEM_LIMIT is not set!"
    exit 2
  fi

  echo "Tenants benchmark, run search with memory limit: ${CONTAINER_MEM_LIMIT}"

  SERVER_CONTAINER_NAME=${SERVER_CONTAINER_NAME:-"qdrant-continuous-benchmarks-with-volume"}

  bash -x "${SCRIPT_PATH}/run_server_container_with_volume.sh" "$SERVER_CONTAINER_NAME" "25Gb"

  bash -x "${SCRIPT_PATH}/run_client_script.sh" "upload"

  bash -x "${SCRIPT_PATH}/run_server_container_with_volume.sh" "$SERVER_CONTAINER_NAME" "$CONTAINER_MEM_LIMIT" "continue"

  bash -x "${SCRIPT_PATH}/run_client_script.sh" "search"

  bash -x "${SCRIPT_PATH}/qdrant_collect_stats.sh" "$SERVER_CONTAINER_NAME"
  ;;

  "collection-reload")
  echo "Collection load time benchmark"

  SERVER_CONTAINER_NAME=${SERVER_CONTAINER_NAME:-"qdrant-continuous-benchmarks-snapshot"}

  bash -x "${SCRIPT_PATH}/run_server_container_with_volume.sh" "$SERVER_CONTAINER_NAME" "25Gb"

  bash -x "${SCRIPT_PATH}/run_client_script.sh" "snapshot"

  bash -x "${SCRIPT_PATH}/run_server_container_with_volume.sh" "$SERVER_CONTAINER_NAME" "25Gb" "continue"

  sleep 10

  bash -x "${SCRIPT_PATH}/qdrant_collect_stats.sh" "$SERVER_CONTAINER_NAME"
  ;;

  "parallel")
  echo "Parallel benchmark, run upload&search at the same time"

  SERVER_CONTAINER_NAME=${SERVER_CONTAINER_NAME:-"qdrant-continuous-benchmarks-with-volume"}

  bash -x "${SCRIPT_PATH}/run_server_container_with_volume.sh" "$SERVER_CONTAINER_NAME" "25Gb"

  bash -x "${SCRIPT_PATH}/run_client_script.sh" "upload"

  bash -x "${SCRIPT_PATH}/run_server_container_with_volume.sh" "$SERVER_CONTAINER_NAME" "25Gb" "continue"

  bash -x "${SCRIPT_PATH}/run_client_script.sh" "search"

  bash -x "${SCRIPT_PATH}/run_server_container_with_volume.sh" "$SERVER_CONTAINER_NAME" "25Gb" "continue"

  bash -x "${SCRIPT_PATH}/run_client_script.sh" "parallel"

  bash -x "${SCRIPT_PATH}/qdrant_collect_stats.sh" "$SERVER_CONTAINER_NAME"
  ;;

  *)
    echo "Invalid BENCHMARK_STRATEGY value: $BENCHMARK_STRATEGY"
    exit 1
    ;;
esac