Merge pull request #2856 from fasaxc/combine-bpf-fvs

Combine bpf fvs
projectcalico · Jun 21, 2021 · 169d9f3 · 169d9f3
2 parents e698ad1 + e56d380
commit 169d9f3
Show file tree

Hide file tree

Showing 4 changed files with 173 additions and 15 deletions.
diff --git a/.semaphore/collect-artifacts-from-vms b/.semaphore/collect-artifacts-from-vms
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2019 Tigera, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+vm_name_prefix=$1
+project=unique-caldron-775
+zone=${ZONE:-europe-west3-c}
+my_dir="$(dirname $0)"
+repo_dir="."
+artifacts_dir="$repo_dir/artifacts"
+
+echo "Collecting artifacts..."
+
+num_fv_batches=${NUM_FV_BATCHES:-8}
+batches=(ut $(seq 1 ${num_fv_batches}))
+pids=()
+monitor_pids=()
+for batch in "${batches[@]}"; do
+  vm_name="$vm_name_prefix$batch"
+  echo "Collecting artifacts from $vm_name..."
+  VM_NAME=$vm_name $my_dir/on-test-vm ${REPO_NAME}/.semaphore/collect-artifacts || echo "Failed to collect artifacts from ${vm_name}."
+  gcloud --quiet compute scp "--zone=${ZONE}" "ubuntu@${vm_name}:${REPO_NAME}/artifacts" ./ --recurse || echo "Failed to SCP from ${vm_name}"
+done
diff --git a/.semaphore/create-test-vms b/.semaphore/create-test-vms
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2019 Tigera, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+vm_name_prefix=$1
+project=unique-caldron-775
+zone=${ZONE:-europe-west3-c}
+my_dir="$(dirname $0)"
+repo_dir="."
+artifacts_dir="$repo_dir/artifacts"
+
+num_fv_batches=${NUM_FV_BATCHES:-8}
+batches=(ut $(seq 1 ${num_fv_batches}))
+pids=()
+for batch in "${batches[@]}"; do
+  vm_name="$vm_name_prefix$batch"
+  log_file="$artifacts_dir/create-vm-$batch.log"
+  echo "Creating test VM $vm_name in background. Redirecting log to $log_file."
+  "${my_dir}/create-test-vm" "$vm_name" >& "$log_file" &
+  pids+=( $! )
+  sleep 1
+done
+
+for pid in "${pids[@]}"; do
+  echo "Waiting for create-test-vm process PID=$pid to finish"
+  if ! wait "$pid"; then
+    echo "Creating one of the test VMs failed, exiting. Logs should be attached as artifacts."
+    exit 1
+  fi
+done
+
+echo "All test VMs started."
diff --git a/.semaphore/run-tests-on-vms b/.semaphore/run-tests-on-vms
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2019 Tigera, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+vm_name_prefix=$1
+project=unique-caldron-775
+zone=${ZONE:-europe-west3-c}
+num_fv_batches=${NUM_FV_BATCHES:-8}
+batches=(ut $(seq 1 ${num_fv_batches}))
+my_dir="$(dirname $0)"
+repo_dir="."
+artifacts_dir="$repo_dir/artifacts"
+
+echo "Starting tests..."
+
+pids=()
+monitor_pids=()
+for batch in "${batches[@]}"; do
+  vm_name="$vm_name_prefix$batch"
+  log_file="$artifacts_dir/test-$batch.log"
+  if [ $batch = "ut" ]; then
+    echo "Starting UTs"
+    VM_NAME=$vm_name $my_dir/on-test-vm make --directory=${REPO_NAME} ut-bpf check-wireguard >& "$log_file" &
+    pid=$!
+    pids+=( $pid )
+    echo "UT PID: $pid"
+  else
+    echo "Starting FV batch $batch"
+    VM_NAME=$vm_name ./.semaphore/on-test-vm make --directory=${REPO_NAME} fv-bpf GINKGO_FOCUS=BPF-SAFE FV_NUM_BATCHES=8 FV_BATCHES_TO_RUN="$batch" >& "$log_file" &
+    pid=$!
+    pids+=( $pid )
+    echo "FV batch PID: $pid"
+  fi
+
+  prefix="[batch=${batch} pid=${pid}]"
+  echo "$prefix Monitoring $log_file in background for Failure/Success messages..."
+  (
+    tail -f $log_file | grep --line-buffered \
+      -e "Failure" \
+      -e "SUCCESS" \
+      -e "Parallel test node" \
+      -e "Test batch" \
+      -C10 | sed 's/.*/'"${prefix}"' &/';
+  ) &
+  mon_pid=$!
+  monitor_pids+=( $mon_pid )
+done
+
+final_result=0
+
+echo "Waiting for background test runners to finish..."
+for pid in "${pids[@]}"; do
+  echo "Waiting for test process batch PID=$pid to finish"
+  if wait "$pid"; then
+    echo "Test batch succeeded PID=$pid."
+  else
+    echo "Test batch failed PID=$pid."
+    final_result=1
+  fi
+done
+
+echo "Shutting down test monitors..."
+for pid in "${monitor_pids[@]}"; do
+  kill $pid || true
+done
+echo "Done, exiting with RC=$final_result"
+
+exit $final_result
diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
@@ -111,31 +111,24 @@ blocks:
       - export GOOGLE_APPLICATION_CREDENTIALS=$HOME/secrets/secret.google-service-account-key.json
       - export SHORT_WORKFLOW_ID=$(echo ${SEMAPHORE_WORKFLOW_ID} | sha256sum | cut -c -8)
       - export ZONE=europe-west3-c
-      - export VM_NAME=sem-${SEMAPHORE_PROJECT_NAME}-${SHORT_WORKFLOW_ID}-${SEMAPHORE_JOB_INDEX:-ut}
-      - echo VM_NAME=${VM_NAME}
+      - export VM_PREFIX=sem-${SEMAPHORE_PROJECT_NAME}-${SHORT_WORKFLOW_ID}-
+      - echo VM_PREFIX=${VM_PREFIX}
       - export REPO_NAME=$(basename $(pwd))
+      - export NUM_FV_BATCHES=8
       - mkdir artifacts
-      - ./.semaphore/create-test-vm ${VM_NAME}
+      - ./.semaphore/create-test-vms ${VM_PREFIX}
     jobs:
-    - name: UT and Wireguard non-BPF FV on newer kernel
+    - name: UT/FV tests on new kernel
       execution_time_limit:
         minutes: 120
       commands:
-      - ./.semaphore/on-test-vm make --directory=${REPO_NAME} ut-bpf
-      - ./.semaphore/on-test-vm make --directory=${REPO_NAME} check-wireguard
-    - name: FV on newer kernel
-      execution_time_limit:
-        minutes: 120
-      commands:
-      - ./.semaphore/on-test-vm make --directory=${REPO_NAME} fv-bpf GINKGO_FOCUS=BPF-SAFE FV_NUM_BATCHES=${SEMAPHORE_JOB_COUNT} FV_BATCHES_TO_RUN="${SEMAPHORE_JOB_INDEX}"
-      parallelism: 8
+      - ./.semaphore/run-tests-on-vms ${VM_PREFIX}
     epilogue:
       always:
         commands:
-        - ./.semaphore/on-test-vm ${REPO_NAME}/.semaphore/collect-artifacts
-        - gcloud --quiet compute scp "--zone=${ZONE}" "ubuntu@${VM_NAME}:${REPO_NAME}/artifacts" ./ --recurse
+        - ./.semaphore/collect-artifacts-from-vms ${VM_PREFIX}
         - ./.semaphore/publish-artifacts
-        - gcloud --quiet compute instances delete ${VM_NAME} --zone=${ZONE}
+        - ./.semaphore/clean-up-vms ${VM_PREFIX}
     secrets:
     - name: google-service-account-for-gce
 - name: Static checks on e1-standard-8