From 693c242a766289a2a5a292bbc7a561a6142fbd7a Mon Sep 17 00:00:00 2001
From: Shaun Crampton <shaun@tigera.io>
Date: Mon, 21 Jun 2021 16:57:12 +0100
Subject: [PATCH 1/2] Run all FV batches from one Semaphore VM.

---
 .semaphore/collect-artifacts-from-vms | 37 ++++++++++++
 .semaphore/create-test-vms            | 46 +++++++++++++++
 .semaphore/run-tests-on-vms           | 84 +++++++++++++++++++++++++++
 .semaphore/semaphore.yml              | 23 +++-----
 4 files changed, 175 insertions(+), 15 deletions(-)
 create mode 100755 .semaphore/collect-artifacts-from-vms
 create mode 100755 .semaphore/create-test-vms
 create mode 100755 .semaphore/run-tests-on-vms

diff --git a/.semaphore/collect-artifacts-from-vms b/.semaphore/collect-artifacts-from-vms
new file mode 100755
index 0000000000..86871dbf7f
--- /dev/null
+++ b/.semaphore/collect-artifacts-from-vms
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2019 Tigera, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+vm_name_prefix=$1
+project=unique-caldron-775
+zone=${ZONE:-europe-west3-c}
+my_dir="$(dirname $0)"
+repo_dir="."
+artifacts_dir="$repo_dir/artifacts"
+
+echo "Collecting artifacts..."
+
+num_fv_batches=${NUM_FV_BATCHES:-8}
+batches=(ut $(seq 1 ${num_fv_batches}))
+pids=()
+monitor_pids=()
+for batch in "${batches[@]}"; do
+  vm_name="$vm_name_prefix$batch"
+  echo "Collecting artifacts from $vm_name..."
+  VM_NAME=$vm_name $my_dir/on-test-vm ${REPO_NAME}/.semaphore/collect-artifacts || echo "Failed to collect artifacts from ${vm_name}."
+  gcloud --quiet compute scp "--zone=${ZONE}" "ubuntu@${vm_name}:${REPO_NAME}/artifacts" ./ --recurse || echo "Failed to SCP from ${vm_name}"
+done
diff --git a/.semaphore/create-test-vms b/.semaphore/create-test-vms
new file mode 100755
index 0000000000..24497e8563
--- /dev/null
+++ b/.semaphore/create-test-vms
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2019 Tigera, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+vm_name_prefix=$1
+project=unique-caldron-775
+zone=${ZONE:-europe-west3-c}
+my_dir="$(dirname $0)"
+repo_dir="."
+artifacts_dir="$repo_dir/artifacts"
+
+num_fv_batches=${NUM_FV_BATCHES:-8}
+batches=(ut $(seq 1 ${num_fv_batches}))
+pids=()
+for batch in "${batches[@]}"; do
+  vm_name="$vm_name_prefix$batch"
+  log_file="$artifacts_dir/create-vm-$batch.log"
+  echo "Creating test VM $vm_name in background. Redirecting log to $log_file."
+  "${my_dir}/create-test-vm" "$vm_name" >& "$log_file" &
+  pids+=( $! )
+  sleep 1
+done
+
+for pid in "${pids[@]}"; do
+  echo "Waiting for create-test-vm process PID=$pid to finish"
+  if ! wait "$pid"; then
+    echo "Creating one of the test VMs failed, exiting. Logs should be attached as artifacts."
+    exit 1
+  fi
+done
+
+echo "All test VMs started."
diff --git a/.semaphore/run-tests-on-vms b/.semaphore/run-tests-on-vms
new file mode 100755
index 0000000000..230a88860b
--- /dev/null
+++ b/.semaphore/run-tests-on-vms
@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2019 Tigera, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+vm_name_prefix=$1
+project=unique-caldron-775
+zone=${ZONE:-europe-west3-c}
+num_fv_batches=${NUM_FV_BATCHES:-8}
+batches=(ut $(seq 1 ${num_fv_batches}))
+my_dir="$(dirname $0)"
+repo_dir="."
+artifacts_dir="$repo_dir/artifacts"
+
+echo "Starting tests..."
+
+pids=()
+monitor_pids=()
+for batch in "${batches[@]}"; do
+  vm_name="$vm_name_prefix$batch"
+  log_file="$artifacts_dir/test-$batch.log"
+  if [ $batch = "ut" ]; then
+    echo "Starting UTs"
+    VM_NAME=$vm_name $my_dir/on-test-vm make --directory=${REPO_NAME} ut-bpf check-wireguard >& "$log_file" &
+    pid=$!
+    pids+=( $pid )
+    echo "UT PID: $pid"
+  else
+    echo "Starting FV batch $batch"
+    VM_NAME=$vm_name ./.semaphore/on-test-vm make --directory=${REPO_NAME} fv-bpf GINKGO_FOCUS=BPF-SAFE FV_NUM_BATCHES=8 FV_BATCHES_TO_RUN="$batch" >& "$log_file" &
+    pid=$!
+    pids+=( $pid )
+    echo "FV batch PID: $pid"
+  fi
+
+  prefix="[batch=${batch} pid=${pid}]"
+  echo "$prefix Monitoring $log_file in background for Failure/Success messages..."
+  (
+    tail -f $log_file | grep --line-buffered \
+      -e "Failure" \
+      -e "SUCCESS" \
+      -e "Parallel test node" \
+      -e "Test batch" \
+      -C10 | sed 's/.*/'"${prefix}"' &/';
+  ) &
+  mon_pid=$!
+  monitor_pids+=( $mon_pid )
+done
+
+final_result=0
+
+echo "Waiting for background test runners to finish..."
+for pid in "${pids[@]}"; do
+  echo "Waiting for test process batch PID=$pid to finish"
+  wait "$pid"
+  rc=$?
+  if [ $rc = 0 ]; then
+    echo "Test batch succeeded PID=$pid."
+  else
+    echo "Test batch failed PID=$pid."
+    final_result=1
+  fi
+done
+
+echo "Shutting down test monitors..."
+for pid in "${monitor_pids[@]}"; do
+  kill $pid
+done
+echo "Done, exiting with RC=$final_result"
+
+exit $final_result
diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml
index 0698bc5f98..d28edc43d4 100644
--- a/.semaphore/semaphore.yml
+++ b/.semaphore/semaphore.yml
@@ -111,31 +111,24 @@ blocks:
       - export GOOGLE_APPLICATION_CREDENTIALS=$HOME/secrets/secret.google-service-account-key.json
       - export SHORT_WORKFLOW_ID=$(echo ${SEMAPHORE_WORKFLOW_ID} | sha256sum | cut -c -8)
       - export ZONE=europe-west3-c
-      - export VM_NAME=sem-${SEMAPHORE_PROJECT_NAME}-${SHORT_WORKFLOW_ID}-${SEMAPHORE_JOB_INDEX:-ut}
-      - echo VM_NAME=${VM_NAME}
+      - export VM_PREFIX=sem-${SEMAPHORE_PROJECT_NAME}-${SHORT_WORKFLOW_ID}-
+      - echo VM_PREFIX=${VM_PREFIX}
       - export REPO_NAME=$(basename $(pwd))
+      - export NUM_FV_BATCHES=8
       - mkdir artifacts
-      - ./.semaphore/create-test-vm ${VM_NAME}
+      - ./.semaphore/create-test-vms ${VM_PREFIX}
     jobs:
-    - name: UT and Wireguard non-BPF FV on newer kernel
+    - name: UT/FV tests on new kernel
       execution_time_limit:
         minutes: 120
       commands:
-      - ./.semaphore/on-test-vm make --directory=${REPO_NAME} ut-bpf
-      - ./.semaphore/on-test-vm make --directory=${REPO_NAME} check-wireguard
-    - name: FV on newer kernel
-      execution_time_limit:
-        minutes: 120
-      commands:
-      - ./.semaphore/on-test-vm make --directory=${REPO_NAME} fv-bpf GINKGO_FOCUS=BPF-SAFE FV_NUM_BATCHES=${SEMAPHORE_JOB_COUNT} FV_BATCHES_TO_RUN="${SEMAPHORE_JOB_INDEX}"
-      parallelism: 8
+      - ./.semaphore/run-tests-on-vms ${VM_PREFIX}
     epilogue:
       always:
         commands:
-        - ./.semaphore/on-test-vm ${REPO_NAME}/.semaphore/collect-artifacts
-        - gcloud --quiet compute scp "--zone=${ZONE}" "ubuntu@${VM_NAME}:${REPO_NAME}/artifacts" ./ --recurse
+        - ./.semaphore/collect-artifacts-from-vms ${VM_PREFIX}
         - ./.semaphore/publish-artifacts
-        - gcloud --quiet compute instances delete ${VM_NAME} --zone=${ZONE}
+        - ./.semaphore/clean-up-vms ${VM_PREFIX}
     secrets:
     - name: google-service-account-for-gce
 - name: Static checks on e1-standard-8

From e56d38061be8cfebd57a552c31cafa6ffe788c69 Mon Sep 17 00:00:00 2001
From: Shaun Crampton <shaun@tigera.io>
Date: Mon, 21 Jun 2021 19:18:30 +0100
Subject: [PATCH 2/2] Fix that wait would terminate the script early.

---
 .semaphore/run-tests-on-vms | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.semaphore/run-tests-on-vms b/.semaphore/run-tests-on-vms
index 230a88860b..9352314eb3 100755
--- a/.semaphore/run-tests-on-vms
+++ b/.semaphore/run-tests-on-vms
@@ -65,9 +65,7 @@ final_result=0
 echo "Waiting for background test runners to finish..."
 for pid in "${pids[@]}"; do
   echo "Waiting for test process batch PID=$pid to finish"
-  wait "$pid"
-  rc=$?
-  if [ $rc = 0 ]; then
+  if wait "$pid"; then
     echo "Test batch succeeded PID=$pid."
   else
     echo "Test batch failed PID=$pid."
@@ -77,7 +75,7 @@ done
 
 echo "Shutting down test monitors..."
 for pid in "${monitor_pids[@]}"; do
-  kill $pid
+  kill $pid || true
 done
 echo "Done, exiting with RC=$final_result"