Skip to content

Commit 74e59ec

Browse files
NickfostNickfost
committed
fix(installer): propagate staging failures
1 parent 50f0e4f commit 74e59ec

2 files changed

Lines changed: 51 additions & 7 deletions

File tree

‎scripts/install-worker-controller.sh‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -958,7 +958,7 @@ install_release() {
958958
install -d -m 0755 "$releases_dir"
959959
archive=$temporary/engine-$install_ref.tar
960960
if is_git_checkout "$repo_root" && git -C "$repo_root" cat-file -e "$install_ref^{commit}" 2>/dev/null; then
961-
git -C "$repo_root" archive --format=tar --output "$archive" "$install_ref"
961+
git -C "$repo_root" archive --format=tar --output "$archive" "$install_ref" || return 1
962962
else
963963
[[ "$engine_repository" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || die 'delivery engine repository is invalid'
964964
checkout=$temporary/engine-repository-$install_ref
@@ -967,12 +967,12 @@ install_release() {
967967
GIT_TERMINAL_PROMPT=0 git -C "$checkout" fetch -q --depth=1 origin "$install_ref" || die 'pinned ci-fleet engine commit could not be fetched'
968968
resolved=$(git -C "$checkout" rev-parse 'FETCH_HEAD^{commit}')
969969
[[ "$resolved" == "$install_ref" ]] || die 'fetched ci-fleet engine commit does not match desired state'
970-
git -C "$checkout" archive --format=tar --output "$archive" FETCH_HEAD
970+
git -C "$checkout" archive --format=tar --output "$archive" FETCH_HEAD || return 1
971971
fi
972972
staged_release=$(mktemp -d "$releases_dir/.${install_ref}.staging.XXXXXX")
973973
staging_paths+=("$staged_release")
974974
chmod 0755 "$staged_release"
975-
(umask 0022; tar --no-same-permissions -xf "$archive" -C "$staged_release")
975+
(umask 0022; tar --no-same-permissions -xf "$archive" -C "$staged_release") || return 1
976976
printf '%s\n' "$install_ref" >"$staged_release/.ci-fleet-engine-ref"
977977
chmod 0644 "$staged_release/.ci-fleet-engine-ref"
978978
release_tree_digest "$staged_release" >"$staged_release/.ci-fleet-tree-sha256"
@@ -981,7 +981,7 @@ install_release() {
981981
! release_tree_permissions_trusted "$staged_release"; then
982982
die 'staged engine release is incomplete or untrusted'
983983
fi
984-
atomic_replace_directory "$staged_release" "$install_dir"
984+
atomic_replace_directory "$staged_release" "$install_dir" || return 1
985985
}
986986

987987
install_manager() {
@@ -995,18 +995,18 @@ install_manager() {
995995
! release_tree_permissions_trusted "$manager_release"; then
996996
install -d -m 0755 "$manager_releases"
997997
archive=$temporary/manager-$manager_commit.tar
998-
tar -cf "$archive" -C "$source_release" .
998+
tar -cf "$archive" -C "$source_release" . || return 1
999999
staged_manager=$(mktemp -d "$manager_releases/.${manager_commit}.staging.XXXXXX")
10001000
staging_paths+=("$staged_manager")
10011001
chmod 0755 "$staged_manager"
1002-
(umask 0022; tar --no-same-permissions -xf "$archive" -C "$staged_manager")
1002+
(umask 0022; tar --no-same-permissions -xf "$archive" -C "$staged_manager") || return 1
10031003
printf '%s\n' "$manager_commit" >"$staged_manager/.ci-fleet-engine-ref"
10041004
chmod 0644 "$staged_manager/.ci-fleet-engine-ref"
10051005
if ! manager_release_complete "$staged_manager" "$manager_commit" "$require_status" "$require_schema" ||
10061006
! release_tree_permissions_trusted "$staged_manager"; then
10071007
die 'staged installer manager release is incomplete or untrusted'
10081008
fi
1009-
atomic_replace_directory "$staged_manager" "$manager_release"
1009+
atomic_replace_directory "$staged_manager" "$manager_release" || return 1
10101010
fi
10111011
[[ "$activate" == true ]] || return 0
10121012
install -d -m 0755 "$manager_root"

‎scripts/test-install-worker-controller.sh‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,11 @@ chmod 700 "$fake_bin/dpkg"
326326

327327
cat >"$fake_bin/tar" <<'EOF'
328328
#!/usr/bin/env bash
329+
if [[ -n "${FAKE_FAIL_TAR_EXTRACT_ONCE:-}" && -f "$FAKE_FAIL_TAR_EXTRACT_ONCE" && " $* " == *' -xf '* ]]; then
330+
rm -f "$FAKE_FAIL_TAR_EXTRACT_ONCE"
331+
"$REAL_TAR" "$@"
332+
exit 45
333+
fi
329334
if [[ -n "${FAKE_FAIL_TAR_ONCE:-}" && -f "$FAKE_FAIL_TAR_ONCE" ]]; then
330335
rm -f "$FAKE_FAIL_TAR_ONCE"
331336
exit 45
@@ -361,6 +366,11 @@ chmod 700 "$fake_bin/mv"
361366

362367
cat >"$fake_bin/git" <<'EOF'
363368
#!/usr/bin/env bash
369+
if [[ -n ${FAKE_FAIL_GIT_ARCHIVE_ONCE:-} && -f "$FAKE_FAIL_GIT_ARCHIVE_ONCE" && " $* " == *" archive "* ]]; then
370+
rm -f "$FAKE_FAIL_GIT_ARCHIVE_ONCE"
371+
"$REAL_GIT" "$@" -- . ':(exclude)README.md'
372+
exit 90
373+
fi
364374
if [[ -n ${FAKE_FAIL_GIT_FETCH:-} && " $* " == *" fetch "* ]]; then
365375
exit 90
366376
fi
@@ -678,6 +688,40 @@ grep -Fq 'NETWORK_POLICY_APPLIED' <<<"$first" || fail 'fresh install did not app
678688
[[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$root/opt/ci-fleet/manager/releases/$engine_ref" ]] || fail 'installer manager did not activate the desired engine release'
679689
[[ -f "$FAKE_DOCKER_STATE" ]] || fail 'active controller was not started'
680690

691+
staging_failure_ref=$(write_config active 2 2)
692+
export FAKE_FAIL_UP_ONCE=$tmp/staging-failure-up
693+
: >"$FAKE_FAIL_UP_ONCE"
694+
staging_failure_output=$tmp/staging-failure.out
695+
if "$installer" --upgrade "${base_args[@]}" --ref "$staging_failure_ref" >"$staging_failure_output" 2>&1; then
696+
fail 'staging failure checkpoint fixture unexpectedly succeeded'
697+
fi
698+
unset FAKE_FAIL_UP_ONCE
699+
staging_failure_checkpoint=$(awk '$1 == "CHECKPOINT_CREATED" {sub(/^path=/, "", $2); value=$2} END {print value}' "$staging_failure_output")
700+
[[ -n "$staging_failure_checkpoint" ]] || fail 'staging failure checkpoint was not created'
701+
active_release=$(readlink -f "$root/opt/ci-fleet/current")
702+
manager_release=$(readlink -f "$root/opt/ci-fleet/manager/current")
703+
printf '%s\n' "$manager_release" >"$staging_failure_checkpoint/release-target"
704+
rm -f "$active_release/deploy/compose.yaml"
705+
printf 'preserve\n' >"$active_release/preserve-on-staging-failure"
706+
707+
export FAKE_FAIL_GIT_ARCHIVE_ONCE=$tmp/fail-git-archive-once
708+
: >"$FAKE_FAIL_GIT_ARCHIVE_ONCE"
709+
expect_command_failure "$installer" --rollback
710+
unset FAKE_FAIL_GIT_ARCHIVE_ONCE
711+
[[ -f "$active_release/preserve-on-staging-failure" && ! -f "$active_release/deploy/compose.yaml" ]] || fail 'failed git archive replaced the existing final release'
712+
713+
export FAKE_FAIL_TAR_EXTRACT_ONCE=$tmp/fail-tar-extract-once
714+
: >"$FAKE_FAIL_TAR_EXTRACT_ONCE"
715+
expect_command_failure "$installer" --rollback
716+
unset FAKE_FAIL_TAR_EXTRACT_ONCE
717+
[[ -f "$active_release/preserve-on-staging-failure" && ! -f "$active_release/deploy/compose.yaml" ]] || fail 'failed archive extraction replaced the existing final release'
718+
719+
expect_success "$installer" --rollback >/dev/null
720+
[[ -f "$active_release/deploy/compose.yaml" && ! -e "$active_release/preserve-on-staging-failure" ]] || fail 'successful release staging did not replace the incomplete final release'
721+
rm -rf "$staging_failure_checkpoint"
722+
git -C "$config_repo" reset -q --hard "$ref_one"
723+
[[ ${CI_FLEET_TEST_STOP_AFTER_RELEASE_STAGING_FAILURE:-0} != 1 ]] || { printf 'RELEASE_STAGING_FAILURE_REGRESSION_OK\n'; exit 0; }
724+
681725
daemon_config=$root/etc/docker/daemon.json
682726
python3 - "$daemon_config" <<'PY' || fail 'fresh install did not apply the advertised Docker network policy'
683727
import json

0 commit comments

Comments
 (0)