From d554db0d02806342123fc6a143fd84494c53ca9b Mon Sep 17 00:00:00 2001 From: Polina Bungina <27892524+hughcapet@users.noreply.github.com> Date: Fri, 6 Oct 2023 14:56:31 +0200 Subject: [PATCH] Refactor tests (#933) - wait_all_streaming/wait_zero_lag with configurable replica count - create table with oids using special func, not in the initial schema - add pg_repack extension to the initial schema - separate func for creating timescaledb objects - set WALE_BACKUP_THRESHOLD_PERCENTAGE to a higher value to ensure wal-e replica bootsrap method in test suite 5 - single verify_clone_upgrade func - adjust comments and formatting - ensure test suite 4 actually restores latest backup from PG10 and then upgrades --- postgres-appliance/tests/schema.sql | 3 + postgres-appliance/tests/test_spilo.sh | 133 +++++++++--------- .../tests/{schema2.sql => timescaledb.sql} | 4 - 3 files changed, 71 insertions(+), 69 deletions(-) rename postgres-appliance/tests/{schema2.sql => timescaledb.sql} (66%) diff --git a/postgres-appliance/tests/schema.sql b/postgres-appliance/tests/schema.sql index 41bea8672..b4ea8224f 100644 --- a/postgres-appliance/tests/schema.sql +++ b/postgres-appliance/tests/schema.sql @@ -1,3 +1,6 @@ +CREATE EXTENSION pg_repack; +CREATE EXTENSION amcheck_next; /* the upgrade script must delete it before running pg_upgrade --check! */ + CREATE DATABASE test_db; \c test_db diff --git a/postgres-appliance/tests/test_spilo.sh b/postgres-appliance/tests/test_spilo.sh index c35cfe990..7a6c5ca3e 100755 --- a/postgres-appliance/tests/test_spilo.sh +++ b/postgres-appliance/tests/test_spilo.sh @@ -100,27 +100,33 @@ function wait_query() { } function wait_all_streaming() { - log_info "Waiting for all replicas to start streaming from the leader..." - wait_query "$1" "SELECT COUNT(*) FROM pg_stat_replication WHERE application_name LIKE 'spilo_'" 2 + local repl_count=${2:-2} + log_info "Waiting for all replicas to start streaming from the leader ($1)..." + wait_query "$1" "SELECT COUNT(*) FROM pg_stat_replication WHERE application_name LIKE 'spilo_'" "$repl_count" } function wait_zero_lag() { + local repl_count=${2:-2} log_info "Waiting for all replicas to catch up with WAL replay..." - wait_query "$1" "SELECT COUNT(*) FROM pg_stat_replication WHERE application_name LIKE 'spilo_' AND pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_current_wal_lsn(), COALESCE(replay_lsn, '0/0')) < 16*1024*1024" 2 + wait_query "$1" "SELECT COUNT(*) FROM pg_stat_replication WHERE application_name LIKE 'spilo_' AND pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_current_wal_lsn(), COALESCE(replay_lsn, '0/0')) < 16*1024*1024" "$repl_count" } function create_schema() { docker_exec -i "$1" "psql -U postgres" < schema.sql } -function create_schema2() { - docker_exec -i "$1" "psql -U postgres" < schema2.sql +function create_table_with_oids() { + docker_exec "$1" "psql -U postgres -d test_db -c 'CREATE TABLE with_oids() WITH OIDS'" } function drop_table_with_oids() { docker_exec "$1" "psql -U postgres -d test_db -c 'DROP TABLE with_oids'" } +function create_timescaledb() { + docker_exec -i "$1" "psql -U postgres" < timescaledb.sql +} + function drop_timescaledb() { docker_exec "$1" "psql -U postgres -d test_db -c 'DROP EXTENSION timescaledb CASCADE'" } @@ -181,6 +187,7 @@ function start_clone_with_wale_upgrade_container() { -e CLONE_SCOPE=demo \ -e CLONE_METHOD=CLONE_WITH_WALE \ -e CLONE_TARGET_TIME="$(next_minute)" \ + -e WALE_BACKUP_THRESHOLD_PERCENTAGE=80 \ --name "${PREFIX}upgrade$ID" \ -d "spilo$ID" } @@ -228,20 +235,13 @@ function start_clone_with_basebackup_upgrade_container() { -d spilo3 } -function verify_clone_with_wale_upgrade() { - wait_query "$1" "SELECT current_setting('server_version_num')::int/10000" 11 2> /dev/null -} - -function verify_clone_with_basebackup_upgrade() { - log_info "Waiting for clone with basebackup and upgrade 11->12 to complete..." +function verify_clone_upgrade() { + local type=$2 + local from_version=$3 + local to_version=$4 + log_info "Waiting for clone with $type and upgrade $from_version->$to_version to complete..." find_leader "$1" 1 - wait_query "$1" "SELECT current_setting('server_version_num')::int/10000" 12 2> /dev/null -} - -function verify_clone_with_wale_upgrade_to_15() { - log_info "Waiting for clone with wal-e and upgrade 10->15 to complete..." - find_leader "$1" 1 - wait_query "$1" "SELECT current_setting('server_version_num')::int/10000" 15 2> /dev/null + wait_query "$1" "SELECT current_setting('server_version_num')::int/10000" "$to_version" 2> /dev/null } function verify_archive_mode_is_on() { @@ -251,105 +251,110 @@ function verify_archive_mode_is_on() { # TEST SUITE 1 - In-place major upgrade 10->11->...->15 -# TEST SUITE 2 - Major upgrade 10->15 after wal-e clone +# TEST SUITE 2 - Major upgrade 10->15 after wal-e clone (with CLONE_PGVERSION set) # TEST SUITE 3 - PITR (clone with wal-e) with unreachable target (13+) -# TEST SUITE 4 - Major upgrade 10->11 after wal-e clone +# TEST SUITE 4 - Major upgrade 10->11 after wal-e clone (no CLONE_PGVERSION) # TEST SUITE 5 - Replica bootstrap with wal-e +# TEST SUITE 6 - Major upgrade 11->12 after clone with basebackup function test_spilo() { # TEST SUITE 1 local container=$1 run_test test_envdir_suffix "$container" 10 + log_info "[TS1] Testing wrong upgrade setups" run_test test_inplace_upgrade_wrong_version "$container" run_test test_inplace_upgrade_wrong_capacity "$container" wait_all_streaming "$container" - - create_schema "$container" || exit 1 - + create_schema "$container" || exit 1 # incompatible upgrade exts, custom tbl with statistics and data # run_test test_failed_inplace_upgrade_big_replication_lag "$container" wait_zero_lag "$container" run_test verify_archive_mode_is_on "$container" wait_backup "$container" + # TEST SUITE 2 + local upgrade3_container + upgrade3_container=$(start_clone_with_wale_upgrade_to_15_container) # SCOPE=upgrade3 PGVERSION=15 CLONE: _SCOPE=demo _PGVERSION=10 _TARGET_TIME= + log_info "[TS2] Started $upgrade3_container for testing major upgrade 10->15 after clone with wal-e" + + + # TEST SUITE 4 local upgrade_container - upgrade_container=$(start_clone_with_wale_upgrade_to_15_container) - log_info "Started $upgrade_container for testing major upgrade 10->15 after clone with wal-e" + upgrade_container=$(start_clone_with_wale_upgrade_container) # SCOPE=upgrade PGVERSION=11 CLONE: _SCOPE=demo _TARGET_TIME= + log_info "[TS4] Started $upgrade_container for testing major upgrade 10->11 after clone with wal-e" - # TEST SUITE 1 - wait_backup "$container" - log_info "Testing in-place major upgrade 10->11" + # TEST SUITE 1 + log_info "[TS1] Testing in-place major upgrade 10->11" run_test test_successful_inplace_upgrade_to_11 "$container" - wait_all_streaming "$container" - run_test test_envdir_updated_to_x 11 - create_schema2 "$container" || exit 1 + create_timescaledb "$container" # we don't install it at the beginning, as we do 10->15 in a clone + log_info "[TS1] Testing in-place major upgrade 11->13 with failing check" + create_table_with_oids "$container" run_test test_pg_upgrade_to_13_check_failed "$container" # pg_upgrade --check complains about OID drop_table_with_oids "$container" + # TEST SUITE 2 - run_test verify_clone_with_wale_upgrade_to_15 "$upgrade_container" + log_info "[TS2] Testing in-place major upgrade 10->15 after wal-e clone" + run_test verify_clone_upgrade "$upgrade3_container" "wal-e" 10 15 + + run_test verify_archive_mode_is_on "$upgrade3_container" + wait_backup "$upgrade3_container" - run_test verify_archive_mode_is_on "$upgrade_container" - wait_backup "$upgrade_container" - rm_container "$upgrade_container" # TEST SUITE 3 local clone15_container - clone15_container=$(start_clone_with_wale_15_container) - log_info "Started $clone15_container for testing point-in-time recovery (clone with wal-e) with unreachable target on 13+" + clone15_container=$(start_clone_with_wale_15_container) # SCOPE=clone15 CLONE: _SCOPE=upgrade3 _PGVERSION=15 _TARGET_TIME= + log_info "[TS3] Started $clone15_container for testing point-in-time recovery (clone with wal-e) with unreachable target on 13+" - # TEST SUITE 4 - wait_backup "$container" - wait_zero_lag "$container" - upgrade_container=$(start_clone_with_wale_upgrade_container) - log_info "Started $upgrade_container for testing major upgrade 10->11 after clone with wal-e" # TEST SUITE 1 - log_info "Testing in-place major upgrade 11->13" + log_info "[TS1] Testing in-place major upgrade 11->13" run_test test_successful_inplace_upgrade_to_13 "$container" - wait_all_streaming "$container" - run_test test_envdir_updated_to_x 13 + # TEST SUITE 3 find_leader "$clone15_container" run_test verify_archive_mode_is_on "$clone15_container" + # TEST SUITE 1 wait_backup "$container" - log_info "Testing in-place major upgrade to 13->14" + log_info "[TS1] Testing in-place major upgrade 13->14" run_test test_successful_inplace_upgrade_to_14 "$container" - wait_all_streaming "$container" - run_test test_envdir_updated_to_x 14 + # TEST SUITE 4 - log_info "Waiting for clone with wal-e and upgrade 10->11 to complete..." - find_leader "$upgrade_container" 1 - run_test verify_clone_with_wale_upgrade "$upgrade_container" + log_info "[TS4] Testing in-place major upgrade 10->11 after clone with wal-e" + run_test verify_clone_upgrade "$upgrade_container" "wal-e" 10 11 + run_test verify_archive_mode_is_on "$upgrade_container" wait_backup "$upgrade_container" + # TEST SUITE 5 local upgrade_replica_container - upgrade_replica_container=$(start_clone_with_wale_upgrade_replica_container) - log_info "Started $upgrade_replica_container for testing replica bootstrap with wal-e" + upgrade_replica_container=$(start_clone_with_wale_upgrade_replica_container) # SCOPE=upgrade + log_info "[TS5] Started $upgrade_replica_container for testing replica bootstrap with wal-e" - # TEST SUITE 4 + + # TEST SUITE 6 local basebackup_container - basebackup_container=$(start_clone_with_basebackup_upgrade_container "$upgrade_container") - log_info "Started $basebackup_container for testing major upgrade 11->12 after clone with basebackup" + basebackup_container=$(start_clone_with_basebackup_upgrade_container "$upgrade_container") # SCOPE=upgrade2 PGVERSION=12 CLONE: _SCOPE=upgrade + log_info "[TS6] Started $basebackup_container for testing major upgrade 11->12 after clone with basebackup" + # TEST SUITE 1 # run_test test_pg_upgrade_to_15_check_failed "$container" # pg_upgrade --check complains about timescaledb @@ -357,22 +362,20 @@ function test_spilo() { wait_backup "$container" # drop_timescaledb "$container" - - log_info "Testing in-place major upgrade to 14->15" + log_info "[TS1] Testing in-place major upgrade 14->15" run_test test_successful_inplace_upgrade_to_15 "$container" - wait_all_streaming "$container" - run_test test_envdir_updated_to_x 15 - wait_backup "$container" # TEST SUITE 5 - log_info "Waiting for postgres to start in the $upgrade_replica_container..." - run_test verify_clone_with_wale_upgrade "$upgrade_replica_container" + log_info "[TS5] Waiting for postgres to start in $upgrade_replica_container and stream from primary..." + wait_all_streaming "$upgrade_container" 1 - # TEST SUITE 4 - run_test verify_clone_with_basebackup_upgrade "$basebackup_container" + + # TEST SUITE 6 + log_info "[TS6] Testing in-place major upgrade 11->12 after clone with basebackup" + run_test verify_clone_upgrade "$basebackup_container" "basebackup" 11 12 run_test verify_archive_mode_is_on "$basebackup_container" } diff --git a/postgres-appliance/tests/schema2.sql b/postgres-appliance/tests/timescaledb.sql similarity index 66% rename from postgres-appliance/tests/schema2.sql rename to postgres-appliance/tests/timescaledb.sql index 52df54b70..8b397772a 100644 --- a/postgres-appliance/tests/schema2.sql +++ b/postgres-appliance/tests/timescaledb.sql @@ -1,9 +1,5 @@ -CREATE EXTENSION amcheck_next; /* the upgrade script must delete it before running pg_upgrade --check! */ - \c test_db -CREATE TABLE with_oids() WITH OIDS; - CREATE EXTENSION timescaledb; CREATE TABLE "fOo" (id bigint NOT NULL PRIMARY KEY);