From 6a0ab4fd167f429b78338e686c805418aeb918cf Mon Sep 17 00:00:00 2001 From: Chill Validation <92176880+chillyvee@users.noreply.github.com> Date: Fri, 14 Jul 2023 04:00:37 +0900 Subject: [PATCH] feat(cosmovisor): graceful shutdown (#16963) --- tools/cosmovisor/CHANGELOG.md | 1 + tools/cosmovisor/README.md | 1 + tools/cosmovisor/args.go | 15 ++ tools/cosmovisor/args_test.go | 143 ++++++++++-------- tools/cosmovisor/process.go | 28 +++- tools/cosmovisor/process_test.go | 45 ++++++ .../dontdie/cosmovisor/genesis/bin/dummyd | 17 +++ .../cosmovisor/upgrades/chain2/bin/dummyd | 6 + .../cosmovisor/testdata/dontdie/data/.gitkeep | 0 9 files changed, 190 insertions(+), 66 deletions(-) create mode 100755 tools/cosmovisor/testdata/dontdie/cosmovisor/genesis/bin/dummyd create mode 100755 tools/cosmovisor/testdata/dontdie/cosmovisor/upgrades/chain2/bin/dummyd create mode 100644 tools/cosmovisor/testdata/dontdie/data/.gitkeep diff --git a/tools/cosmovisor/CHANGELOG.md b/tools/cosmovisor/CHANGELOG.md index c8557e9833fe..1be2bba58148 100644 --- a/tools/cosmovisor/CHANGELOG.md +++ b/tools/cosmovisor/CHANGELOG.md @@ -41,6 +41,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ * [#16413](https://github.com/cosmos/cosmos-sdk/issues/16413) Add `cosmovisor pre-upgrade` command to manually add an upgrade to cosmovisor. * [#16573](https://github.com/cosmos/cosmos-sdk/pull/16573) Extend `cosmovisor` configuration with new log format options * [#16550](https://github.com/cosmos/cosmos-sdk/pull/16550) Add COSMOVISOR_CUSTOM_PREUPGRADE to cosmovisor to execute custom pre-upgrade scripts (separate from daemon pre-upgrade). +* [#16963](https://github.com/cosmos/cosmos-sdk/pull/69630) Add DAEMON_SHUTDOWN_GRACE to send interrupt and wait before sending kill * [#15361](https://github.com/cosmos/cosmos-sdk/pull/15361) Add `cosmovisor config` command to display the configuration used by cosmovisor. ## Improvements diff --git a/tools/cosmovisor/README.md b/tools/cosmovisor/README.md index 46a9d1159d72..b6191307335a 100644 --- a/tools/cosmovisor/README.md +++ b/tools/cosmovisor/README.md @@ -90,6 +90,7 @@ Use of `cosmovisor` without one of the action arguments is deprecated. For backw * `DAEMON_DOWNLOAD_MUST_HAVE_CHECKSUM` (*optional*, default = `false`), if `true` cosmovisor will require that a checksum is provided in the upgrade plan for the binary to be downloaded. If `false`, cosmovisor will not require a checksum to be provided, but still check the checksum if one is provided. * `DAEMON_RESTART_AFTER_UPGRADE` (*optional*, default = `true`), if `true`, restarts the subprocess with the same command-line arguments and flags (but with the new binary) after a successful upgrade. Otherwise (`false`), `cosmovisor` stops running after an upgrade and requires the system administrator to manually restart it. Note restart is only after the upgrade and does not auto-restart the subprocess after an error occurs. * `DAEMON_RESTART_DELAY` (*optional*, default none), allow a node operator to define a delay between the node halt (for upgrade) and backup by the specified time. The value must be a duration (e.g. `1s`). +* `DAEMON_SHUTDOWN_GRACE` (*optional*, default none), if set, send interrupt to binary and wait the specified time to allow for cleanup/cache flush to disk before sending the kill signal. The value must be a duration (e.g. `1s`). * `DAEMON_POLL_INTERVAL` (*optional*, default 300 milliseconds), is the interval length for polling the upgrade plan file. The value must be a duration (e.g. `1s`). * `DAEMON_DATA_BACKUP_DIR` option to set a custom backup directory. If not set, `DAEMON_HOME` is used. * `UNSAFE_SKIP_BACKUP` (defaults to `false`), if set to `true`, upgrades directly without performing a backup. Otherwise (`false`, default) backs up the data before trying the upgrade. The default value of false is useful and recommended in case of failures and when a backup needed to rollback. We recommend using the default backup option `UNSAFE_SKIP_BACKUP=false`. diff --git a/tools/cosmovisor/args.go b/tools/cosmovisor/args.go index 84a202a04590..4501e8e7d880 100644 --- a/tools/cosmovisor/args.go +++ b/tools/cosmovisor/args.go @@ -25,6 +25,7 @@ const ( EnvDownloadMustHaveChecksum = "DAEMON_DOWNLOAD_MUST_HAVE_CHECKSUM" EnvRestartUpgrade = "DAEMON_RESTART_AFTER_UPGRADE" EnvRestartDelay = "DAEMON_RESTART_DELAY" + EnvShutdownGrace = "DAEMON_SHUTDOWN_GRACE" EnvSkipBackup = "UNSAFE_SKIP_BACKUP" EnvDataBackupPath = "DAEMON_DATA_BACKUP_DIR" EnvInterval = "DAEMON_POLL_INTERVAL" @@ -51,6 +52,7 @@ type Config struct { DownloadMustHaveChecksum bool RestartAfterUpgrade bool RestartDelay time.Duration + ShutdownGrace time.Duration PollInterval time.Duration UnsafeSkipBackup bool DataBackupPath string @@ -207,6 +209,17 @@ func GetConfigFromEnv() (*Config, error) { } } + cfg.ShutdownGrace = 0 // default value but makes it explicit + shutdownGrace := os.Getenv(EnvShutdownGrace) + if shutdownGrace != "" { + val, err := parseEnvDuration(shutdownGrace) + if err != nil { + errs = append(errs, fmt.Errorf("invalid: %s: %w", EnvShutdownGrace, err)) + } else { + cfg.ShutdownGrace = val + } + } + envPreupgradeMaxRetriesVal := os.Getenv(EnvPreupgradeMaxRetries) if cfg.PreupgradeMaxRetries, err = strconv.Atoi(envPreupgradeMaxRetriesVal); err != nil && envPreupgradeMaxRetriesVal != "" { errs = append(errs, fmt.Errorf("%s could not be parsed to int: %w", EnvPreupgradeMaxRetries, err)) @@ -428,6 +441,7 @@ func (cfg Config) DetailString() string { {EnvDownloadMustHaveChecksum, fmt.Sprintf("%t", cfg.DownloadMustHaveChecksum)}, {EnvRestartUpgrade, fmt.Sprintf("%t", cfg.RestartAfterUpgrade)}, {EnvRestartDelay, cfg.RestartDelay.String()}, + {EnvShutdownGrace, cfg.ShutdownGrace.String()}, {EnvInterval, cfg.PollInterval.String()}, {EnvSkipBackup, fmt.Sprintf("%t", cfg.UnsafeSkipBackup)}, {EnvDataBackupPath, cfg.DataBackupPath}, @@ -436,6 +450,7 @@ func (cfg Config) DetailString() string { {EnvColorLogs, fmt.Sprintf("%t", cfg.ColorLogs)}, {EnvTimeFormatLogs, cfg.TimeFormatLogs}, {EnvCustomPreupgrade, cfg.CustomPreupgrade}, + {EnvDisableRecase, fmt.Sprintf("%t", cfg.DisableRecase)}, } derivedEntries := []struct{ name, value string }{ diff --git a/tools/cosmovisor/args_test.go b/tools/cosmovisor/args_test.go index 399e31925350..4645a562da94 100644 --- a/tools/cosmovisor/args_test.go +++ b/tools/cosmovisor/args_test.go @@ -39,6 +39,7 @@ type cosmovisorEnv struct { TimeFormatLogs string CustomPreupgrade string DisableRecase string + ShutdownGrace string } type envMap struct { @@ -55,6 +56,7 @@ func (c cosmovisorEnv) ToMap() map[string]envMap { EnvDownloadMustHaveChecksum: {val: c.DownloadMustHaveChecksum, allowEmpty: false}, EnvRestartUpgrade: {val: c.RestartUpgrade, allowEmpty: false}, EnvRestartDelay: {val: c.RestartDelay, allowEmpty: false}, + EnvShutdownGrace: {val: c.ShutdownGrace, allowEmpty: false}, EnvSkipBackup: {val: c.SkipBackup, allowEmpty: false}, EnvDataBackupPath: {val: c.DataBackupPath, allowEmpty: false}, EnvInterval: {val: c.Interval, allowEmpty: false}, @@ -82,6 +84,8 @@ func (c *cosmovisorEnv) Set(envVar, envVal string) { c.RestartUpgrade = envVal case EnvRestartDelay: c.RestartDelay = envVal + case EnvShutdownGrace: + c.ShutdownGrace = envVal case EnvSkipBackup: c.SkipBackup = envVal case EnvDataBackupPath: @@ -101,7 +105,7 @@ func (c *cosmovisorEnv) Set(envVar, envVal string) { case EnvDisableRecase: c.DisableRecase = envVal default: - panic(fmt.Errorf("Unknown environment variable [%s]. Ccannot set field to [%s]. ", envVar, envVal)) + panic(fmt.Errorf("Unknown environment variable [%s]. Cannot set field to [%s]. ", envVar, envVal)) } } @@ -461,6 +465,7 @@ func (s *argsTestSuite) TestGetConfigFromEnv() { timeFormatLogs string, customPreUpgrade string, disableRecase bool, + shutdownGrace int, ) *Config { return &Config{ Home: home, @@ -478,6 +483,7 @@ func (s *argsTestSuite) TestGetConfigFromEnv() { TimeFormatLogs: timeFormatLogs, CustomPreupgrade: customPreUpgrade, DisableRecase: disableRecase, + ShutdownGrace: time.Duration(shutdownGrace), } } @@ -503,19 +509,20 @@ func (s *argsTestSuite) TestGetConfigFromEnv() { TimeFormatLogs: "bad", CustomPreupgrade: "", DisableRecase: "bad", + ShutdownGrace: "bad", }, expectedCfg: nil, - expectedErrCount: 12, + expectedErrCount: 13, }, { name: "all good", - envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "true"}, - expectedCfg: newConfig(absPath, "testname", true, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", true), + envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "true", "10s"}, + expectedCfg: newConfig(absPath, "testname", true, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", true, 10000000000), expectedErrCount: 0, }, { name: "nothing set", - envVals: cosmovisorEnv{"", "", "", "", "", "", "", "", "", "", "false", "false", "", "", ""}, + envVals: cosmovisorEnv{"", "", "", "", "", "", "", "", "", "", "false", "false", "", "", "", ""}, expectedCfg: nil, expectedErrCount: 3, }, @@ -523,231 +530,237 @@ func (s *argsTestSuite) TestGetConfigFromEnv() { // timeformat tests are done in the TestTimeFormat { name: "download bin bad", - envVals: cosmovisorEnv{absPath, "testname", "bad", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "", ""}, + envVals: cosmovisorEnv{absPath, "testname", "bad", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "", "", ""}, expectedCfg: nil, expectedErrCount: 1, }, { name: "download bin not set", - envVals: cosmovisorEnv{absPath, "testname", "", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "", ""}, - expectedCfg: newConfig(absPath, "testname", false, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "", false), + envVals: cosmovisorEnv{absPath, "testname", "", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "", "", ""}, + expectedCfg: newConfig(absPath, "testname", false, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "", false, 0), expectedErrCount: 0, }, { name: "download bin true", - envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""}, - expectedCfg: newConfig(absPath, "testname", true, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false), + envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""}, + expectedCfg: newConfig(absPath, "testname", true, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false, 0), expectedErrCount: 0, }, { name: "download bin false", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""}, - expectedCfg: newConfig(absPath, "testname", false, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false), + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""}, + expectedCfg: newConfig(absPath, "testname", false, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false, 0), expectedErrCount: 0, }, { name: "download ensure checksum true", - envVals: cosmovisorEnv{absPath, "testname", "true", "false", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""}, - expectedCfg: newConfig(absPath, "testname", true, false, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false), + envVals: cosmovisorEnv{absPath, "testname", "true", "false", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""}, + expectedCfg: newConfig(absPath, "testname", true, false, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false, 0), expectedErrCount: 0, }, { name: "restart upgrade bad", - envVals: cosmovisorEnv{absPath, "testname", "true", "true", "bad", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""}, + envVals: cosmovisorEnv{absPath, "testname", "true", "true", "bad", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""}, expectedCfg: nil, expectedErrCount: 1, }, { name: "restart upgrade not set", - envVals: cosmovisorEnv{absPath, "testname", "true", "true", "", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""}, - expectedCfg: newConfig(absPath, "testname", true, true, true, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false), + envVals: cosmovisorEnv{absPath, "testname", "true", "true", "", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""}, + expectedCfg: newConfig(absPath, "testname", true, true, true, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false, 0), expectedErrCount: 0, }, { name: "restart upgrade true", - envVals: cosmovisorEnv{absPath, "testname", "true", "true", "true", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""}, - expectedCfg: newConfig(absPath, "testname", true, true, true, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false), + envVals: cosmovisorEnv{absPath, "testname", "true", "true", "true", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""}, + expectedCfg: newConfig(absPath, "testname", true, true, true, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false, 0), expectedErrCount: 0, }, { name: "restart upgrade true", - envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""}, - expectedCfg: newConfig(absPath, "testname", true, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false), + envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""}, + expectedCfg: newConfig(absPath, "testname", true, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false, 0), expectedErrCount: 0, }, { name: "skip unsafe backups bad", - envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "bad", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""}, + envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "bad", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""}, expectedCfg: nil, expectedErrCount: 1, }, { name: "skip unsafe backups not set", - envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""}, - expectedCfg: newConfig(absPath, "testname", true, true, false, 600, false, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false), + envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""}, + expectedCfg: newConfig(absPath, "testname", true, true, false, 600, false, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false, 0), expectedErrCount: 0, }, { name: "skip unsafe backups true", - envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""}, - expectedCfg: newConfig(absPath, "testname", true, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false), + envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "true", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""}, + expectedCfg: newConfig(absPath, "testname", true, true, false, 600, true, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false, 0), expectedErrCount: 0, }, { name: "skip unsafe backups false", - envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "false", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""}, - expectedCfg: newConfig(absPath, "testname", true, true, false, 600, false, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false), + envVals: cosmovisorEnv{absPath, "testname", "true", "true", "false", "600ms", "false", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""}, + expectedCfg: newConfig(absPath, "testname", true, true, false, 600, false, absPath, 303, 1, false, true, time.Kitchen, "preupgrade.sh", false, 0), expectedErrCount: 0, }, { name: "poll interval bad", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "bad", "1", "false", "true", "kitchen", "preupgrade.sh", ""}, + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "bad", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""}, expectedCfg: nil, expectedErrCount: 1, }, { name: "poll interval 0", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "0", "1", "false", "true", "kitchen", "preupgrade.sh", ""}, + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "0", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""}, expectedCfg: nil, expectedErrCount: 1, }, { name: "poll interval not set", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "", "1", "false", "false", "kitchen", "preupgrade.sh", ""}, - expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 300, 1, false, false, time.Kitchen, "preupgrade.sh", false), + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "", "1", "false", "false", "kitchen", "preupgrade.sh", "", ""}, + expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 300, 1, false, false, time.Kitchen, "preupgrade.sh", false, 0), expectedErrCount: 0, }, { name: "poll interval 600", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "600", "1", "false", "true", "kitchen", "preupgrade.sh", ""}, + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "600", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""}, expectedCfg: nil, expectedErrCount: 1, }, { name: "poll interval 1s", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "1s", "1", "false", "false", "kitchen", "preupgrade.sh", ""}, - expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 1000, 1, false, false, time.Kitchen, "preupgrade.sh", false), + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "1s", "1", "false", "false", "kitchen", "preupgrade.sh", "", ""}, + expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 1000, 1, false, false, time.Kitchen, "preupgrade.sh", false, 0), expectedErrCount: 0, }, { name: "poll interval -3m", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "-3m", "1", "false", "true", "kitchen", "preupgrade.sh", ""}, + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "-3m", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""}, expectedCfg: nil, expectedErrCount: 1, }, { name: "restart delay bad", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "bad", "false", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""}, + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "bad", "false", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""}, expectedCfg: nil, expectedErrCount: 1, }, { name: "restart delay 0", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "0", "false", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""}, + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "0", "false", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""}, expectedCfg: nil, expectedErrCount: 1, }, { name: "restart delay not set", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "", "false", "", "303ms", "1", "false", "false", "kitchen", "preupgrade.sh", ""}, - expectedCfg: newConfig(absPath, "testname", false, true, false, 0, false, absPath, 303, 1, false, false, time.Kitchen, "preupgrade.sh", false), + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "", "false", "", "303ms", "1", "false", "false", "kitchen", "preupgrade.sh", "", ""}, + expectedCfg: newConfig(absPath, "testname", false, true, false, 0, false, absPath, 303, 1, false, false, time.Kitchen, "preupgrade.sh", false, 0), expectedErrCount: 0, }, { name: "restart delay 600", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600", "false", "", "300ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""}, + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600", "false", "", "300ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""}, expectedCfg: nil, expectedErrCount: 1, }, { name: "restart delay 1s", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "1s", "false", "", "303ms", "1", "false", "false", "kitchen", "preupgrade.sh", ""}, - expectedCfg: newConfig(absPath, "testname", false, true, false, 1000, false, absPath, 303, 1, false, false, time.Kitchen, "preupgrade.sh", false), + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "1s", "false", "", "303ms", "1", "false", "false", "kitchen", "preupgrade.sh", "", ""}, + expectedCfg: newConfig(absPath, "testname", false, true, false, 1000, false, absPath, 303, 1, false, false, time.Kitchen, "preupgrade.sh", false, 0), expectedErrCount: 0, }, { name: "restart delay -3m", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "-3m", "false", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", ""}, + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "-3m", "false", "", "303ms", "1", "false", "true", "kitchen", "preupgrade.sh", "", ""}, expectedCfg: nil, expectedErrCount: 1, }, { name: "prepupgrade max retries bad", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "bad", "false", "true", "kitchen", "preupgrade.sh", ""}, + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "bad", "false", "true", "kitchen", "preupgrade.sh", "", ""}, expectedCfg: nil, expectedErrCount: 1, }, { name: "prepupgrade max retries 0", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "0", "false", "false", "kitchen", "preupgrade.sh", ""}, - expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, false, false, time.Kitchen, "preupgrade.sh", false), + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "0", "false", "false", "kitchen", "preupgrade.sh", "", ""}, + expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, false, false, time.Kitchen, "preupgrade.sh", false, 0), expectedErrCount: 0, }, { name: "prepupgrade max retries not set", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "false", "false", "kitchen", "preupgrade.sh", ""}, - expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, false, false, time.Kitchen, "preupgrade.sh", false), + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "false", "false", "kitchen", "preupgrade.sh", "", ""}, + expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, false, false, time.Kitchen, "preupgrade.sh", false, 0), expectedErrCount: 0, }, { name: "prepupgrade max retries 5", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "5", "false", "false", "kitchen", "preupgrade.sh", ""}, - expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 5, false, false, time.Kitchen, "preupgrade.sh", false), + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "5", "false", "false", "kitchen", "preupgrade.sh", "", ""}, + expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 5, false, false, time.Kitchen, "preupgrade.sh", false, 0), expectedErrCount: 0, }, { name: "disable logs bad", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "5", "bad", "true", "kitchen", "preupgrade.sh", ""}, + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "5", "bad", "true", "kitchen", "preupgrade.sh", "", ""}, expectedCfg: nil, expectedErrCount: 1, }, { name: "disable logs good", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "false", "kitchen", "preupgrade.sh", ""}, - expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, true, false, time.Kitchen, "preupgrade.sh", false), + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "false", "kitchen", "preupgrade.sh", "", ""}, + expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, true, false, time.Kitchen, "preupgrade.sh", false, 0), expectedErrCount: 0, }, { name: "disable logs color bad", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "5", "true", "bad", "kitchen", "preupgrade.sh", ""}, + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "5", "true", "bad", "kitchen", "preupgrade.sh", "", ""}, expectedCfg: nil, expectedErrCount: 1, }, { name: "disable logs color good", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "false", "kitchen", "preupgrade.sh", ""}, - expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, true, false, time.Kitchen, "preupgrade.sh", false), + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "false", "kitchen", "preupgrade.sh", "", ""}, + expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, true, false, time.Kitchen, "preupgrade.sh", false, 0), expectedErrCount: 0, }, { name: "disable logs timestamp", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "false", "", "preupgrade.sh", ""}, - expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, true, false, "", "preupgrade.sh", false), + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "false", "", "preupgrade.sh", "", ""}, + expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, true, false, "", "preupgrade.sh", false, 0), expectedErrCount: 0, }, { name: "enable rf3339 logs timestamp", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "true", "rfc3339", "preupgrade.sh", ""}, - expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, true, true, time.RFC3339, "preupgrade.sh", false), + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "true", "rfc3339", "preupgrade.sh", "", ""}, + expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, true, true, time.RFC3339, "preupgrade.sh", false, 0), expectedErrCount: 0, }, { name: "invalid logs timestamp format", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "true", "invalid", "preupgrade.sh", ""}, + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "true", "invalid", "preupgrade.sh", "", ""}, expectedCfg: nil, expectedErrCount: 1, }, { name: "disable recase good", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "true", "rfc3339", "preupgrade.sh", "true"}, - expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, true, true, time.RFC3339, "preupgrade.sh", true), + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "true", "rfc3339", "preupgrade.sh", "true", ""}, + expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, true, true, time.RFC3339, "preupgrade.sh", true, 0), expectedErrCount: 0, }, { name: "disable recase bad", - envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "true", "rfc3339", "preupgrade.sh", "bad"}, + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "true", "rfc3339", "preupgrade.sh", "bad", ""}, expectedErrCount: 1, }, + { + name: "shutdown grace good", + envVals: cosmovisorEnv{absPath, "testname", "false", "true", "false", "600ms", "false", "", "406ms", "", "true", "true", "rfc3339", "preupgrade.sh", "true", "15s"}, + expectedCfg: newConfig(absPath, "testname", false, true, false, 600, false, absPath, 406, 0, true, true, time.RFC3339, "preupgrade.sh", true, 15000000000), + expectedErrCount: 0, + }, } for _, tc := range tests { diff --git a/tools/cosmovisor/process.go b/tools/cosmovisor/process.go index 9dcf219b3c4a..051a3621eb9e 100644 --- a/tools/cosmovisor/process.go +++ b/tools/cosmovisor/process.go @@ -118,7 +118,33 @@ func (l Launcher) WaitForUpgradeOrExit(cmd *exec.Cmd) (bool, error) { case <-l.fw.MonitorUpdate(currentUpgrade): // upgrade - kill the process and restart l.logger.Info("daemon shutting down in an attempt to restart") - _ = cmd.Process.Kill() + + if l.cfg.ShutdownGrace > 0 { + // Interrupt signal + l.logger.Info("sent interrupt to app, waiting for exit") + _ = cmd.Process.Signal(os.Interrupt) + + // Wait app exit + psChan := make(chan *os.ProcessState) + go func() { + pstate, _ := cmd.Process.Wait() + psChan <- pstate + }() + + // Timeout and kill + select { + case <-psChan: + // Normal Exit + l.logger.Info("app exited normally") + case <-time.After(l.cfg.ShutdownGrace): + l.logger.Info("DAEMON_SHUTDOWN_GRACE exceeded, killing app") + // Kill after grace period + _ = cmd.Process.Kill() + } + } else { + // Default: Immediate app kill + _ = cmd.Process.Kill() + } case err := <-cmdDone: l.fw.Stop() // no error -> command exits normally (eg. short command like `gaiad version`) diff --git a/tools/cosmovisor/process_test.go b/tools/cosmovisor/process_test.go index f9b5137f396a..2ecea4cb780f 100644 --- a/tools/cosmovisor/process_test.go +++ b/tools/cosmovisor/process_test.go @@ -149,6 +149,51 @@ func (s *processTestSuite) TestLaunchProcessWithRestartDelay() { } } +// TestPlanShutdownGrace will test upgrades without lower case plan names +func (s *processTestSuite) TestPlanShutdownGrace() { + // binaries from testdata/validate directory + require := s.Require() + home := copyTestData(s.T(), "dontdie") + cfg := &cosmovisor.Config{Home: home, Name: "dummyd", PollInterval: 20, UnsafeSkipBackup: true, ShutdownGrace: 2 * time.Second} + logger := log.NewTestLogger(s.T()).With(log.ModuleKey, "cosmosvisor") + + // should run the genesis binary and produce expected output + stdout, stderr := newBuffer(), newBuffer() + currentBin, err := cfg.CurrentBin() + require.NoError(err) + require.Equal(cfg.GenesisBin(), currentBin) + + launcher, err := cosmovisor.NewLauncher(logger, cfg) + require.NoError(err) + + upgradeFile := cfg.UpgradeInfoFilePath() + + args := []string{"foo", "bar", "1234", upgradeFile} + doUpgrade, err := launcher.Run(args, stdout, stderr) + require.NoError(err) + require.True(doUpgrade) + require.Equal("", stderr.String()) + require.Equal(fmt.Sprintf("Genesis foo bar 1234 %s\nUPGRADE \"Chain2\" NEEDED at height: 49: {}\nWARN Need Flush\nFlushed\n", upgradeFile), stdout.String()) + + // ensure this is upgraded now and produces new output + currentBin, err = cfg.CurrentBin() + require.NoError(err) + + require.Equal(cfg.UpgradeBin("chain2"), currentBin) + args = []string{"second", "run", "--verbose"} + stdout.Reset() + stderr.Reset() + + doUpgrade, err = launcher.Run(args, stdout, stderr) + require.NoError(err) + require.False(doUpgrade) + require.Equal("", stderr.String()) + require.Equal("Chain 2 is live!\nArgs: second run --verbose\nFinished successfully\n", stdout.String()) + + // ended without other upgrade + require.Equal(cfg.UpgradeBin("chain2"), currentBin) +} + // TestLaunchProcess will try running the script a few times and watch upgrades work properly // and args are passed through func (s *processTestSuite) TestLaunchProcessWithDownloads() { diff --git a/tools/cosmovisor/testdata/dontdie/cosmovisor/genesis/bin/dummyd b/tools/cosmovisor/testdata/dontdie/cosmovisor/genesis/bin/dummyd new file mode 100755 index 000000000000..49e60a1e206d --- /dev/null +++ b/tools/cosmovisor/testdata/dontdie/cosmovisor/genesis/bin/dummyd @@ -0,0 +1,17 @@ +#!/bin/sh + + +warn() { + echo "WARN Need Flush" +} + +trap warn INT +echo Genesis $@ +sleep 1 +test -z $4 && exit 1001 +echo 'UPGRADE "Chain2" NEEDED at height: 49: {}' +echo '{"name":"Chain2","height":49,"info":""}' > $4 +sleep 1 +echo 'Flushed' +sleep 1 +echo Did not kill in time. Never should be printed!!! diff --git a/tools/cosmovisor/testdata/dontdie/cosmovisor/upgrades/chain2/bin/dummyd b/tools/cosmovisor/testdata/dontdie/cosmovisor/upgrades/chain2/bin/dummyd new file mode 100755 index 000000000000..0022b84af292 --- /dev/null +++ b/tools/cosmovisor/testdata/dontdie/cosmovisor/upgrades/chain2/bin/dummyd @@ -0,0 +1,6 @@ +#!/bin/sh + +echo Chain 2 is live! +echo Args: $@ +sleep 1 +echo Finished successfully diff --git a/tools/cosmovisor/testdata/dontdie/data/.gitkeep b/tools/cosmovisor/testdata/dontdie/data/.gitkeep new file mode 100644 index 000000000000..e69de29bb2d1