Disallow -v2-deprecation>'not-yet' combined with --enable-v2

server/config/config.go

@@ -192,6 +192,9 @@ type ServerConfig struct {
 	// ExperimentalBootstrapDefragThresholdMegabytes is the minimum number of megabytes needed to be freed for etcd server to
 	// consider running defrag during bootstrap. Needs to be set to non-zero value to take effect.
 	ExperimentalBootstrapDefragThresholdMegabytes uint `json:"experimental-bootstrap-defrag-threshold-megabytes"`
+
+	// V2Deprecation defines a phase of v2store deprecation process.
+	V2Deprecation V2DeprecationEnum `json:"v2-deprecation"`
 }
 
 // VerifyBootstrap sanity-checks the initial config for bootstrap case

server/embed/v2_deprecation.go → server/config/v2_deprecation.go

@@ -12,35 +12,39 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package embed
+package config
 
 type V2DeprecationEnum string
 
 const (
-// Default in v3.5.  Issues a warning if v2store have meaningful content.
-V2_DEPR_0_NOT_YET = V2DeprecationEnum("not-yet")
-// Default in v3.6.  Meaningful v2 state is not allowed.
-// The V2 files are maintained for v3.5 rollback.
-V2_DEPR_1_WRITE_ONLY = V2DeprecationEnum("write-only")
-// V2store is WIPED if found !!!
-V2_DEPR_1_WRITE_ONLY_DROP = V2DeprecationEnum("write-only-drop-data")
-// V2store is neither written nor read. Usage of this configuration is blocking
-// ability to rollback to etcd v3.5.
-V2_DEPR_2_GONE = V2DeprecationEnum("gone")
+	// Default in v3.5.  Issues a warning if v2store have meaningful content.
+	V2_DEPR_0_NOT_YET = V2DeprecationEnum("not-yet")
+	// Default in v3.6.  Meaningful v2 state is not allowed.
+	// The V2 files are maintained for v3.5 rollback.
+	V2_DEPR_1_WRITE_ONLY = V2DeprecationEnum("write-only")
+	// V2store is WIPED if found !!!
+	V2_DEPR_1_WRITE_ONLY_DROP = V2DeprecationEnum("write-only-drop-data")
+	// V2store is neither written nor read. Usage of this configuration is blocking
+	// ability to rollback to etcd v3.5.
+	V2_DEPR_2_GONE = V2DeprecationEnum("gone")
 
-V2_DEPR_DEFAULT = V2_DEPR_0_NOT_YET
+	V2_DEPR_DEFAULT = V2_DEPR_0_NOT_YET
 )
 
 func (e V2DeprecationEnum) IsAtLeast(v2d V2DeprecationEnum) bool {
- return e.level() >= v2d.level()
+	return e.level() >= v2d.level()
 }
 
 func (e V2DeprecationEnum) level() int {
 	switch e {
-	case V2_DEPR_0_NOT_YET: return 0
-	case V2_DEPR_1_WRITE_ONLY: return 1
-	case V2_DEPR_1_WRITE_ONLY_DROP: return 2
-	case V2_DEPR_2_GONE: return 3
+	case V2_DEPR_0_NOT_YET:
+		return 0
+	case V2_DEPR_1_WRITE_ONLY:
+		return 1
+	case V2_DEPR_1_WRITE_ONLY_DROP:
+		return 2
+	case V2_DEPR_2_GONE:
+		return 3
 	}
 	panic("Unknown V2DeprecationEnum: " + e)
-}
+}

server/embed/v2_deprecation_test.go → server/config/v2_deprecation_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package embed
+package config
 
 import "testing"
 
@@ -32,7 +32,7 @@ func TestV2DeprecationEnum_IsAtLeast(t *testing.T) {
 		{V2_DEPR_1_WRITE_ONLY_DROP, V2_DEPR_1_WRITE_ONLY, true},
 	}
 	for _, tt := range tests {
-		t.Run(string(tt.e) + " >= " + string(tt.v2d), func(t *testing.T) {
+		t.Run(string(tt.e)+" >= "+string(tt.v2d), func(t *testing.T) {
 			if got := tt.e.IsAtLeast(tt.v2d); got != tt.want {
 				t.Errorf("IsAtLeast() = %v, want %v", got, tt.want)
 			}

server/embed/config.go

@@ -33,6 +33,7 @@ import (
 	"go.etcd.io/etcd/client/pkg/v3/types"
 	"go.etcd.io/etcd/pkg/v3/flags"
 	"go.etcd.io/etcd/pkg/v3/netutil"
+	"go.etcd.io/etcd/server/v3/config"
 	"go.etcd.io/etcd/server/v3/etcdserver"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/v3compactor"
 
@@ -402,7 +403,7 @@ type Config struct {
 	ExperimentalTxnModeWriteWithSharedBuffer bool `json:"experimental-txn-mode-write-with-shared-buffer"`
 
 	// V2Deprecation describes phase of API & Storage V2 support
-	V2Deprecation V2DeprecationEnum `json:"v2-deprecation"`
+	V2Deprecation config.V2DeprecationEnum `json:"v2-deprecation"`
 }
 
 // configYAML holds the config suitable for yaml parsing
@@ -498,7 +499,7 @@ func NewConfig() *Config {
 		ExperimentalMemoryMlock:                  false,
 		ExperimentalTxnModeWriteWithSharedBuffer: true,
 
-		V2Deprecation: V2_DEPR_DEFAULT,
+		V2Deprecation: config.V2_DEPR_DEFAULT,
 	}
 	cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name)
 	return cfg
@@ -800,12 +801,11 @@ func (cfg Config) InitialClusterFromName(name string) (ret string) {
 func (cfg Config) IsNewCluster() bool { return cfg.ClusterState == ClusterStateFlagNew }
 func (cfg Config) ElectionTicks() int { return int(cfg.ElectionMs / cfg.TickMs) }
 
-func (cfg Config) V2DeprecationEffective() V2DeprecationEnum {
+func (cfg Config) V2DeprecationEffective() config.V2DeprecationEnum {
 	if cfg.V2Deprecation == "" {
-		return V2_DEPR_DEFAULT
-	} else {
-		return cfg.V2Deprecation
+		return config.V2_DEPR_DEFAULT
 	}
+	return cfg.V2Deprecation
 }
 
 func (cfg Config) defaultPeerHost() bool {

server/embed/etcd.go

@@ -226,6 +226,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
 		ExperimentalMemoryMlock:                  cfg.ExperimentalMemoryMlock,
 		ExperimentalTxnModeWriteWithSharedBuffer: cfg.ExperimentalTxnModeWriteWithSharedBuffer,
 		ExperimentalBootstrapDefragThresholdMegabytes: cfg.ExperimentalBootstrapDefragThresholdMegabytes,
+		V2Deprecation: cfg.V2DeprecationEffective(),
 	}
 
 	if srvcfg.ExperimentalEnableDistributedTracing {
@@ -696,6 +697,9 @@ func (e *Etcd) serveClients() (err error) {
 	// Start a client server goroutine for each listen address
 	var h http.Handler
 	if e.Config().EnableV2 {
+		if e.Config().V2DeprecationEffective().IsAtLeast(config.V2_DEPR_1_WRITE_ONLY) {
+			return fmt.Errorf("--enable-v2 and --v2-deprecation=%s are mutually exclusive", e.Config().V2DeprecationEffective())
+		}
 		e.cfg.logger.Warn("Flag `enable-v2` is deprecated and will get removed in etcd 3.6.")
 		if len(e.Config().ExperimentalEnableV2V3) > 0 {
 			e.cfg.logger.Warn("Flag `experimental-enable-v2v3` is deprecated and will get removed in etcd 3.6.")

server/etcdmain/config.go

@@ -27,6 +27,7 @@ import (
 	"go.etcd.io/etcd/api/v3/version"
 	"go.etcd.io/etcd/client/pkg/v3/logutil"
 	"go.etcd.io/etcd/pkg/v3/flags"
+	cconfig "go.etcd.io/etcd/server/v3/config"
 	"go.etcd.io/etcd/server/v3/embed"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp"
 
@@ -121,10 +122,10 @@ func newConfig() *config {
 			proxyFlagOn,
 		),
 		v2deprecation: flags.NewSelectiveStringsValue(
-			string(embed.V2_DEPR_0_NOT_YET),
-			string(embed.V2_DEPR_1_WRITE_ONLY),
-			string(embed.V2_DEPR_1_WRITE_ONLY_DROP),
-			string(embed.V2_DEPR_2_GONE)),
+			string(cconfig.V2_DEPR_0_NOT_YET),
+			string(cconfig.V2_DEPR_1_WRITE_ONLY),
+			string(cconfig.V2_DEPR_1_WRITE_ONLY_DROP),
+			string(cconfig.V2_DEPR_2_GONE)),
 	}
 
 	fs := cfg.cf.flagSet
@@ -343,7 +344,7 @@ func (cfg *config) parse(arguments []string) error {
 	}
 
 	if cfg.ec.V2Deprecation == "" {
-		cfg.ec.V2Deprecation = embed.V2_DEPR_DEFAULT
+		cfg.ec.V2Deprecation = cconfig.V2_DEPR_DEFAULT
 	}
 
 	// now logger is set up
@@ -400,7 +401,7 @@ func (cfg *config) configFromCmdLine() error {
 	cfg.cp.Fallback = cfg.cf.fallback.String()
 	cfg.cp.Proxy = cfg.cf.proxy.String()
 
-	cfg.ec.V2Deprecation = embed.V2DeprecationEnum(cfg.cf.v2deprecation.String())
+	cfg.ec.V2Deprecation = cconfig.V2DeprecationEnum(cfg.cf.v2deprecation.String())
 
 	// disable default advertise-client-urls if lcurls is set
 	missingAC := flags.IsSet(cfg.cf.flagSet, "listen-client-urls") && !flags.IsSet(cfg.cf.flagSet, "advertise-client-urls")

server/etcdmain/help.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"strconv"
 
+	cconfig "go.etcd.io/etcd/server/v3/config"
 	"go.etcd.io/etcd/server/v3/embed"
 	"golang.org/x/crypto/bcrypt"
 )
@@ -124,8 +125,8 @@ Clustering:
     Interpret 'auto-compaction-retention' one of: periodic|revision. 'periodic' for duration based retention, defaulting to hours if no time unit is provided (e.g. '5m'). 'revision' for revision number based retention.
   --enable-v2 '` + strconv.FormatBool(embed.DefaultEnableV2) + `'
     Accept etcd V2 client requests. Deprecated and to be decommissioned in v3.6.
-  --v2-deprecation '` + string(embed.V2_DEPR_DEFAULT) + `'
-    Phase of v2store deprecation. Allows to optin for higher compatibility mode.
+  --v2-deprecation '` + string(cconfig.V2_DEPR_DEFAULT) + `'
+    Phase of v2store deprecation. Allows to opt-in for higher compatibility mode.
     Supported values:
       'not-yet'                // Issues a warning if v2store have meaningful content (default in v3.5)
       'write-only'             // Custom v2 state is not allowed (planned default in v3.6)

server/etcdserver/server.go

@@ -479,6 +479,11 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) {
 				cfg.Logger.Panic("failed to recover from snapshot", zap.Error(err))
 			}
 
+			if err = assertNoV2StoreContent(cfg.Logger, st, cfg.V2Deprecation); err != nil {
+				cfg.Logger.Error("illegal v2store content", zap.Error(err))
+				return nil, err
+			}
+
 			cfg.Logger.Info(
 				"recovered v2 store from snapshot",
 				zap.Uint64("snapshot-index", snapshot.Metadata.Index),
@@ -496,6 +501,8 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) {
 				zap.Int64("backend-size-in-use-bytes", s2),
 				zap.String("backend-size-in-use", humanize.Bytes(uint64(s2))),
 			)
+		} else {
+			cfg.Logger.Info("No snapshot found. Recovering WAL from scratch!")
 		}
 
 		if !cfg.ForceNewCluster {
@@ -662,6 +669,23 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) {
 	return srv, nil
 }
 
+// assertNoV2StoreContent -> depending on the deprecation stage, warns or report an error
+// if the v2store contains custom content.
+func assertNoV2StoreContent(lg *zap.Logger, st v2store.Store, deprecationStage config.V2DeprecationEnum) error {
+	metaOnly, err := membership.IsMetaStoreOnly(st)
+	if err != nil {
+		return err
+	}
+	if metaOnly {
+		return nil
+	}
+	if deprecationStage.IsAtLeast(config.V2_DEPR_1_WRITE_ONLY) {
+		return fmt.Errorf("detected disallowed custom content in v2store for stage --v2-deprecation=%s", deprecationStage)
+	}
+	lg.Warn("detected custom v2store content. Etcd v3.5 is the last version allowing to access it using API v2. Please remove the content.")
+	return nil
+}
+
 func (s *EtcdServer) Logger() *zap.Logger {
 	s.lgMu.RLock()
 	l := s.lg
@@ -1252,6 +1276,10 @@ func (s *EtcdServer) applySnapshot(ep *etcdProgress, apply *apply) {
 		lg.Panic("failed to restore v2 store", zap.Error(err))
 	}
 
+	if err := assertNoV2StoreContent(lg, s.v2store, s.Cfg.V2Deprecation); err != nil {
+		lg.Panic("illegal v2store content", zap.Error(err))
+	}
+
 	lg.Info("restored v2 store")
 
 	s.cluster.SetBackend(newbe)

tests/integration/cluster.go

@@ -706,6 +706,8 @@ func mustNewMember(t testutil.TB, mcfg memberConfig) *member {
 	m.InitialCorruptCheck = true
 	m.WarningApplyDuration = embed.DefaultWarningApplyDuration
 
+	m.V2Deprecation = config.V2_DEPR_DEFAULT
+
 	m.Logger = memberLogger(t, mcfg.name)
 	t.Cleanup(func() {
 		// if we didn't cleanup the logger, the consecutive test