Fix ingesters with less tokens stuck in LEAVING

CHANGELOG.md

@@ -21,6 +21,7 @@
 * [BUGFIX] Updated `golang.org/x/net` dependency to fix CVE-2022-27664. #5008
 * [BUGFIX] Fix panic when otel and xray tracing is enabled. #5044
 * [BUGFIX] Fixed no compact block got grouped in shuffle sharding grouper. #5055
+* [BUGFIX] Fixed ingesters with less tokens stuck in LEAVING. #5061
 * [BUGFIX] Tracing: Fix missing object storage span instrumentation. #5074
 
 ## 1.14.0 2022-12-02

pkg/ring/lifecycler.go

@@ -622,7 +622,7 @@ func (i *Lifecycler) initRing(ctx context.Context) error {
 		// OR unregister_on_shutdown=false
 		// if autoJoinOnStartup, move it into ACTIVE to ensure the ingester joins the ring.
 		// else set to PENDING
-		if instanceDesc.State == LEAVING && len(instanceDesc.Tokens) == i.cfg.NumTokens {
+		if instanceDesc.State == LEAVING && len(instanceDesc.Tokens) != 0 {
 			if i.autoJoinOnStartup {
 				instanceDesc.State = ACTIVE
 			} else {

pkg/ring/lifecycler_test.go

@@ -794,6 +794,59 @@ func TestJoinInLeavingState(t *testing.T) {
 	})
 }
 
+func TestJoinInLeavingStateAndLessTokens(t *testing.T) {
+	ringStore, closer := consul.NewInMemoryClient(GetCodec(), log.NewNopLogger(), nil)
+	t.Cleanup(func() { assert.NoError(t, closer.Close()) })
+
+	var ringConfig Config
+	flagext.DefaultValues(&ringConfig)
+	ringConfig.KVStore.Mock = ringStore
+
+	r, err := New(ringConfig, "ingester", ringKey, log.NewNopLogger(), nil)
+	require.NoError(t, err)
+	require.NoError(t, services.StartAndAwaitRunning(context.Background(), r))
+	defer services.StopAndAwaitTerminated(context.Background(), r) //nolint:errcheck
+
+	cfg := testLifecyclerConfig(ringConfig, "ing1")
+	cfg.NumTokens = 3
+	cfg.MinReadyDuration = 1 * time.Nanosecond
+
+	// Set state as LEAVING and 1 less token because of conflict resolution
+	err = r.KVClient.CAS(context.Background(), ringKey, func(in interface{}) (interface{}, bool, error) {
+		r := &Desc{
+			Ingesters: map[string]InstanceDesc{
+				"ing1": {
+					State:  LEAVING,
+					Tokens: []uint32{1, 4},
+				},
+				"ing2": {
+					Tokens: []uint32{2, 3, 5},
+				},
+			},
+		}
+
+		return r, true, nil
+	})
+	require.NoError(t, err)
+
+	l1, err := NewLifecycler(cfg, &nopFlushTransferer{}, "ingester", ringKey, true, true, log.NewNopLogger(), nil)
+	require.NoError(t, err)
+	require.NoError(t, services.StartAndAwaitRunning(context.Background(), l1))
+
+	// Check that the lifecycler was able to join after coming up in LEAVING
+	test.Poll(t, 1000*time.Millisecond, true, func() interface{} {
+		d, err := r.KVClient.Get(context.Background(), ringKey)
+		require.NoError(t, err)
+
+		desc, ok := d.(*Desc)
+		return ok &&
+			len(desc.Ingesters) == 2 &&
+			desc.Ingesters["ing1"].State == ACTIVE &&
+			len(desc.Ingesters["ing1"].Tokens) == 2 &&
+			len(desc.Ingesters["ing2"].Tokens) == cfg.NumTokens
+	})
+}
+
 // JoinInJoiningState ensures that if the lifecycler starts up and the ring already has it in a JOINING state that it still is able to auto join
 func TestJoinInJoiningState(t *testing.T) {
 	ringStore, closer := consul.NewInMemoryClient(GetCodec(), log.NewNopLogger(), nil)