Fix/lifecycler/stuck in leaving

CHANGELOG.md

@@ -16,6 +16,7 @@
 * [FEATURE] Added `global` ingestion rate limiter strategy. Deprecated `-distributor.limiter-reload-period` flag. #1766
 * [FEATURE] Added support for Microsoft Azure blob storage to be used for storing chunk data. #1913
 * [BUGFIX] Fixed unnecessary CAS operations done by the HA tracker when the jitter is enabled. #1861
+* [BUGFIX] Fixed #1904 ingesters getting stuck in a LEAVING state after coming up from an ungraceful exit. #1921
 
 ## 0.4.0 / 2019-12-02
 

pkg/ring/lifecycler.go

@@ -497,6 +497,12 @@ func (i *Lifecycler) initRing(ctx context.Context) error {
 			return ringDesc, true, nil
 		}
 
+		// If the ingester failed to clean it's ring entry up in can leave it's state in LEAVING.
+		// Move it into ACTIVE to ensure the ingester joins the ring.
+		if ingesterDesc.State == LEAVING && len(ingesterDesc.Tokens) == i.cfg.NumTokens {
+			ingesterDesc.State = ACTIVE
+		}
+
 		// We exist in the ring, so assume the ring is right and copy out tokens & state out of there.
 		i.setState(ingesterDesc.State)
 		tokens, _ := ringDesc.TokensFor(i.ID)

pkg/ring/lifecycler_test.go

@@ -418,3 +418,60 @@ func TestTokensOnDisk(t *testing.T) {
 		require.Equal(t, expTokens, actTokens)
 	}
 }
+
+// JoinInLeavingState ensures that if the lifecycler starts up and the ring already has it in a LEAVING state that it still is able to auto join
+func TestJoinInLeavingState(t *testing.T) {
+	var ringConfig Config
+	flagext.DefaultValues(&ringConfig)
+	codec := GetCodec()
+	ringConfig.KVStore.Mock = consul.NewInMemoryClient(codec)
+
+	r, err := New(ringConfig, "ingester", IngesterRingKey)
+	require.NoError(t, err)
+	defer r.Stop()
+
+	cfg := testLifecyclerConfig(ringConfig, "ing1")
+	cfg.NumTokens = 2
+	cfg.MinReadyDuration = 1 * time.Nanosecond
+
+	// Set state as LEAVING
+	err = r.KVClient.CAS(context.Background(), IngesterRingKey, func(in interface{}) (interface{}, bool, error) {
+		r := &Desc{
+			Ingesters: map[string]IngesterDesc{
+				"ing1": {
+					State:  LEAVING,
+					Tokens: []uint32{1, 4},
+				},
+			},
+			Tokens: []TokenDesc{
+				{
+					Ingester: "ing2",
+					Token:    2,
+				},
+				{
+					Ingester: "ing2",
+					Token:    3,
+				},
+			},
+		}
+
+		return r, true, nil
+	})
+	require.NoError(t, err)
+
+	l1, err := NewLifecycler(cfg, &nopFlushTransferer{}, "ingester", IngesterRingKey)
+	l1.Start()
+	require.NoError(t, err)
+
+	// Check that the lifecycler was able to join after coming up in LEAVING
+	test.Poll(t, 1000*time.Millisecond, true, func() interface{} {
+		d, err := r.KVClient.Get(context.Background(), IngesterRingKey)
+		require.NoError(t, err)
+		desc, ok := d.(*Desc)
+		return ok &&
+			len(desc.Ingesters) == 1 &&
+			desc.Ingesters["ing1"].State == ACTIVE &&
+			len(desc.Ingesters["ing1"].Tokens) == cfg.NumTokens &&
+			len(desc.Tokens) == 2
+	})
+}