Retry teleporter POST and config PATCH requests

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/Microsoft/hcsshim v0.11.5 // indirect
+	github.com/avast/retry-go v3.0.0+incompatible // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/containerd/containerd v1.7.18 // indirect
 	github.com/containerd/errdefs v0.1.0 // indirect

go.sum

@@ -8,6 +8,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/Microsoft/hcsshim v0.11.5 h1:haEcLNpj9Ka1gd3B3tAEs9CpE0c+1IhoL59w/exYU38=
 github.com/Microsoft/hcsshim v0.11.5/go.mod h1:MV8xMfmECjl5HdO7U/3/hFVnkmSBjAjmA09d4bExKcU=
+github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
+github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
 github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
 github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao=

internal/pihole/client.go

@@ -28,7 +28,7 @@ func NewClient(piHole model.PiHole, httpClient *http.Client) Client {
 }
 
 type Client interface {
-	Authenticate() error
+	PostAuth() error
 	DeleteSession() error
 	GetVersion() (*model.VersionResponse, error)
 	GetTeleporter() ([]byte, error)
@@ -61,8 +61,8 @@ func (a *auth) verify() error {
 	return nil
 }
 
-func (client *client) Authenticate() error {
-	client.logger.Debug().Msg("Authenticate")
+func (client *client) PostAuth() error {
+	client.logger.Debug().Msg("PostAuth")
 	authResponse := model.AuthResponse{}
 
 	reqBytes, err := json.Marshal(model.AuthRequest{Password: client.piHole.Password})

internal/pihole/client_test.go

@@ -29,7 +29,7 @@ type clientTestSuite struct {
 
 func (suite *clientTestSuite) SetupTest() {
 	client := createClient(piHole)
-	err := client.Authenticate()
+	err := client.PostAuth()
 	require.NoError(suite.T(), err)
 	suite.client = client
 }
@@ -39,7 +39,7 @@ func TestClientIntegration(t *testing.T) {
 }
 
 func (suite *clientTestSuite) TestClient_Authenticate() {
-	err := suite.client.Authenticate()
+	err := suite.client.PostAuth()
 
 	assert.NoError(suite.T(), err)
 }

internal/sync/retry.go

@@ -0,0 +1,26 @@
+package sync
+
+import (
+	"fmt"
+	"github.com/avast/retry-go"
+	"github.com/rs/zerolog/log"
+	"time"
+)
+
+const (
+	attempts uint          = 3
+	delay    time.Duration = 1 * time.Second
+)
+
+func withRetry(retryFunc func() error) error {
+	return retry.Do(
+		func() error {
+			return retryFunc()
+		},
+		retry.Attempts(attempts),
+		retry.Delay(delay),
+		retry.OnRetry(func(n uint, err error) {
+			log.Debug().Msg(fmt.Sprintf("Retrying(%d): %v", n+1, err))
+		}),
+	)
+}

internal/sync/sync.go

@@ -27,6 +27,7 @@ func NewTarget(primary pihole.Client, replicas []pihole.Client) Target {
 
 func (target *target) FullSync() error {
 	log.Info().Int("replicas", len(target.Replicas)).Msg("Running full sync")
+
 	if err := target.authenticate(); err != nil {
 		return fmt.Errorf("authenticate: %w", err)
 	}
@@ -66,12 +67,12 @@ func (target *target) ManualSync(syncSettings *config.SyncSettings) error {
 
 func (target *target) authenticate() (err error) {
 	log.Info().Msg("Authenticating clients...")
-	if err := target.Primary.Authenticate(); err != nil {
+	if err := target.Primary.PostAuth(); err != nil {
 		return err
 	}
 
 	for _, replica := range target.Replicas {
-		if err := replica.Authenticate(); err != nil {
+		if err := replica.PostAuth(); err != nil {
 			return err
 		}
 	}
@@ -107,7 +108,9 @@ func (target *target) syncTeleporters(manualGravity *config.ManualGravity) error
 	}
 
 	for _, replica := range target.Replicas {
-		if err := replica.PostTeleporter(conf, teleporterRequest); err != nil {
+		if err := withRetry(func() error {
+			return replica.PostTeleporter(conf, teleporterRequest)
+		}); err != nil {
 			return err
 		}
 	}
@@ -125,7 +128,9 @@ func (target *target) syncConfigs(manualConfig *config.ManualConfig) error {
 	configRequest := createPatchConfigRequest(manualConfig, configResponse)
 
 	for _, replica := range target.Replicas {
-		if err := replica.PatchConfig(configRequest); err != nil {
+		if err := withRetry(func() error {
+			return replica.PatchConfig(configRequest)
+		}); err != nil {
 			return err
 		}
 	}

internal/sync/sync_test.go

@@ -19,12 +19,12 @@ func TestTarget_FullSync(t *testing.T) {
 
 	primary.
 		EXPECT().
-		Authenticate().
+		PostAuth().
 		Times(1).
 		Return(nil)
 	replica.
 		EXPECT().
-		Authenticate().
+		PostAuth().
 		Times(1).
 		Return(nil)
 
@@ -86,12 +86,12 @@ func TestTarget_ManualSync(t *testing.T) {
 
 	primary.
 		EXPECT().
-		Authenticate().
+		PostAuth().
 		Times(1).
 		Return(nil)
 	replica.
 		EXPECT().
-		Authenticate().
+		PostAuth().
 		Times(1).
 		Return(nil)
 
@@ -143,12 +143,12 @@ func Test_target_authenticate(t *testing.T) {
 
 	primary.
 		EXPECT().
-		Authenticate().
+		PostAuth().
 		Times(1).
 		Return(nil)
 	replica.
 		EXPECT().
-		Authenticate().
+		PostAuth().
 		Times(1).
 		Return(nil)