Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DPDV-6531: Retry when server does not return JSON #81

Merged
merged 3 commits into from
Mar 1, 2024
Merged

DPDV-6531: Retry when server does not return JSON #81

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
59d22e8
DPDV-6531: Retry when server does not return JSON
martin-majlis-s1 Feb 29, 2024
bdf215d
Update dependabot to update examples as well
martin-majlis-s1 Feb 29, 2024
9cd8492
Update error message in test
martin-majlis-s1 Feb 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions .github/dependabot.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,18 @@ updates:
directory: "/"
schedule:
interval: "weekly"
- package-ecosystem: "gomod"
directory: "/examples/client/"
schedule:
interval: "weekly"
- package-ecosystem: "gomod"
directory: "/examples/readme/"
schedule:
interval: "weekly"
- package-ecosystem: "gomod"
directory: "/examples/stress/"
schedule:
interval: "weekly"

- package-ecosystem: "github-actions"
directory: "/"
Expand Down
8 changes: 4 additions & 4 deletions examples/client/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ go 1.21

require (
github.com/scalyr/dataset-go v0.0.0
go.opentelemetry.io/otel v1.23.1
go.uber.org/zap v1.26.0
go.opentelemetry.io/otel v1.24.0
go.uber.org/zap v1.27.0
)

require (
Expand All @@ -14,8 +14,8 @@ require (
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/google/uuid v1.6.0 // indirect
go.opentelemetry.io/otel/metric v1.23.1 // indirect
go.opentelemetry.io/otel/trace v1.23.1 // indirect
go.opentelemetry.io/otel/metric v1.24.0 // indirect
go.opentelemetry.io/otel/trace v1.24.0 // indirect
go.uber.org/multierr v1.10.0 // indirect
golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df // indirect
)
Expand Down
4 changes: 4 additions & 0 deletions examples/client/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,20 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
go.opentelemetry.io/otel v1.23.1 h1:Za4UzOqJYS+MUczKI320AtqZHZb7EqxO00jAHE0jmQY=
go.opentelemetry.io/otel v1.23.1/go.mod h1:Td0134eafDLcTS4y+zQ26GE8u3dEuRBiBCTUIRHaikA=
go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
go.opentelemetry.io/otel/metric v1.23.1 h1:PQJmqJ9u2QaJLBOELl1cxIdPcpbwzbkjfEyelTl2rlo=
go.opentelemetry.io/otel/metric v1.23.1/go.mod h1:mpG2QPlAfnK8yNhNJAxDZruU9Y1/HubbC+KyH8FaCWI=
go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
go.opentelemetry.io/otel/trace v1.23.1 h1:4LrmmEd8AU2rFvU1zegmvqW7+kWarxtNOPyeL6HmYY8=
go.opentelemetry.io/otel/trace v1.23.1/go.mod h1:4IpnpJFwr1mo/6HL8XIPJaE9y0+u1KcVmuW7dwFSVrI=
go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME=
golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
Expand Down
8 changes: 4 additions & 4 deletions examples/readme/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ go 1.21

require (
github.com/scalyr/dataset-go v0.0.0
go.opentelemetry.io/otel v1.23.1
go.uber.org/zap v1.26.0
go.opentelemetry.io/otel v1.24.0
go.uber.org/zap v1.27.0
)

require (
Expand All @@ -14,8 +14,8 @@ require (
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/google/uuid v1.6.0 // indirect
go.opentelemetry.io/otel/metric v1.23.1 // indirect
go.opentelemetry.io/otel/trace v1.23.1 // indirect
go.opentelemetry.io/otel/metric v1.24.0 // indirect
go.opentelemetry.io/otel/trace v1.24.0 // indirect
go.uber.org/multierr v1.10.0 // indirect
golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df // indirect
)
Expand Down
4 changes: 4 additions & 0 deletions examples/readme/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,20 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
go.opentelemetry.io/otel v1.23.1 h1:Za4UzOqJYS+MUczKI320AtqZHZb7EqxO00jAHE0jmQY=
go.opentelemetry.io/otel v1.23.1/go.mod h1:Td0134eafDLcTS4y+zQ26GE8u3dEuRBiBCTUIRHaikA=
go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
go.opentelemetry.io/otel/metric v1.23.1 h1:PQJmqJ9u2QaJLBOELl1cxIdPcpbwzbkjfEyelTl2rlo=
go.opentelemetry.io/otel/metric v1.23.1/go.mod h1:mpG2QPlAfnK8yNhNJAxDZruU9Y1/HubbC+KyH8FaCWI=
go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
go.opentelemetry.io/otel/trace v1.23.1 h1:4LrmmEd8AU2rFvU1zegmvqW7+kWarxtNOPyeL6HmYY8=
go.opentelemetry.io/otel/trace v1.23.1/go.mod h1:4IpnpJFwr1mo/6HL8XIPJaE9y0+u1KcVmuW7dwFSVrI=
go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME=
golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
Expand Down
6 changes: 3 additions & 3 deletions examples/stress/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,15 @@ go 1.21

require (
github.com/scalyr/dataset-go v0.17.0
go.uber.org/zap v1.26.0
go.uber.org/zap v1.27.0
)

require (
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/cskr/pubsub v1.0.2 // indirect
github.com/google/uuid v1.6.0 // indirect
go.opentelemetry.io/otel v1.23.1 // indirect
go.opentelemetry.io/otel/metric v1.23.1 // indirect
go.opentelemetry.io/otel v1.24.0 // indirect
go.opentelemetry.io/otel/metric v1.24.0 // indirect
go.uber.org/multierr v1.10.0 // indirect
golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df // indirect
)
Expand Down
3 changes: 3 additions & 0 deletions examples/stress/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,10 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
go.opentelemetry.io/otel v1.23.1 h1:Za4UzOqJYS+MUczKI320AtqZHZb7EqxO00jAHE0jmQY=
go.opentelemetry.io/otel v1.23.1/go.mod h1:Td0134eafDLcTS4y+zQ26GE8u3dEuRBiBCTUIRHaikA=
go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
go.opentelemetry.io/otel/metric v1.23.1 h1:PQJmqJ9u2QaJLBOELl1cxIdPcpbwzbkjfEyelTl2rlo=
go.opentelemetry.io/otel/metric v1.23.1/go.mod h1:mpG2QPlAfnK8yNhNJAxDZruU9Y1/HubbC+KyH8FaCWI=
go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
go.opentelemetry.io/otel/trace v1.23.1 h1:4LrmmEd8AU2rFvU1zegmvqW7+kWarxtNOPyeL6HmYY8=
go.opentelemetry.io/otel/trace v1.23.1/go.mod h1:4IpnpJFwr1mo/6HL8XIPJaE9y0+u1KcVmuW7dwFSVrI=
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
Expand All @@ -30,6 +32,7 @@ go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME=
golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
Expand Down
12 changes: 9 additions & 3 deletions pkg/client/add_events.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,12 @@ import (
"go.uber.org/zap"
)

const (
errMsgUnableToSentRequest = "unable to send request"
errMsgUnableToReadResponse = "unable to read response"
errMsgUnableToParseResponse = "unable to parse response"
)

/*
Wrapper around: https://app.scalyr.com/help/api#addEvents
*/
Expand Down Expand Up @@ -566,7 +572,7 @@ func (client *DataSetClient) sendAddEventsBuffer(buf *buffer.Buffer) (*add_event
func (client *DataSetClient) apiCall(req *http.Request, response response.ResponseObjSetter) error {
resp, err := client.Client.Do(req)
if err != nil {
return fmt.Errorf("unable to send request: %w", err)
return fmt.Errorf("%s: %w", errMsgUnableToSentRequest, err)
}

defer func() {
Expand All @@ -591,12 +597,12 @@ func (client *DataSetClient) apiCall(req *http.Request, response response.Respon

responseBody, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("unable to read response: %w", err)
return fmt.Errorf("%s: %w", errMsgUnableToReadResponse, err)
}

err = json.Unmarshal(responseBody, &response)
if err != nil {
return fmt.Errorf("unable to parse response body: %w, url: %s, response: %s", err, client.addEventsEndpointUrl, truncateText(string(responseBody), 1000))
return fmt.Errorf("%s: %w, url: %s, status: %d, response: %s", errMsgUnableToParseResponse, err, client.addEventsEndpointUrl, resp.StatusCode, truncateText(string(responseBody), 1000))
}

response.SetResponseObj(resp)
Expand Down
75 changes: 65 additions & 10 deletions pkg/client/add_events_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -342,11 +342,6 @@ func TestAddEventsRetry(t *testing.T) {
assert.Equal(t, 1.0, stats.Buffers.SuccessRate())
assert.Equal(t, 1.0/float64(succeedInAttempt), stats.Transfer.SuccessRate())
assert.Equal(t, uint64(1), stats.Transfer.BuffersProcessed())
/* TODO: on my Mac it's 337 in GitHub action on ubuntu-latest it's 339
assert.Equal(t, uint64(0x3f3), stats.Transfer.BytesSent())
assert.Equal(t, uint64(0x151), stats.Transfer.BytesAccepted())
assert.Equal(t, 337.0, stats.Transfer.AvgBufferBytes())
*/
}

func TestAddEventsRetryAfterSec(t *testing.T) {
Expand Down Expand Up @@ -427,8 +422,6 @@ func TestAddEventsRetryAfterSec(t *testing.T) {
assert.Equal(t, attempt.Load(), int32(2))
assert.Nil(t, err1)
assert.Nil(t, sc.LastError())
// info1 := httpmock.GetCallCountInfo()
// assert.CmpDeeply(info1, map[string]int{"POST https://example.com/api/addEvents": 2})

// send second request to make sure that nothing is blocked
event2 := &add_events.Event{Thread: "5", Sev: 3, Ts: "0", Attrs: map[string]interface{}{"message": "test - 22"}}
Expand Down Expand Up @@ -507,8 +500,70 @@ func TestAddEventsRetryAfterTime(t *testing.T) {
assert.True(t, wasSuccessful.Load())
assert.Nil(t, err)
assert.Nil(t, sc.LastError())
// info := httpmock.GetCallCountInfo()
// assert.CmpDeeply(info, map[string]int{"POST https://example.com/api/addEvents": 2})
}

func TestAddEventsRetryWhenNonJSONResponseIsReturned(t *testing.T) {
attempt.Store(0)
wasSuccessful := atomic.Bool{}
wasSuccessful.Store(false)
const succeedInAttempt = int32(3)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
attempt.Add(1)
_, err := extract(req)

assert.Nil(t, err, "Error reading request: %v", err)

payload := make([]byte, 0)
if attempt.Load() < succeedInAttempt {
w.WriteHeader(500)
payload, err = json.Marshal("this is not JSON")
} else {
wasSuccessful.Store(true)
payload, err = json.Marshal(map[string]interface{}{
"status": "success",
"bytesCharged": 42,
})
}

assert.NoError(t, err)
l, err := w.Write(payload)
assert.Greater(t, l, 1)
assert.NoError(t, err)
}))
defer server.Close()

config := newDataSetConfig(server.URL, *newBufferSettings(
buffer_config.WithRetryMaxElapsedTime(10*RetryBase),
buffer_config.WithRetryInitialInterval(RetryBase),
buffer_config.WithRetryMaxInterval(RetryBase),
), server_host_config.NewDefaultDataSetServerHostSettings())
sc, err := NewClient(config, &http.Client{}, zap.Must(zap.NewDevelopment()), nil, nil)
require.Nil(t, err)

event1 := &add_events.Event{Thread: "5", Sev: 3, Ts: "0", Attrs: map[string]interface{}{"message": "test - 1"}}
eventBundle1 := &add_events.EventBundle{Event: event1, Thread: &add_events.Thread{Id: "5", Name: "fred"}}
err = sc.AddEvents([]*add_events.EventBundle{eventBundle1})
assert.Nil(t, err)
err = sc.Shutdown()
assert.Nil(t, err)
assert.True(t, wasSuccessful.Load())
assert.Equal(t, attempt.Load(), succeedInAttempt)

stats := sc.Statistics()
assert.Equal(t, uint64(1), stats.Events.Enqueued())
assert.Equal(t, uint64(1), stats.Events.Processed())
assert.Equal(t, uint64(0), stats.Events.Waiting())
assert.Equal(t, uint64(0), stats.Events.Dropped())
assert.Equal(t, uint64(0), stats.Events.Broken())
assert.Equal(t, 1.0, stats.Events.SuccessRate())
assert.Equal(t, uint64(1), stats.Buffers.Enqueued())
assert.Equal(t, uint64(1), stats.Buffers.Processed())
assert.Equal(t, uint64(0), stats.Buffers.Waiting())
assert.Equal(t, uint64(0), stats.Buffers.Dropped())
assert.Equal(t, uint64(0), stats.Buffers.Broken())
assert.Equal(t, 1.0, stats.Buffers.SuccessRate())
assert.Equal(t, 1.0/float64(succeedInAttempt), stats.Transfer.SuccessRate())
assert.Equal(t, uint64(1), stats.Transfer.BuffersProcessed())
}

func TestAddEventsLargeEvent(t *testing.T) {
Expand Down Expand Up @@ -827,7 +882,7 @@ func TestAddEventsLogResponseBodyOnInvalidJson(t *testing.T) {
lastError := sc.LastError()

assert.NotNil(t, lastError)
assert.Equal(t, fmt.Errorf("unable to parse response body: invalid character '<' looking for beginning of value, url: %s, response: <html>not valid json</html>", sc.addEventsEndpointUrl).Error(), lastError.Error())
assert.Equal(t, fmt.Errorf("unable to parse response: invalid character '<' looking for beginning of value, url: %s, status: 503, response: <html>not valid json</html>", sc.addEventsEndpointUrl).Error(), lastError.Error())

assert.NotNil(t, err)
assert.Errorf(t, err, "some buffers were dropped during finishing - 1")
Expand Down
14 changes: 6 additions & 8 deletions pkg/client/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ import (
)

const (
HttpErrorCannotConnect = 600
HttpErrorHasErrorMessage = 499
HttpErrorCannotConnect = 600
HttpErrorCannotProcess = 601
)

// isOkStatus returns true if status code is 200, false otherwise.
Expand Down Expand Up @@ -366,13 +366,11 @@ func (client *DataSetClient) sendBufferWithRetryPolicy(buf *buffer.Buffer) bool
if err != nil {
client.Logger.Error("unable to send addEvents buffers", zap.Error(err))
client.setLastErrorTimestamp(time.Now())
if !strings.Contains(err.Error(), "Unable to send request") {
lastHttpStatus = HttpErrorHasErrorMessage
client.LastHttpStatus.Store(lastHttpStatus)
client.onBufferDrop(buf, lastHttpStatus, err)
return false // exit loop (failed to send buffer)
if strings.Contains(err.Error(), errMsgUnableToSentRequest) {
lastHttpStatus = HttpErrorCannotConnect
} else {
lastHttpStatus = HttpErrorCannotProcess
}
lastHttpStatus = HttpErrorCannotConnect
client.LastHttpStatus.Store(lastHttpStatus)
}
zaps := make([]zap.Field, 0)
Expand Down
Loading