Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Change to report total deleted by default for stream info. #2109

Merged
merged 1 commit into from
Apr 13, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions server/filestore.go
Original file line number Diff line number Diff line change
Expand Up @@ -2673,6 +2673,7 @@ func (fs *fileStore) State() StreamState {
sort.Slice(state.Deleted, func(i, j int) bool {
return state.Deleted[i] < state.Deleted[j]
})
state.NumDeleted = len(state.Deleted)
}
return state
}
Expand Down
32 changes: 27 additions & 5 deletions server/jetstream_api.go
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,11 @@ type JSApiStreamDeleteResponse struct {

const JSApiStreamDeleteResponseType = "io.nats.jetstream.api.v1.stream_delete_response"

// JSApiStreamInfoRequest.
type JSApiStreamInfoRequest struct {
DeletedDetails bool `json:"deleted_details,omitempty"`
}

// JSApiStreamInfoResponse.
type JSApiStreamInfoResponse struct {
ApiResponse
Expand Down Expand Up @@ -1546,7 +1551,13 @@ func (s *Server) jsStreamListRequest(sub *subscription, c *client, subject, repl
}

for _, mset := range msets[offset:] {
resp.Streams = append(resp.Streams, &StreamInfo{Created: mset.createdTime(), State: mset.state(), Config: mset.config(), Mirror: mset.mirrorInfo(), Sources: mset.sourcesInfo()})
resp.Streams = append(resp.Streams, &StreamInfo{
Created: mset.createdTime(),
State: mset.state(),
Config: mset.config(),
Mirror: mset.mirrorInfo(),
Sources: mset.sourcesInfo()},
)
if len(resp.Streams) >= JSApiListLimit {
break
}
Expand Down Expand Up @@ -1621,10 +1632,16 @@ func (s *Server) jsStreamInfoRequest(sub *subscription, c *client, subject, repl
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
return
}

var details bool
if !isEmptyRequest(msg) {
resp.Error = jsNotEmptyRequestErr
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
return
var req JSApiStreamInfoRequest
if err := json.Unmarshal(msg, &req); err != nil {
resp.Error = jsInvalidJSONErr
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
return
}
details = req.DeletedDetails
}

mset, err := acc.lookupStream(streamName)
Expand All @@ -1637,7 +1654,12 @@ func (s *Server) jsStreamInfoRequest(sub *subscription, c *client, subject, repl

js, _ := s.getJetStreamCluster()

resp.StreamInfo = &StreamInfo{Created: mset.createdTime(), State: mset.state(), Config: config, Cluster: js.clusterInfo(mset.raftGroup())}
resp.StreamInfo = &StreamInfo{
Created: mset.createdTime(),
State: mset.stateWithDetail(details),
Config: config,
Cluster: js.clusterInfo(mset.raftGroup()),
}
if mset.isMirror() {
resp.StreamInfo.Mirror = mset.mirrorInfo()
} else if mset.hasSources() {
Expand Down
73 changes: 72 additions & 1 deletion server/jetstream_cluster_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5442,7 +5442,6 @@ func TestJetStreamClusterAckPendingWithMaxRedelivered(t *testing.T) {

// Send in 100 messages.
msg, toSend := []byte("HELLO"), 100
rand.Read(msg)

for i := 0; i < toSend; i++ {
if _, err = js.Publish("foo", msg); err != nil {
Expand Down Expand Up @@ -5569,6 +5568,78 @@ func TestJetStreamClusterSuperClusterAndLeafNodesWithSharedSystemAccount(t *test
}
}

func TestJetStreamClusterStreamInfoDeletedDetails(t *testing.T) {
c := createJetStreamClusterExplicit(t, "R2", 2)
defer c.shutdown()

// Client based API
nc, js := jsClientConnect(t, c.randomServer())
defer nc.Close()

_, err := js.AddStream(&nats.StreamConfig{
Name: "TEST",
Subjects: []string{"foo"},
Replicas: 1,
})
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}

// Send in 10 messages.
msg, toSend := []byte("HELLO"), 10

for i := 0; i < toSend; i++ {
if _, err = js.Publish("foo", msg); err != nil {
t.Fatalf("Unexpected publish error: %v", err)
}
}

// Now remove some messages.
deleteMsg := func(seq uint64) {
if err := js.DeleteMsg("TEST", seq); err != nil {
t.Fatalf("Unexpected error: %v", err)
}
}
deleteMsg(2)
deleteMsg(4)
deleteMsg(6)

// Need to do these via direct server request for now.
resp, err := nc.Request(fmt.Sprintf(JSApiStreamInfoT, "TEST"), nil, time.Second)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
var si StreamInfo
if err = json.Unmarshal(resp.Data, &si); err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if si.State.NumDeleted != 3 {
t.Fatalf("Expected %d deleted, got %d", 3, si.State.NumDeleted)
}
if len(si.State.Deleted) != 0 {
t.Fatalf("Expected not deleted details, but got %+v", si.State.Deleted)
}

// Now request deleted details.
req := JSApiStreamInfoRequest{DeletedDetails: true}
b, _ := json.Marshal(req)

resp, err = nc.Request(fmt.Sprintf(JSApiStreamInfoT, "TEST"), b, time.Second)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
if err = json.Unmarshal(resp.Data, &si); err != nil {
t.Fatalf("Unexpected error: %v", err)
}

if si.State.NumDeleted != 3 {
t.Fatalf("Expected %d deleted, got %d", 3, si.State.NumDeleted)
}
if len(si.State.Deleted) != 3 {
t.Fatalf("Expected deleted details, but got %+v", si.State.Deleted)
}
}

// Support functions

// Used to setup superclusters for tests.
Expand Down
4 changes: 3 additions & 1 deletion server/jetstream_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8274,9 +8274,11 @@ func TestJetStreamDeleteMsg(t *testing.T) {
}
expectedState.FirstTime = sm.Time
expectedState.Deleted = nil
expectedState.NumDeleted = 0

afterState := mset.state()
afterState.Deleted = nil
afterState.NumDeleted = 0

// Ignore first time in this test.
if !reflect.DeepEqual(afterState, expectedState) {
Expand Down Expand Up @@ -8326,7 +8328,7 @@ func TestJetStreamDeleteMsg(t *testing.T) {

expected := StreamState{Msgs: 6, Bytes: 6 * bytesPerMsg, FirstSeq: 12, LastSeq: 20}
state = mset.state()
state.FirstTime, state.LastTime, state.Deleted = time.Time{}, time.Time{}, nil
state.FirstTime, state.LastTime, state.Deleted, state.NumDeleted = time.Time{}, time.Time{}, nil, 0

if !reflect.DeepEqual(expected, state) {
t.Fatalf("State not what we expected. Expected %+v, got %+v\n", expected, state)
Expand Down
1 change: 1 addition & 0 deletions server/memstore.go
Original file line number Diff line number Diff line change
Expand Up @@ -562,6 +562,7 @@ func (ms *memStore) State() StreamState {
sort.Slice(state.Deleted, func(i, j int) bool {
return state.Deleted[i] < state.Deleted[j]
})
state.NumDeleted = len(state.Deleted)
}
return state
}
Expand Down
19 changes: 10 additions & 9 deletions server/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,15 +113,16 @@ const (

// StreamState is information about the given stream.
type StreamState struct {
Msgs uint64 `json:"messages"`
Bytes uint64 `json:"bytes"`
FirstSeq uint64 `json:"first_seq"`
FirstTime time.Time `json:"first_ts"`
LastSeq uint64 `json:"last_seq"`
LastTime time.Time `json:"last_ts"`
Deleted []uint64 `json:"deleted,omitempty"`
Lost *LostStreamData `json:"lost,omitempty"`
Consumers int `json:"consumer_count"`
Msgs uint64 `json:"messages"`
Bytes uint64 `json:"bytes"`
FirstSeq uint64 `json:"first_seq"`
FirstTime time.Time `json:"first_ts"`
LastSeq uint64 `json:"last_seq"`
LastTime time.Time `json:"last_ts"`
NumDeleted int `json:"num_deleted,omitempty"`
Deleted []uint64 `json:"deleted,omitempty"`
Lost *LostStreamData `json:"lost,omitempty"`
Consumers int `json:"consumer_count"`
}

// LostStreamData indicates msgs that have been lost.
Expand Down
10 changes: 9 additions & 1 deletion server/stream.go
Original file line number Diff line number Diff line change
Expand Up @@ -2914,14 +2914,22 @@ func (mset *stream) lookupConsumer(name string) *consumer {

// State will return the current state for this stream.
func (mset *stream) state() StreamState {
return mset.stateWithDetail(false)
}

func (mset *stream) stateWithDetail(details bool) StreamState {
mset.mu.RLock()
c, store := mset.client, mset.store
mset.mu.RUnlock()
if c == nil || store == nil {
return StreamState{}
}
// Currently rely on store.
return store.State()
state := store.State()
if !details {
state.Deleted = nil
}
return state
}

// Determines if the new proposed partition is unique amongst all consumers.
Expand Down