-
Notifications
You must be signed in to change notification settings - Fork 176
/
backend_events.go
377 lines (321 loc) · 12.6 KB
/
backend_events.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
package backend
import (
"context"
"encoding/hex"
"errors"
"fmt"
"sort"
"time"
"github.com/onflow/flow/protobuf/go/flow/entities"
execproto "github.com/onflow/flow/protobuf/go/flow/execution"
"github.com/rs/zerolog"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"github.com/onflow/flow-go/engine/access/rpc/connection"
"github.com/onflow/flow-go/engine/common/rpc"
"github.com/onflow/flow-go/engine/common/rpc/convert"
"github.com/onflow/flow-go/model/flow"
"github.com/onflow/flow-go/module/irrecoverable"
"github.com/onflow/flow-go/state/protocol"
"github.com/onflow/flow-go/storage"
)
type backendEvents struct {
headers storage.Headers
events storage.Events
executionReceipts storage.ExecutionReceipts
state protocol.State
connFactory connection.ConnectionFactory
log zerolog.Logger
maxHeightRange uint
nodeCommunicator Communicator
}
// GetEventsForHeightRange retrieves events for all sealed blocks between the start block height and
// the end block height (inclusive) that have the given type.
func (b *backendEvents) GetEventsForHeightRange(
ctx context.Context,
eventType string,
startHeight, endHeight uint64,
requiredEventEncodingVersion entities.EventEncodingVersion,
) ([]flow.BlockEvents, error) {
if endHeight < startHeight {
return nil, status.Error(codes.InvalidArgument, "start height must not be larger than end height")
}
rangeSize := endHeight - startHeight + 1 // range is inclusive on both ends
if rangeSize > uint64(b.maxHeightRange) {
return nil, status.Errorf(codes.InvalidArgument,
"requested block range (%d) exceeded maximum (%d)", rangeSize, b.maxHeightRange)
}
// get the latest sealed block header
sealed, err := b.state.Sealed().Head()
if err != nil {
// sealed block must be in the store, so throw an exception for any error
err := irrecoverable.NewExceptionf("failed to lookup sealed header: %w", err)
irrecoverable.Throw(ctx, err)
return nil, err
}
// start height should not be beyond the last sealed height
if startHeight > sealed.Height {
return nil, status.Errorf(codes.OutOfRange,
"start height %d is greater than the last sealed block height %d", startHeight, sealed.Height)
}
// limit max height to last sealed block in the chain
//
// Note: this causes unintuitive behavior for clients making requests through a proxy that
// fronts multiple nodes. With that setup, clients may receive responses for a smaller range
// than requested because the node serving the request has a slightly delayed view of the chain.
//
// An alternative option is to return an error here, but that's likely to cause more pain for
// these clients since the requests would intermittently fail. it's recommended instead to
// check the block height of the last message in the response. this will be the last block
// height searched, and can be used to determine the start height for the next range.
if endHeight > sealed.Height {
endHeight = sealed.Height
}
// find the block headers for all the blocks between min and max height (inclusive)
blockHeaders := make([]*flow.Header, 0)
for i := startHeight; i <= endHeight; i++ {
header, err := b.headers.ByHeight(i)
if err != nil {
return nil, rpc.ConvertStorageError(fmt.Errorf("failed to get events: %w", err))
}
blockHeaders = append(blockHeaders, header)
}
return b.getBlockEvents(ctx, blockHeaders, eventType, requiredEventEncodingVersion)
}
// GetEventsForBlockIDs retrieves events for all the specified block IDs that have the given type
func (b *backendEvents) GetEventsForBlockIDs(
ctx context.Context,
eventType string,
blockIDs []flow.Identifier,
requiredEventEncodingVersion entities.EventEncodingVersion,
) ([]flow.BlockEvents, error) {
if uint(len(blockIDs)) > b.maxHeightRange {
return nil, status.Errorf(codes.InvalidArgument, "requested block range (%d) exceeded maximum (%d)", len(blockIDs), b.maxHeightRange)
}
// find the block headers for all the block IDs
blockHeaders := make([]*flow.Header, 0)
for _, blockID := range blockIDs {
header, err := b.headers.ByBlockID(blockID)
if err != nil {
return nil, rpc.ConvertStorageError(fmt.Errorf("failed to get events: %w", err))
}
blockHeaders = append(blockHeaders, header)
}
return b.getBlockEvents(ctx, blockHeaders, eventType, requiredEventEncodingVersion)
}
// getBlockEvents retrieves events for all the specified blocks that have the given type
// It gets all events available on storage, and requests the rest to an execution node.
func (b *backendEvents) getBlockEvents(
ctx context.Context,
blockHeaders []*flow.Header,
eventType string,
requiredEventEncodingVersion entities.EventEncodingVersion,
) ([]flow.BlockEvents, error) {
localResponse, missingHeaders, err := b.getBlockEventsFromStorage(ctx, blockHeaders, eventType, requiredEventEncodingVersion)
if err != nil {
return nil, err
}
if len(missingHeaders) == 0 {
return localResponse, nil
}
enResponse, err := b.getBlockEventsFromExecutionNode(ctx, missingHeaders, eventType, requiredEventEncodingVersion)
if err != nil {
return nil, err
}
// sort ascending by block height
// this is needed because some blocks may be retrieved from storage and others from execution nodes.
// most likely, the earlier blocks will all be found in local storage, but that's not guaranteed,
// especially for nodes started after a spork, or once pruning is enabled.
// Note: this may not match the order of the original request for clients using GetEventsForBlockIDs
// that provide out of order block IDs
response := append(localResponse, enResponse...)
sort.Slice(response, func(i, j int) bool {
return response[i].BlockHeight < response[j].BlockHeight
})
return response, nil
}
// getBlockEventsFromStorage retrieves events for all the specified blocks that have the given type
// from the local storage
func (b *backendEvents) getBlockEventsFromStorage(
ctx context.Context,
blockHeaders []*flow.Header,
eventType string,
requiredEventEncodingVersion entities.EventEncodingVersion,
) ([]flow.BlockEvents, []*flow.Header, error) {
target := flow.EventType(eventType)
missing := make([]*flow.Header, 0)
resp := make([]flow.BlockEvents, 0)
for _, header := range blockHeaders {
if ctx.Err() != nil {
return nil, nil, rpc.ConvertError(ctx.Err(), "failed to get events from storage", codes.Canceled)
}
events, err := b.events.ByBlockID(header.ID())
if err != nil {
// Note: if there are no events for a block, an empty slice is returned
if errors.Is(err, storage.ErrNotFound) {
missing = append(missing, header)
continue
}
return nil, nil, rpc.ConvertError(err, "failed to get events from storage", codes.Internal)
}
filteredEvents := make([]flow.Event, 0)
for _, e := range events {
if e.Type != target {
continue
}
// events are encoded in CCF format in storage. convert to JSON-CDC if requested
if requiredEventEncodingVersion == entities.EventEncodingVersion_JSON_CDC_V0 {
payload, err := convert.CcfPayloadToJsonPayload(e.Payload)
if err != nil {
return nil, nil, rpc.ConvertError(err, "failed to convert event payload", codes.Internal)
}
e.Payload = payload
}
filteredEvents = append(filteredEvents, e)
}
resp = append(resp, flow.BlockEvents{
BlockID: header.ID(),
BlockHeight: header.Height,
BlockTimestamp: header.Timestamp,
Events: filteredEvents,
})
}
return resp, missing, nil
}
// getBlockEventsFromExecutionNode retrieves events for all the specified blocks that have the given type
// from an execution node
func (b *backendEvents) getBlockEventsFromExecutionNode(
ctx context.Context,
blockHeaders []*flow.Header,
eventType string,
requiredEventEncodingVersion entities.EventEncodingVersion,
) ([]flow.BlockEvents, error) {
// create an execution API request for events at block ID
blockIDs := make([]flow.Identifier, len(blockHeaders))
for i := range blockIDs {
blockIDs[i] = blockHeaders[i].ID()
}
if len(blockIDs) == 0 {
return []flow.BlockEvents{}, nil
}
req := &execproto.GetEventsForBlockIDsRequest{
Type: eventType,
BlockIds: convert.IdentifiersToMessages(blockIDs),
}
// choose the last block ID to find the list of execution nodes
lastBlockID := blockIDs[len(blockIDs)-1]
execNodes, err := executionNodesForBlockID(ctx, lastBlockID, b.executionReceipts, b.state, b.log)
if err != nil {
b.log.Error().Err(err).Msg("failed to retrieve events from execution node")
return nil, rpc.ConvertError(err, "failed to retrieve events from execution node", codes.Internal)
}
var resp *execproto.GetEventsForBlockIDsResponse
var successfulNode *flow.Identity
resp, successfulNode, err = b.getEventsFromAnyExeNode(ctx, execNodes, req)
if err != nil {
b.log.Error().Err(err).Msg("failed to retrieve events from execution nodes")
return nil, rpc.ConvertError(err, "failed to retrieve events from execution nodes", codes.Internal)
}
b.log.Trace().
Str("execution_id", successfulNode.String()).
Str("last_block_id", lastBlockID.String()).
Msg("successfully got events")
// convert execution node api result to access node api result
results, err := verifyAndConvertToAccessEvents(
resp.GetResults(),
blockHeaders,
resp.GetEventEncodingVersion(),
requiredEventEncodingVersion,
)
if err != nil {
return nil, status.Errorf(codes.Internal, "failed to verify retrieved events from execution node: %v", err)
}
return results, nil
}
// verifyAndConvertToAccessEvents converts execution node api result to access node api result, and verifies that the results contains
// results from each block that was requested
func verifyAndConvertToAccessEvents(
execEvents []*execproto.GetEventsForBlockIDsResponse_Result,
requestedBlockHeaders []*flow.Header,
from entities.EventEncodingVersion,
to entities.EventEncodingVersion,
) ([]flow.BlockEvents, error) {
if len(execEvents) != len(requestedBlockHeaders) {
return nil, errors.New("number of results does not match number of blocks requested")
}
requestedBlockHeaderSet := map[string]*flow.Header{}
for _, header := range requestedBlockHeaders {
requestedBlockHeaderSet[header.ID().String()] = header
}
results := make([]flow.BlockEvents, len(execEvents))
for i, result := range execEvents {
header, expected := requestedBlockHeaderSet[hex.EncodeToString(result.GetBlockId())]
if !expected {
return nil, fmt.Errorf("unexpected blockID from exe node %x", result.GetBlockId())
}
if result.GetBlockHeight() != header.Height {
return nil, fmt.Errorf("unexpected block height %d for block %x from exe node",
result.GetBlockHeight(),
result.GetBlockId())
}
events, err := convert.MessagesToEventsWithEncodingConversion(result.GetEvents(), from, to)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal events in event %d with encoding version %s: %w",
i, to.String(), err)
}
results[i] = flow.BlockEvents{
BlockID: header.ID(),
BlockHeight: header.Height,
BlockTimestamp: header.Timestamp,
Events: events,
}
}
return results, nil
}
// getEventsFromAnyExeNode retrieves the given events from any EN in `execNodes`.
// We attempt querying each EN in sequence. If any EN returns a valid response, then errors from
// other ENs are logged and swallowed. If all ENs fail to return a valid response, then an
// error aggregating all failures is returned.
func (b *backendEvents) getEventsFromAnyExeNode(ctx context.Context,
execNodes flow.IdentityList,
req *execproto.GetEventsForBlockIDsRequest,
) (*execproto.GetEventsForBlockIDsResponse, *flow.Identity, error) {
var resp *execproto.GetEventsForBlockIDsResponse
var execNode *flow.Identity
errToReturn := b.nodeCommunicator.CallAvailableNode(
execNodes,
func(node *flow.Identity) error {
var err error
start := time.Now()
resp, err = b.tryGetEvents(ctx, node, req)
duration := time.Since(start)
logger := b.log.With().
Str("execution_node", node.String()).
Str("event", req.GetType()).
Int("blocks", len(req.BlockIds)).
Int64("rtt_ms", duration.Milliseconds()).
Logger()
if err == nil {
// return if any execution node replied successfully
logger.Debug().Msg("Successfully got events")
execNode = node
return nil
}
logger.Err(err).Msg("failed to execute GetEvents")
return err
},
nil,
)
return resp, execNode, errToReturn
}
func (b *backendEvents) tryGetEvents(ctx context.Context,
execNode *flow.Identity,
req *execproto.GetEventsForBlockIDsRequest,
) (*execproto.GetEventsForBlockIDsResponse, error) {
execRPCClient, closer, err := b.connFactory.GetExecutionAPIClient(execNode.Address)
if err != nil {
return nil, err
}
defer closer.Close()
return execRPCClient.GetEventsForBlockIDs(ctx, req)
}