From 333f1813bfc73ecc7d9eb107e16d212c562d59a8 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Mon, 27 Apr 2020 12:28:13 +0100 Subject: [PATCH] Fixed and added more regressions tests for reset counter dedup bug. Fixes https://github.com/thanos-io/thanos/issues/2401 Signed-off-by: Bartlomiej Plotka --- pkg/query/querier.go | 2 +- pkg/query/querier_test.go | 436 ++++++++++++++-- .../testdata/issue2401-seriesresponses2.json | 480 ++++++++++++++++++ 3 files changed, 862 insertions(+), 56 deletions(-) create mode 100644 pkg/query/testdata/issue2401-seriesresponses2.json diff --git a/pkg/query/querier.go b/pkg/query/querier.go index 0299374d3fe..504c5e3c5a5 100644 --- a/pkg/query/querier.go +++ b/pkg/query/querier.go @@ -52,7 +52,7 @@ type queryable struct { // Querier returns a new storage querier against the underlying proxy store API. func (q *queryable) Querier(ctx context.Context, mint, maxt int64) (storage.Querier, error) { - return newQuerier(ctx, q.logger, mint, maxt, q.replicaLabels, q.proxy, q.deduplicate, int64(q.maxResolutionMillis), q.partialResponse, q.skipChunks), nil + return newQuerier(ctx, q.logger, mint, maxt, q.replicaLabels, q.proxy, q.deduplicate, q.maxResolutionMillis, q.partialResponse, q.skipChunks), nil } type querier struct { diff --git a/pkg/query/querier_test.go b/pkg/query/querier_test.go index 58f1ecc35a0..2335984eca4 100644 --- a/pkg/query/querier_test.go +++ b/pkg/query/querier_test.go @@ -11,15 +11,17 @@ import ( "math/rand" "os" "reflect" + "sort" "strconv" "testing" - "time" "github.com/fortytw2/leaktest" "github.com/go-kit/kit/log" "github.com/pkg/errors" "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/pkg/timestamp" + "github.com/prometheus/prometheus/pkg/value" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/tsdb/chunkenc" @@ -165,21 +167,139 @@ func TestQuerier_DownsampledData(t *testing.T) { } } +var ( + expectedRealSeriesReplica0 = []sample{ + {t: 1582329611249, v: 3.395395e+06}, {t: 1582329626245, v: 3.39592e+06}, {t: 1582329641245, v: 3.396402e+06}, {t: 1582329656245, v: 3.396882e+06}, {t: 1582329671245, v: 3.397394e+06}, {t: 1582329686245, v: 3.397804e+06}, + {t: 1582329701245, v: 3.398296e+06}, {t: 1582329716245, v: 3.398878e+06}, {t: 1582329731245, v: 3.399373e+06}, {t: 1582329746245, v: 3.39992e+06}, {t: 1582329761245, v: 3.400436e+06}, {t: 1582329776245, v: 3.400974e+06}, + {t: 1582329791245, v: 3.401408e+06}, {t: 1582329806245, v: 3.401901e+06}, {t: 1582329821245, v: 3.402411e+06}, {t: 1582329836245, v: 3.402944e+06}, {t: 1582329851245, v: 3.403404e+06}, {t: 1582329866245, v: 3.403735e+06}, + {t: 1582329881245, v: 3.404301e+06}, {t: 1582329896245, v: 3.404818e+06}, {t: 1582329911245, v: 3.405327e+06}, {t: 1582329926245, v: 3.405843e+06}, {t: 1582329941245, v: 3.406297e+06}, {t: 1582329956245, v: 3.406737e+06}, + {t: 1582329971245, v: 3.407212e+06}, {t: 1582329986245, v: 3.407647e+06}, {t: 1582330001245, v: 3.407929e+06}, {t: 1582330016245, v: 3.408614e+06}, {t: 1582330031245, v: 3.409217e+06}, {t: 1582330046245, v: 3.409658e+06}, + {t: 1582330061245, v: 3.410276e+06}, {t: 1582330076245, v: 3.410795e+06}, {t: 1582330091245, v: 3.411509e+06}, {t: 1582330106245, v: 3.411974e+06}, {t: 1582330121245, v: 3.412368e+06}, {t: 1582330136245, v: 3.412863e+06}, + {t: 1582330151245, v: 3.413412e+06}, {t: 1582330166245, v: 3.413783e+06}, {t: 1582330181245, v: 3.41424e+06}, {t: 1582330196245, v: 3.414634e+06}, {t: 1582330211245, v: 3.415246e+06}, {t: 1582330226245, v: 3.415615e+06}, + {t: 1582330241245, v: 3.416287e+06}, {t: 1582330256245, v: 3.416753e+06}, {t: 1582330271245, v: 3.417229e+06}, {t: 1582330286245, v: 3.417748e+06}, {t: 1582330301245, v: 3.418185e+06}, {t: 1582330316245, v: 3.418707e+06}, + {t: 1582330331245, v: 3.419142e+06}, {t: 1582330346245, v: 3.419595e+06}, {t: 1582330361245, v: 3.420042e+06}, {t: 1582330376245, v: 3.420544e+06}, {t: 1582330391245, v: 3.420926e+06}, {t: 1582330406245, v: 3.421447e+06}, + {t: 1582330421245, v: 3.421899e+06}, {t: 1582330436245, v: 3.422238e+06}, {t: 1582330451245, v: 3.422619e+06}, {t: 1582330466245, v: 3.422958e+06}, {t: 1582330481251, v: 3.423455e+06}, {t: 1582330496245, v: 3.423991e+06}, + {t: 1582330511245, v: 3.424478e+06}, {t: 1582330526245, v: 3.425018e+06}, {t: 1582330541245, v: 3.425455e+06}, {t: 1582330556245, v: 3.42597e+06}, {t: 1582330571245, v: 3.426343e+06}, {t: 1582330586245, v: 3.426881e+06}, + {t: 1582330601245, v: 3.427435e+06}, {t: 1582330616245, v: 3.42803e+06}, {t: 1582330631261, v: 3.42861e+06}, {t: 1582330646245, v: 3.429152e+06}, {t: 1582330661245, v: 3.429711e+06}, {t: 1582330676245, v: 3.430061e+06}, + {t: 1582330691245, v: 3.430594e+06}, {t: 1582330706245, v: 3.431031e+06}, {t: 1582330721245, v: 3.431528e+06}, {t: 1582330736245, v: 3.432125e+06}, {t: 1582330751245, v: 3.432611e+06}, {t: 1582330766245, v: 3.433042e+06}, + {t: 1582330781245, v: 3.433544e+06}, {t: 1582330796245, v: 3.434109e+06}, {t: 1582330811245, v: 3.434386e+06}, {t: 1582330826245, v: 3.434891e+06}, {t: 1582330841245, v: 3.43544e+06}, {t: 1582330856245, v: 3.435977e+06}, + {t: 1582330871245, v: 3.436528e+06}, {t: 1582330886245, v: 3.437145e+06}, {t: 1582330901245, v: 3.437773e+06}, {t: 1582330916245, v: 3.438278e+06}, {t: 1582330931245, v: 3.438664e+06}, {t: 1582330946245, v: 3.439135e+06}, + {t: 1582330961245, v: 3.439635e+06}, {t: 1582330976245, v: 3.440136e+06}, {t: 1582330991245, v: 3.440572e+06}, {t: 1582331006245, v: 3.441014e+06}, {t: 1582331021256, v: 3.441373e+06}, {t: 1582331036245, v: 3.441745e+06}, + {t: 1582331051245, v: 3.442336e+06}, {t: 1582331066245, v: 3.44291e+06}, {t: 1582331081245, v: 3.443478e+06}, {t: 1582331096245, v: 3.443938e+06}, {t: 1582331111245, v: 3.444436e+06}, {t: 1582331126245, v: 3.445027e+06}, + {t: 1582331141245, v: 3.445382e+06}, {t: 1582331156245, v: 3.445853e+06}, {t: 1582331171262, v: 3.446356e+06}, {t: 1582331186245, v: 3.446899e+06}, {t: 1582331201245, v: 3.44743e+06}, {t: 1582331216245, v: 3.448002e+06}, + {t: 1582331231245, v: 3.448451e+06}, {t: 1582331246245, v: 3.448974e+06}, {t: 1582331261259, v: 3.449337e+06}, {t: 1582331276245, v: 3.449841e+06}, {t: 1582331291245, v: 3.450478e+06}, {t: 1582331306245, v: 3.450903e+06}, + {t: 1582331321245, v: 3.451404e+06}, {t: 1582331336245, v: 3.451778e+06}, {t: 1582331351245, v: 3.452278e+06}, {t: 1582331366245, v: 3.452711e+06}, {t: 1582331381245, v: 3.453142e+06}, + } + expectedRealSeriesReplica1 = []sample{ + {t: 1582329618053, v: 3.395638e+06}, {t: 1582329633053, v: 3.396163e+06}, {t: 1582329648053, v: 3.396612e+06}, {t: 1582329663053, v: 3.397174e+06}, {t: 1582329678053, v: 3.397564e+06}, {t: 1582329693053, v: 3.398039e+06}, + {t: 1582329708053, v: 3.39847e+06}, {t: 1582329723053, v: 3.399088e+06}, {t: 1582329738053, v: 3.399668e+06}, {t: 1582329753053, v: 3.400198e+06}, {t: 1582329768053, v: 3.400705e+06}, {t: 1582329783053, v: 3.401209e+06}, + {t: 1582329798053, v: 3.401606e+06}, {t: 1582329813053, v: 3.402171e+06}, {t: 1582329828054, v: 3.402601e+06}, {t: 1582329843053, v: 3.403164e+06}, {t: 1582329858053, v: 3.40357e+06}, {t: 1582329873053, v: 3.404005e+06}, + {t: 1582329888053, v: 3.404553e+06}, {t: 1582329903053, v: 3.405076e+06}, {t: 1582329918053, v: 3.40554e+06}, {t: 1582329933053, v: 3.406012e+06}, {t: 1582329948053, v: 3.406466e+06}, {t: 1582329963053, v: 3.407003e+06}, + {t: 1582329978053, v: 3.407378e+06}, {t: 1582329993053, v: 3.407728e+06}, {t: 1582330008053, v: 3.40835e+06}, {t: 1582330023053, v: 3.408935e+06}, {t: 1582330038053, v: 3.409436e+06}, {t: 1582330053053, v: 3.409904e+06}, + {t: 1582330068053, v: 3.410449e+06}, {t: 1582330083053, v: 3.411078e+06}, {t: 1582330098053, v: 3.411737e+06}, {t: 1582330113053, v: 3.41214e+06}, {t: 1582330128053, v: 3.412598e+06}, {t: 1582330143053, v: 3.413069e+06}, + {t: 1582330158053, v: 3.4136e+06}, {t: 1582330173053, v: 3.413985e+06}, {t: 1582330188053, v: 3.414443e+06}, {t: 1582330203053, v: 3.414959e+06}, {t: 1582330218053, v: 3.415411e+06}, {t: 1582330233053, v: 3.415962e+06}, + {t: 1582330248064, v: 3.416517e+06}, {t: 1582330263053, v: 3.416999e+06}, {t: 1582330278053, v: 3.417466e+06}, {t: 1582330293053, v: 3.418001e+06}, {t: 1582330308053, v: 3.418434e+06}, {t: 1582330323053, v: 3.41892e+06}, + {t: 1582330338053, v: 3.419327e+06}, {t: 1582330353053, v: 3.419814e+06}, {t: 1582330368053, v: 3.420281e+06}, {t: 1582330383053, v: 3.420671e+06}, {t: 1582330398053, v: 3.421132e+06}, {t: 1582330413058, v: 3.421673e+06}, + {t: 1582330428053, v: 3.42204e+06}, {t: 1582330443053, v: 3.422405e+06}, {t: 1582330458053, v: 3.422753e+06}, {t: 1582330473053, v: 3.423144e+06}, {t: 1582330488067, v: 3.423663e+06}, {t: 1582330503053, v: 3.424217e+06}, + {t: 1582330518053, v: 3.424743e+06}, {t: 1582330533053, v: 3.425246e+06}, {t: 1582330548053, v: 3.425641e+06}, {t: 1582330563054, v: 3.426142e+06}, {t: 1582330578053, v: 3.426571e+06}, {t: 1582330593053, v: 3.427067e+06}, + {t: 1582330608053, v: 3.427776e+06}, {t: 1582330623053, v: 3.428342e+06}, {t: 1582330638053, v: 3.428903e+06}, {t: 1582330653077, v: 3.429352e+06}, {t: 1582330668053, v: 3.429892e+06}, {t: 1582330683053, v: 3.430238e+06}, + {t: 1582330698053, v: 3.430812e+06}, {t: 1582330713053, v: 3.431257e+06}, {t: 1582330728056, v: 3.431899e+06}, {t: 1582330743053, v: 3.432422e+06}, {t: 1582330758053, v: 3.432863e+06}, {t: 1582330773053, v: 3.433215e+06}, + {t: 1582330788053, v: 3.433822e+06}, {t: 1582330803053, v: 3.434217e+06}, {t: 1582330818053, v: 3.434559e+06}, {t: 1582330833053, v: 3.43516e+06}, {t: 1582330848053, v: 3.435718e+06}, {t: 1582330863053, v: 3.436289e+06}, + {t: 1582330878053, v: 3.436776e+06}, {t: 1582330893053, v: 3.437315e+06}, {t: 1582330908053, v: 3.437971e+06}, {t: 1582330923053, v: 3.438459e+06}, {t: 1582330938053, v: 3.438912e+06}, {t: 1582330953053, v: 3.439307e+06}, + {t: 1582330968053, v: 3.439861e+06}, {t: 1582330983053, v: 3.440302e+06}, {t: 1582330998053, v: 3.440744e+06}, {t: 1582331013053, v: 3.441185e+06}, {t: 1582331028053, v: 3.441574e+06}, {t: 1582331043053, v: 3.442031e+06}, + {t: 1582331058053, v: 3.442613e+06}, {t: 1582331073053, v: 3.443145e+06}, {t: 1582331088053, v: 3.443649e+06}, {t: 1582331103053, v: 3.444181e+06}, {t: 1582331118053, v: 3.444643e+06}, {t: 1582331133053, v: 3.445203e+06}, + {t: 1582331148053, v: 3.445629e+06}, {t: 1582331163053, v: 3.446097e+06}, {t: 1582331178053, v: 3.446645e+06}, {t: 1582331193053, v: 3.447223e+06}, {t: 1582331208054, v: 3.447765e+06}, {t: 1582331223053, v: 3.448241e+06}, + {t: 1582331238053, v: 3.448659e+06}, {t: 1582331253053, v: 3.449075e+06}, {t: 1582331268053, v: 3.449566e+06}, {t: 1582331283053, v: 3.450086e+06}, {t: 1582331298053, v: 3.450673e+06}, {t: 1582331313053, v: 3.451119e+06}, + {t: 1582331328053, v: 3.451611e+06}, {t: 1582331343053, v: 3.452079e+06}, {t: 1582331358053, v: 3.452434e+06}, {t: 1582331373053, v: 3.452943e+06}, + } + expectedRealSeriesWithStaleMarkerReplica0 = []sample{ + {t: 1587691207139, v: 479638}, {t: 1587691222154, v: 479907}, {t: 1587691237139, v: 480008}, {t: 1587691252139, v: 480167}, {t: 1587691267139, v: 480472}, {t: 1587691282157, v: 480615}, {t: 1587691297139, v: 480771}, + {t: 1587691312139, v: 481027}, {t: 1587691327139, v: 481212}, {t: 1587691342159, v: 481395}, {t: 1587691357139, v: 481598}, {t: 1587691372139, v: 481786}, {t: 1587691387139, v: 482003}, {t: 1587691402141, v: 482236}, + {t: 1587691417139, v: 482508}, {t: 1587691432139, v: 482636}, {t: 1587691447139, v: 482780}, {t: 1587691462139, v: 483059}, {t: 1587691477139, v: 483357}, {t: 1587691492139, v: 483566}, {t: 1587691507139, v: 483711}, + {t: 1587691522139, v: 483838}, {t: 1587691537139, v: 484091}, {t: 1587691552139, v: 484254}, {t: 1587691567139, v: 484479}, {t: 1587691582139, v: 484748}, {t: 1587691597139, v: 484978}, {t: 1587691612139, v: 485271}, + {t: 1587691627139, v: 485488}, {t: 1587691642139, v: 485700}, {t: 1587691657139, v: 485945}, {t: 1587691672139, v: 486228}, {t: 1587691687139, v: 486588}, {t: 1587691702139, v: 486691}, {t: 1587691717139, v: 486881}, + {t: 1587691732139, v: 487046}, {t: 1587691747139, v: 487291}, {t: 1587691762177, v: 487410}, {t: 1587691777139, v: 487571}, {t: 1587691792139, v: 487799}, {t: 1587691807139, v: 488050}, {t: 1587691822139, v: 488241}, + {t: 1587691837139, v: 488424}, {t: 1587691852139, v: 488629}, {t: 1587691867139, v: 488875}, {t: 1587691882139, v: 489017}, {t: 1587691897139, v: 489254}, {t: 1587691912139, v: 489545}, {t: 1587691927139, v: 489778}, + {t: 1587691942139, v: 489912}, {t: 1587691957139, v: 490084}, {t: 1587691972139, v: 490364}, {t: 1587691987139, v: 490510}, {t: 1587692002139, v: 490744}, {t: 1587692017139, v: 490880}, {t: 1587692032139, v: 491025}, + {t: 1587692047139, v: 491297}, {t: 1587692062155, v: 491557}, {t: 1587692077139, v: 491839}, {t: 1587692092139, v: 492065}, {t: 1587692107139, v: 492234}, {t: 1587692122139, v: 492526}, {t: 1587692137139, v: 492767}, + {t: 1587692152139, v: 492967}, {t: 1587692167139, v: 493218}, {t: 1587692182139, v: 493442}, {t: 1587692197139, v: 493647}, {t: 1587692212139, v: 493920}, {t: 1587692227139, v: 494170}, {t: 1587692242139, v: 494358}, + {t: 1587692257139, v: 494632}, {t: 1587692272139, v: 494800}, {t: 1587692287139, v: 495026}, {t: 1587692302139, v: 495222}, {t: 1587692317139, v: 495433}, {t: 1587692332139, v: 495677}, {t: 1587692347139, v: 495901}, + {t: 1587692362139, v: 496107}, {t: 1587692377139, v: 496196}, {t: 1587692392139, v: 496245}, {t: 1587692407139, v: 496300}, {t: 1587692422159, v: 496365}, {t: 1587692437139, v: 496401}, {t: 1587692452139, v: 496452}, + {t: 1587692467139, v: 496532}, {t: 1587692482139, v: math.Float64frombits(value.StaleNaN)}, {t: 1587692542149, v: 5}, {t: 1587692557139, v: 101}, {t: 1587692572139, v: 312}, {t: 1587692587139, v: 508}, {t: 1587692602144, v: 725}, + {t: 1587692617139, v: 990}, {t: 1587692632139, v: 1178}, {t: 1587692647139, v: 1406}, {t: 1587692662154, v: 1640}, {t: 1587692677139, v: 1927}, {t: 1587692692139, v: 2103}, {t: 1587692707139, v: 2300}, {t: 1587692722139, v: 2482}, + {t: 1587692737139, v: 2638}, {t: 1587692752139, v: 2806}, {t: 1587692767139, v: 2979}, {t: 1587692782149, v: 3187}, {t: 1587692797139, v: 3441}, {t: 1587692812139, v: 3657}, {t: 1587692827139, v: 3827}, {t: 1587692842139, v: 3985}, + {t: 1587692857139, v: 4195}, {t: 1587692872139, v: 4427}, {t: 1587692887139, v: 4646}, {t: 1587692902139, v: 4714}, {t: 1587692917153, v: 4872}, {t: 1587692932139, v: 5131}, {t: 1587692947139, v: 5318}, {t: 1587692962139, v: 5571}, + {t: 1587692977155, v: 5748}, {t: 1587692992139, v: 6030}, {t: 1587693007139, v: 6210}, {t: 1587693022139, v: 6399}, {t: 1587693037139, v: 6658}, {t: 1587693052139, v: 6896}, {t: 1587693067139, v: 7098}, {t: 1587693082139, v: 7341}, + {t: 1587693097139, v: 7495}, {t: 1587693112139, v: 7647}, {t: 1587693127139, v: 7830}, {t: 1587693142139, v: 8058}, {t: 1587693157139, v: 8209}, {t: 1587693172139, v: 8524}, {t: 1587693187139, v: 8712}, {t: 1587693202139, v: 8904}, + {t: 1587693217139, v: 9103}, {t: 1587693232139, v: 9404}, {t: 1587693247155, v: 9556}, {t: 1587693262139, v: 9777}, {t: 1587693277139, v: 9992}, {t: 1587693292139, v: 10268}, {t: 1587693307139, v: 10478}, {t: 1587693322139, v: 10754}, + {t: 1587693337139, v: 10998}, {t: 1587693352139, v: 11249}, {t: 1587693367139, v: 11459}, {t: 1587693382139, v: 11778}, {t: 1587693397139, v: 12038}, {t: 1587693412139, v: 12238}, {t: 1587693427139, v: 12450}, {t: 1587693442163, v: 12742}, + {t: 1587693457139, v: 12945}, {t: 1587693472139, v: 13181}, {t: 1587693487139, v: 13440}, {t: 1587693502139, v: 13650}, {t: 1587693517139, v: 13966}, {t: 1587693532139, v: 14122}, {t: 1587693547139, v: 14327}, {t: 1587693562139, v: 14592}, + {t: 1587693577139, v: 14782}, {t: 1587693592139, v: 14956}, + } + expectedRealSeriesWithStaleMarkerReplica1 = []sample{ + {t: 1587691205791, v: 479625}, {t: 1587691220805, v: 479866}, {t: 1587691235791, v: 480008}, {t: 1587691250791, v: 480155}, {t: 1587691265791, v: 480472}, {t: 1587691280811, v: 480598}, {t: 1587691295791, v: 480771}, {t: 1587691310791, v: 480996}, + {t: 1587691325791, v: 481200}, {t: 1587691340803, v: 481381}, {t: 1587691355791, v: 481584}, {t: 1587691370791, v: 481759}, {t: 1587691385791, v: 482003}, {t: 1587691400803, v: 482189}, {t: 1587691415791, v: 482457}, {t: 1587691430791, v: 482623}, + {t: 1587691445791, v: 482768}, {t: 1587691460804, v: 483036}, {t: 1587691475791, v: 483322}, {t: 1587691490791, v: 483566}, {t: 1587691505791, v: 483709}, {t: 1587691520807, v: 483838}, {t: 1587691535791, v: 484091}, {t: 1587691550791, v: 484236}, + {t: 1587691565791, v: 484454}, {t: 1587691580816, v: 484710}, {t: 1587691595791, v: 484978}, {t: 1587691610791, v: 485271}, {t: 1587691625791, v: 485476}, {t: 1587691640792, v: 485640}, {t: 1587691655791, v: 485921}, {t: 1587691670791, v: 486201}, + {t: 1587691685791, v: 486555}, {t: 1587691700791, v: 486691}, {t: 1587691715791, v: 486831}, {t: 1587691730791, v: 487033}, {t: 1587691745791, v: 487268}, {t: 1587691760803, v: 487370}, {t: 1587691775791, v: 487571}, {t: 1587691790791, v: 487787}, + {t: 1587691805791, v: 488036}, {t: 1587691820791, v: 488241}, {t: 1587691835791, v: 488411}, {t: 1587691850791, v: 488625}, {t: 1587691865791, v: 488868}, {t: 1587691880791, v: 489005}, {t: 1587691895791, v: 489237}, {t: 1587691910791, v: 489545}, + {t: 1587691925791, v: 489750}, {t: 1587691940791, v: 489899}, {t: 1587691955791, v: 490048}, {t: 1587691970791, v: 490364}, {t: 1587691985791, v: 490485}, {t: 1587692000791, v: 490722}, {t: 1587692015791, v: 490866}, {t: 1587692030791, v: 491025}, + {t: 1587692045791, v: 491286}, {t: 1587692060816, v: 491543}, {t: 1587692075791, v: 491787}, {t: 1587692090791, v: 492065}, {t: 1587692105791, v: 492223}, {t: 1587692120816, v: 492501}, {t: 1587692135791, v: 492767}, {t: 1587692150791, v: 492955}, + {t: 1587692165791, v: 493194}, {t: 1587692180792, v: 493402}, {t: 1587692195791, v: 493647}, {t: 1587692210791, v: 493897}, {t: 1587692225791, v: 494117}, {t: 1587692240805, v: 494356}, {t: 1587692255791, v: 494620}, {t: 1587692270791, v: 494762}, + {t: 1587692285791, v: 495001}, {t: 1587692300805, v: 495222}, {t: 1587692315791, v: 495393}, {t: 1587692330791, v: 495662}, {t: 1587692345791, v: 495875}, {t: 1587692360801, v: 496082}, {t: 1587692375791, v: 496196}, {t: 1587692390791, v: 496245}, + {t: 1587692405791, v: 496295}, {t: 1587692420791, v: 496365}, {t: 1587692435791, v: 496401}, {t: 1587692450791, v: 496452}, {t: 1587692465791, v: 496491}, {t: 1587692480791, v: 496544}, {t: 1587692495791, v: math.Float64frombits(value.StaleNaN)}, {t: 1587692555791, v: 75}, + {t: 1587692570791, v: 308}, {t: 1587692585791, v: 508}, {t: 1587692600791, v: 701}, {t: 1587692615791, v: 985}, {t: 1587692630791, v: 1153}, {t: 1587692645791, v: 1365}, {t: 1587692660791, v: 1612}, {t: 1587692675803, v: 1922}, {t: 1587692690791, v: 2103}, + {t: 1587692705791, v: 2261}, {t: 1587692720791, v: 2469}, {t: 1587692735805, v: 2625}, {t: 1587692750791, v: 2801}, {t: 1587692765791, v: 2955}, {t: 1587692780791, v: 3187}, {t: 1587692795806, v: 3428}, {t: 1587692810791, v: 3657}, {t: 1587692825791, v: 3810}, + {t: 1587692840791, v: 3968}, {t: 1587692855791, v: 4195}, {t: 1587692870791, v: 4414}, {t: 1587692885791, v: 4646}, {t: 1587692900791, v: 4689}, {t: 1587692915791, v: 4847}, {t: 1587692930791, v: 5105}, {t: 1587692945791, v: 5309}, {t: 1587692960791, v: 5521}, + {t: 1587692975791, v: 5695}, {t: 1587692990810, v: 6010}, {t: 1587693005791, v: 6210}, {t: 1587693020791, v: 6394}, {t: 1587693035791, v: 6597}, {t: 1587693050791, v: 6872}, {t: 1587693065791, v: 7098}, {t: 1587693080791, v: 7329}, {t: 1587693095791, v: 7470}, + {t: 1587693110791, v: 7634}, {t: 1587693125821, v: 7830}, {t: 1587693140791, v: 8034}, {t: 1587693155791, v: 8209}, {t: 1587693170791, v: 8499}, {t: 1587693185791, v: 8688}, {t: 1587693200791, v: 8893}, {t: 1587693215791, v: 9052}, {t: 1587693230791, v: 9379}, + {t: 1587693245791, v: 9544}, {t: 1587693260791, v: 9763}, {t: 1587693275791, v: 9974}, {t: 1587693290791, v: 10242}, {t: 1587693305791, v: 10464}, {t: 1587693320803, v: 10716}, {t: 1587693335791, v: 10975}, {t: 1587693350791, v: 11232}, {t: 1587693365791, v: 11459}, + {t: 1587693380791, v: 11778}, {t: 1587693395804, v: 12007}, {t: 1587693410791, v: 12206}, {t: 1587693425791, v: 12450}, {t: 1587693440791, v: 12693}, {t: 1587693455791, v: 12908}, {t: 1587693470791, v: 13158}, {t: 1587693485791, v: 13427}, {t: 1587693500791, v: 13603}, + {t: 1587693515791, v: 13927}, {t: 1587693530816, v: 14122}, {t: 1587693545791, v: 14327}, {t: 1587693560791, v: 14579}, {t: 1587693575791, v: 14759}, {t: 1587693590791, v: 14956}, + } + + expectedRealSeriesWithStaleMarkerDeduplicated = []sample{ + {t: 1587691205791, v: 479625}, {t: 1587691220805, v: 479866}, {t: 1587691235791, v: 480008}, {t: 1587691250791, v: 480155}, {t: 1587691265791, v: 480472}, {t: 1587691280811, v: 480598}, {t: 1587691295791, v: 480771}, {t: 1587691310791, v: 480996}, + {t: 1587691325791, v: 481200}, {t: 1587691340803, v: 481381}, {t: 1587691355791, v: 481584}, {t: 1587691370791, v: 481759}, {t: 1587691385791, v: 482003}, {t: 1587691400803, v: 482189}, {t: 1587691415791, v: 482457}, {t: 1587691430791, v: 482623}, + {t: 1587691445791, v: 482768}, {t: 1587691460804, v: 483036}, {t: 1587691475791, v: 483322}, {t: 1587691490791, v: 483566}, {t: 1587691505791, v: 483709}, {t: 1587691520807, v: 483838}, {t: 1587691535791, v: 484091}, {t: 1587691550791, v: 484236}, + {t: 1587691565791, v: 484454}, {t: 1587691580816, v: 484710}, {t: 1587691595791, v: 484978}, {t: 1587691610791, v: 485271}, {t: 1587691625791, v: 485476}, {t: 1587691640792, v: 485640}, {t: 1587691655791, v: 485921}, {t: 1587691670791, v: 486201}, + {t: 1587691685791, v: 486555}, {t: 1587691700791, v: 486691}, {t: 1587691715791, v: 486831}, {t: 1587691730791, v: 487033}, {t: 1587691745791, v: 487268}, {t: 1587691760803, v: 487370}, {t: 1587691775791, v: 487571}, {t: 1587691790791, v: 487787}, + {t: 1587691805791, v: 488036}, {t: 1587691820791, v: 488241}, {t: 1587691835791, v: 488411}, {t: 1587691850791, v: 488625}, {t: 1587691865791, v: 488868}, {t: 1587691880791, v: 489005}, {t: 1587691895791, v: 489237}, {t: 1587691910791, v: 489545}, + {t: 1587691925791, v: 489750}, {t: 1587691940791, v: 489899}, {t: 1587691955791, v: 490048}, {t: 1587691970791, v: 490364}, {t: 1587691985791, v: 490485}, {t: 1587692000791, v: 490722}, {t: 1587692015791, v: 490866}, {t: 1587692030791, v: 491025}, + {t: 1587692045791, v: 491286}, {t: 1587692060816, v: 491543}, {t: 1587692075791, v: 491787}, {t: 1587692090791, v: 492065}, {t: 1587692105791, v: 492223}, {t: 1587692120816, v: 492501}, {t: 1587692135791, v: 492767}, {t: 1587692150791, v: 492955}, + {t: 1587692165791, v: 493194}, {t: 1587692180792, v: 493402}, {t: 1587692195791, v: 493647}, {t: 1587692210791, v: 493897}, {t: 1587692225791, v: 494117}, {t: 1587692240805, v: 494356}, {t: 1587692255791, v: 494620}, {t: 1587692270791, v: 494762}, + {t: 1587692285791, v: 495001}, {t: 1587692300805, v: 495222}, {t: 1587692315791, v: 495393}, {t: 1587692330791, v: 495662}, {t: 1587692345791, v: 495875}, {t: 1587692360801, v: 496082}, {t: 1587692375791, v: 496196}, {t: 1587692390791, v: 496245}, + {t: 1587692405791, v: 496295}, {t: 1587692420791, v: 496365}, {t: 1587692435791, v: 496401}, {t: 1587692450791, v: 496452}, {t: 1587692465791, v: 496491}, {t: 1587692480791, v: 496544}, {t: 1587692495791, v: math.Float64frombits(value.StaleNaN)}, {t: 1587692555791, v: 75}, + {t: 1587692570791, v: 308}, {t: 1587692585791, v: 508}, {t: 1587692600791, v: 701}, {t: 1587692615791, v: 985}, {t: 1587692630791, v: 1153}, {t: 1587692645791, v: 1365}, {t: 1587692660791, v: 1612}, {t: 1587692675803, v: 1922}, {t: 1587692690791, v: 2103}, + {t: 1587692705791, v: 2261}, {t: 1587692720791, v: 2469}, {t: 1587692735805, v: 2625}, {t: 1587692750791, v: 2801}, {t: 1587692765791, v: 2955}, {t: 1587692780791, v: 3187}, {t: 1587692795806, v: 3428}, {t: 1587692810791, v: 3657}, {t: 1587692825791, v: 3810}, + {t: 1587692840791, v: 3968}, {t: 1587692855791, v: 4195}, {t: 1587692870791, v: 4414}, {t: 1587692885791, v: 4646}, {t: 1587692900791, v: 4689}, {t: 1587692915791, v: 4847}, {t: 1587692930791, v: 5105}, {t: 1587692945791, v: 5309}, {t: 1587692960791, v: 5521}, + {t: 1587692975791, v: 5695}, {t: 1587692990810, v: 6010}, {t: 1587693005791, v: 6210}, {t: 1587693020791, v: 6394}, {t: 1587693035791, v: 6597}, {t: 1587693050791, v: 6872}, {t: 1587693065791, v: 7098}, {t: 1587693080791, v: 7329}, {t: 1587693095791, v: 7470}, + {t: 1587693110791, v: 7634}, {t: 1587693125821, v: 7830}, {t: 1587693140791, v: 8034}, {t: 1587693155791, v: 8209}, {t: 1587693170791, v: 8499}, {t: 1587693185791, v: 8688}, {t: 1587693200791, v: 8893}, {t: 1587693215791, v: 9052}, {t: 1587693230791, v: 9379}, + {t: 1587693245791, v: 9544}, {t: 1587693260791, v: 9763}, {t: 1587693275791, v: 9974}, {t: 1587693290791, v: 10242}, {t: 1587693305791, v: 10464}, {t: 1587693320803, v: 10716}, {t: 1587693335791, v: 10975}, {t: 1587693350791, v: 11232}, {t: 1587693365791, v: 11459}, + {t: 1587693380791, v: 11778}, {t: 1587693395804, v: 12007}, {t: 1587693410791, v: 12206}, {t: 1587693425791, v: 12450}, {t: 1587693440791, v: 12693}, {t: 1587693455791, v: 12908}, {t: 1587693470791, v: 13158}, {t: 1587693485791, v: 13427}, {t: 1587693500791, v: 13603}, + {t: 1587693515791, v: 13927}, {t: 1587693530816, v: 14122}, {t: 1587693545791, v: 14327}, {t: 1587693560791, v: 14579}, {t: 1587693575791, v: 14759}, {t: 1587693590791, v: 14956}, + } +) + func TestQuerier_Select(t *testing.T) { + const hackyStaleMarker = float64(-99999999) + logger := log.NewLogfmtLogger(os.Stderr) + type series struct { lset labels.Labels samples []sample } + for _, tcase := range []struct { name string storeAPI storepb.StoreServer - mint, maxt int64 - matchers []*labels.Matcher - replicaLabels []string - dedup bool - hints *storage.SelectParams - + // Select test. + mint, maxt int64 + matchers []*labels.Matcher + replicaLabels []string + dedup bool + hints *storage.SelectParams expected []series expectedWarning string }{ @@ -213,12 +333,33 @@ func TestQuerier_Select(t *testing.T) { expectedWarning: "partial error", }, { - // Regression test against https://github.com/thanos-io/thanos/issues/2401: Overlapping chunks. - // Thanks to @Superq and GitLab for real data reproducing this! - // TODO(bwplotka): This still does not reproduce bug, waiting for more data (: - name: "overlapping chunks", + name: "select overlapping data with partial error + dedup", + storeAPI: &storeServer{ + resps: []*storepb.SeriesResponse{ + storeSeriesResponse(t, labels.FromStrings("a", "a"), []sample{{0, 0}, {2, 1}, {3, 2}}), + storepb.NewWarnSeriesResponse(errors.New("partial error")), + storeSeriesResponse(t, labels.FromStrings("a", "a"), []sample{{5, 5}, {6, 6}, {7, 7}}), + storeSeriesResponse(t, labels.FromStrings("a", "a"), []sample{{5, 5}, {6, 66}}), // Overlap samples for some reason. + storeSeriesResponse(t, labels.FromStrings("a", "b"), []sample{{2, 2}, {3, 3}, {4, 4}}, []sample{{1, 1}, {2, 2}, {3, 3}}), + storeSeriesResponse(t, labels.FromStrings("a", "c"), []sample{{100, 1}, {300, 3}, {400, 4}}), + }, + }, + replicaLabels: []string{"a"}, + dedup: true, + mint: 1, maxt: 300, + expected: []series{ + { + lset: labels.Labels{}, + // TODO(bwplotka): This seems wrong. + samples: []sample{{1, 1}, {2, 2}, {3, 3}, {4, 4}}, + }, + }, + expectedWarning: "partial error", + }, + { + name: "realistic data", storeAPI: func() storepb.StoreServer { - s, err := store.NewLocalStoreFromJSONMmappableFile(log.NewLogfmtLogger(os.Stderr), component.Debug, nil, "./testdata/issue2401-seriesresponses.json", store.ScanGRPCCurlProtoStreamMessages) + s, err := store.NewLocalStoreFromJSONMmappableFile(logger, component.Debug, nil, "./testdata/issue2401-seriesresponses.json", store.ScanGRPCCurlProtoStreamMessages) testutil.Ok(t, err) return s }(), @@ -249,27 +390,7 @@ func TestQuerier_Select(t *testing.T) { "tier", "sv", "type", "web", ), - samples: []sample{ - {t: 1582329611249, v: 3.395395e+06}, {t: 1582329626245, v: 3.39592e+06}, {t: 1582329641245, v: 3.396402e+06}, {t: 1582329656245, v: 3.396882e+06}, {t: 1582329671245, v: 3.397394e+06}, {t: 1582329686245, v: 3.397804e+06}, - {t: 1582329701245, v: 3.398296e+06}, {t: 1582329716245, v: 3.398878e+06}, {t: 1582329731245, v: 3.399373e+06}, {t: 1582329746245, v: 3.39992e+06}, {t: 1582329761245, v: 3.400436e+06}, {t: 1582329776245, v: 3.400974e+06}, - {t: 1582329791245, v: 3.401408e+06}, {t: 1582329806245, v: 3.401901e+06}, {t: 1582329821245, v: 3.402411e+06}, {t: 1582329836245, v: 3.402944e+06}, {t: 1582329851245, v: 3.403404e+06}, {t: 1582329866245, v: 3.403735e+06}, - {t: 1582329881245, v: 3.404301e+06}, {t: 1582329896245, v: 3.404818e+06}, {t: 1582329911245, v: 3.405327e+06}, {t: 1582329926245, v: 3.405843e+06}, {t: 1582329941245, v: 3.406297e+06}, {t: 1582329956245, v: 3.406737e+06}, - {t: 1582329971245, v: 3.407212e+06}, {t: 1582329986245, v: 3.407647e+06}, {t: 1582330001245, v: 3.407929e+06}, {t: 1582330016245, v: 3.408614e+06}, {t: 1582330031245, v: 3.409217e+06}, {t: 1582330046245, v: 3.409658e+06}, - {t: 1582330061245, v: 3.410276e+06}, {t: 1582330076245, v: 3.410795e+06}, {t: 1582330091245, v: 3.411509e+06}, {t: 1582330106245, v: 3.411974e+06}, {t: 1582330121245, v: 3.412368e+06}, {t: 1582330136245, v: 3.412863e+06}, - {t: 1582330151245, v: 3.413412e+06}, {t: 1582330166245, v: 3.413783e+06}, {t: 1582330181245, v: 3.41424e+06}, {t: 1582330196245, v: 3.414634e+06}, {t: 1582330211245, v: 3.415246e+06}, {t: 1582330226245, v: 3.415615e+06}, - {t: 1582330241245, v: 3.416287e+06}, {t: 1582330256245, v: 3.416753e+06}, {t: 1582330271245, v: 3.417229e+06}, {t: 1582330286245, v: 3.417748e+06}, {t: 1582330301245, v: 3.418185e+06}, {t: 1582330316245, v: 3.418707e+06}, - {t: 1582330331245, v: 3.419142e+06}, {t: 1582330346245, v: 3.419595e+06}, {t: 1582330361245, v: 3.420042e+06}, {t: 1582330376245, v: 3.420544e+06}, {t: 1582330391245, v: 3.420926e+06}, {t: 1582330406245, v: 3.421447e+06}, - {t: 1582330421245, v: 3.421899e+06}, {t: 1582330436245, v: 3.422238e+06}, {t: 1582330451245, v: 3.422619e+06}, {t: 1582330466245, v: 3.422958e+06}, {t: 1582330481251, v: 3.423455e+06}, {t: 1582330496245, v: 3.423991e+06}, - {t: 1582330511245, v: 3.424478e+06}, {t: 1582330526245, v: 3.425018e+06}, {t: 1582330541245, v: 3.425455e+06}, {t: 1582330556245, v: 3.42597e+06}, {t: 1582330571245, v: 3.426343e+06}, {t: 1582330586245, v: 3.426881e+06}, - {t: 1582330601245, v: 3.427435e+06}, {t: 1582330616245, v: 3.42803e+06}, {t: 1582330631261, v: 3.42861e+06}, {t: 1582330646245, v: 3.429152e+06}, {t: 1582330661245, v: 3.429711e+06}, {t: 1582330676245, v: 3.430061e+06}, - {t: 1582330691245, v: 3.430594e+06}, {t: 1582330706245, v: 3.431031e+06}, {t: 1582330721245, v: 3.431528e+06}, {t: 1582330736245, v: 3.432125e+06}, {t: 1582330751245, v: 3.432611e+06}, {t: 1582330766245, v: 3.433042e+06}, - {t: 1582330781245, v: 3.433544e+06}, {t: 1582330796245, v: 3.434109e+06}, {t: 1582330811245, v: 3.434386e+06}, {t: 1582330826245, v: 3.434891e+06}, {t: 1582330841245, v: 3.43544e+06}, {t: 1582330856245, v: 3.435977e+06}, - {t: 1582330871245, v: 3.436528e+06}, {t: 1582330886245, v: 3.437145e+06}, {t: 1582330901245, v: 3.437773e+06}, {t: 1582330916245, v: 3.438278e+06}, {t: 1582330931245, v: 3.438664e+06}, {t: 1582330946245, v: 3.439135e+06}, - {t: 1582330961245, v: 3.439635e+06}, {t: 1582330976245, v: 3.440136e+06}, {t: 1582330991245, v: 3.440572e+06}, {t: 1582331006245, v: 3.441014e+06}, {t: 1582331021256, v: 3.441373e+06}, {t: 1582331036245, v: 3.441745e+06}, - {t: 1582331051245, v: 3.442336e+06}, {t: 1582331066245, v: 3.44291e+06}, {t: 1582331081245, v: 3.443478e+06}, {t: 1582331096245, v: 3.443938e+06}, {t: 1582331111245, v: 3.444436e+06}, {t: 1582331126245, v: 3.445027e+06}, - {t: 1582331141245, v: 3.445382e+06}, {t: 1582331156245, v: 3.445853e+06}, {t: 1582331171262, v: 3.446356e+06}, {t: 1582331186245, v: 3.446899e+06}, {t: 1582331201245, v: 3.44743e+06}, {t: 1582331216245, v: 3.448002e+06}, - {t: 1582331231245, v: 3.448451e+06}, {t: 1582331246245, v: 3.448974e+06}, {t: 1582331261259, v: 3.449337e+06}, {t: 1582331276245, v: 3.449841e+06}, {t: 1582331291245, v: 3.450478e+06}, {t: 1582331306245, v: 3.450903e+06}, - {t: 1582331321245, v: 3.451404e+06}, {t: 1582331336245, v: 3.451778e+06}, {t: 1582331351245, v: 3.452278e+06}, {t: 1582331366245, v: 3.452711e+06}, {t: 1582331381245, v: 3.453142e+06}}, + samples: expectedRealSeriesReplica0, }, { lset: labels.FromStrings( @@ -290,28 +411,107 @@ func TestQuerier_Select(t *testing.T) { "tier", "sv", "type", "web", ), - samples: []sample{ - {t: 1582329618053, v: 3.395638e+06}, {t: 1582329633053, v: 3.396163e+06}, {t: 1582329648053, v: 3.396612e+06}, {t: 1582329663053, v: 3.397174e+06}, {t: 1582329678053, v: 3.397564e+06}, {t: 1582329693053, v: 3.398039e+06}, - {t: 1582329708053, v: 3.39847e+06}, {t: 1582329723053, v: 3.399088e+06}, {t: 1582329738053, v: 3.399668e+06}, {t: 1582329753053, v: 3.400198e+06}, {t: 1582329768053, v: 3.400705e+06}, {t: 1582329783053, v: 3.401209e+06}, - {t: 1582329798053, v: 3.401606e+06}, {t: 1582329813053, v: 3.402171e+06}, {t: 1582329828054, v: 3.402601e+06}, {t: 1582329843053, v: 3.403164e+06}, {t: 1582329858053, v: 3.40357e+06}, {t: 1582329873053, v: 3.404005e+06}, - {t: 1582329888053, v: 3.404553e+06}, {t: 1582329903053, v: 3.405076e+06}, {t: 1582329918053, v: 3.40554e+06}, {t: 1582329933053, v: 3.406012e+06}, {t: 1582329948053, v: 3.406466e+06}, {t: 1582329963053, v: 3.407003e+06}, - {t: 1582329978053, v: 3.407378e+06}, {t: 1582329993053, v: 3.407728e+06}, {t: 1582330008053, v: 3.40835e+06}, {t: 1582330023053, v: 3.408935e+06}, {t: 1582330038053, v: 3.409436e+06}, {t: 1582330053053, v: 3.409904e+06}, - {t: 1582330068053, v: 3.410449e+06}, {t: 1582330083053, v: 3.411078e+06}, {t: 1582330098053, v: 3.411737e+06}, {t: 1582330113053, v: 3.41214e+06}, {t: 1582330128053, v: 3.412598e+06}, {t: 1582330143053, v: 3.413069e+06}, - {t: 1582330158053, v: 3.4136e+06}, {t: 1582330173053, v: 3.413985e+06}, {t: 1582330188053, v: 3.414443e+06}, {t: 1582330203053, v: 3.414959e+06}, {t: 1582330218053, v: 3.415411e+06}, {t: 1582330233053, v: 3.415962e+06}, - {t: 1582330248064, v: 3.416517e+06}, {t: 1582330263053, v: 3.416999e+06}, {t: 1582330278053, v: 3.417466e+06}, {t: 1582330293053, v: 3.418001e+06}, {t: 1582330308053, v: 3.418434e+06}, {t: 1582330323053, v: 3.41892e+06}, - {t: 1582330338053, v: 3.419327e+06}, {t: 1582330353053, v: 3.419814e+06}, {t: 1582330368053, v: 3.420281e+06}, {t: 1582330383053, v: 3.420671e+06}, {t: 1582330398053, v: 3.421132e+06}, {t: 1582330413058, v: 3.421673e+06}, - {t: 1582330428053, v: 3.42204e+06}, {t: 1582330443053, v: 3.422405e+06}, {t: 1582330458053, v: 3.422753e+06}, {t: 1582330473053, v: 3.423144e+06}, {t: 1582330488067, v: 3.423663e+06}, {t: 1582330503053, v: 3.424217e+06}, - {t: 1582330518053, v: 3.424743e+06}, {t: 1582330533053, v: 3.425246e+06}, {t: 1582330548053, v: 3.425641e+06}, {t: 1582330563054, v: 3.426142e+06}, {t: 1582330578053, v: 3.426571e+06}, {t: 1582330593053, v: 3.427067e+06}, - {t: 1582330608053, v: 3.427776e+06}, {t: 1582330623053, v: 3.428342e+06}, {t: 1582330638053, v: 3.428903e+06}, {t: 1582330653077, v: 3.429352e+06}, {t: 1582330668053, v: 3.429892e+06}, {t: 1582330683053, v: 3.430238e+06}, - {t: 1582330698053, v: 3.430812e+06}, {t: 1582330713053, v: 3.431257e+06}, {t: 1582330728056, v: 3.431899e+06}, {t: 1582330743053, v: 3.432422e+06}, {t: 1582330758053, v: 3.432863e+06}, {t: 1582330773053, v: 3.433215e+06}, - {t: 1582330788053, v: 3.433822e+06}, {t: 1582330803053, v: 3.434217e+06}, {t: 1582330818053, v: 3.434559e+06}, {t: 1582330833053, v: 3.43516e+06}, {t: 1582330848053, v: 3.435718e+06}, {t: 1582330863053, v: 3.436289e+06}, - {t: 1582330878053, v: 3.436776e+06}, {t: 1582330893053, v: 3.437315e+06}, {t: 1582330908053, v: 3.437971e+06}, {t: 1582330923053, v: 3.438459e+06}, {t: 1582330938053, v: 3.438912e+06}, {t: 1582330953053, v: 3.439307e+06}, - {t: 1582330968053, v: 3.439861e+06}, {t: 1582330983053, v: 3.440302e+06}, {t: 1582330998053, v: 3.440744e+06}, {t: 1582331013053, v: 3.441185e+06}, {t: 1582331028053, v: 3.441574e+06}, {t: 1582331043053, v: 3.442031e+06}, - {t: 1582331058053, v: 3.442613e+06}, {t: 1582331073053, v: 3.443145e+06}, {t: 1582331088053, v: 3.443649e+06}, {t: 1582331103053, v: 3.444181e+06}, {t: 1582331118053, v: 3.444643e+06}, {t: 1582331133053, v: 3.445203e+06}, - {t: 1582331148053, v: 3.445629e+06}, {t: 1582331163053, v: 3.446097e+06}, {t: 1582331178053, v: 3.446645e+06}, {t: 1582331193053, v: 3.447223e+06}, {t: 1582331208054, v: 3.447765e+06}, {t: 1582331223053, v: 3.448241e+06}, - {t: 1582331238053, v: 3.448659e+06}, {t: 1582331253053, v: 3.449075e+06}, {t: 1582331268053, v: 3.449566e+06}, {t: 1582331283053, v: 3.450086e+06}, {t: 1582331298053, v: 3.450673e+06}, {t: 1582331313053, v: 3.451119e+06}, - {t: 1582331328053, v: 3.451611e+06}, {t: 1582331343053, v: 3.452079e+06}, {t: 1582331358053, v: 3.452434e+06}, {t: 1582331373053, v: 3.452943e+06}, - }, + samples: expectedRealSeriesReplica1, + }, + }, + }, + { + name: "realistic data with stale marker + dedup", + storeAPI: func() storepb.StoreServer { + s, err := store.NewLocalStoreFromJSONMmappableFile(logger, component.Debug, nil, "./testdata/issue2401-seriesresponses2.json", store.ScanGRPCCurlProtoStreamMessages) + testutil.Ok(t, err) + return s + }(), + mint: 1587691200000, // 04/24/2020 01:20:00 GMT + maxt: 1587693600000, // 04/24/2020 02:00:00 GMT + matchers: []*labels.Matcher{{ + Value: "gitlab_transaction_cache_read_hit_count_total", + Name: "__name__", + Type: labels.MatchEqual, + }}, + expected: []series{ + { + lset: labels.FromStrings( + "__name__", "gitlab_transaction_cache_read_hit_count_total", + "action", "widget.json", + "controller", "Projects::MergeRequests::ContentController", + "env", "gprd", + "environment", "gprd", + "fqdn", "web-08-sv-gprd.c.gitlab-production.internal", + "instance", "web-08-sv-gprd.c.gitlab-production.internal:8083", + "job", "gitlab-rails", + "monitor", "app", + "provider", "gcp", + "region", "us-east", + "replica", "01", + "shard", "default", + "stage", "main", + "tier", "sv", + "type", "web", + ), + samples: expectedRealSeriesWithStaleMarkerReplica0, + }, + { + lset: labels.FromStrings( + "__name__", "gitlab_transaction_cache_read_hit_count_total", + "action", "widget.json", + "controller", "Projects::MergeRequests::ContentController", + "env", "gprd", + "environment", "gprd", + "fqdn", "web-08-sv-gprd.c.gitlab-production.internal", + "instance", "web-08-sv-gprd.c.gitlab-production.internal:8083", + "job", "gitlab-rails", + "monitor", "app", + "provider", "gcp", + "region", "us-east", + "replica", "02", + "shard", "default", + "stage", "main", + "tier", "sv", + "type", "web", + ), + samples: expectedRealSeriesWithStaleMarkerReplica1, + }, + }, + }, + { + // Regression test against https://github.com/thanos-io/thanos/issues/2401. + // Thanks to @Superq and GitLab for real data reproducing this! + name: "realistic data with stale marker + dedup", + storeAPI: func() storepb.StoreServer { + s, err := store.NewLocalStoreFromJSONMmappableFile(logger, component.Debug, nil, "./testdata/issue2401-seriesresponses2.json", store.ScanGRPCCurlProtoStreamMessages) + testutil.Ok(t, err) + return s + }(), + replicaLabels: []string{"replica"}, + dedup: true, + mint: 1587691200000, // 04/24/2020 01:20:00 GMT + maxt: 1587693600000, // 04/24/2020 02:00:00 GMT + matchers: []*labels.Matcher{{ + Value: "gitlab_transaction_cache_read_hit_count_total", + Name: "__name__", + Type: labels.MatchEqual, + }}, + expected: []series{ + { + lset: labels.FromStrings( + "__name__", "gitlab_transaction_cache_read_hit_count_total", + "action", "widget.json", + "controller", "Projects::MergeRequests::ContentController", + "env", "gprd", + "environment", "gprd", + "fqdn", "web-08-sv-gprd.c.gitlab-production.internal", + "instance", "web-08-sv-gprd.c.gitlab-production.internal:8083", + "job", "gitlab-rails", + "monitor", "app", + "provider", "gcp", + "region", "us-east", + "shard", "default", + "stage", "main", + "tier", "sv", + "type", "web", + ), + samples: expectedRealSeriesWithStaleMarkerDeduplicated, }, }, }, @@ -335,6 +535,12 @@ func TestQuerier_Select(t *testing.T) { testutil.Equals(t, tcase.expected[i].lset, res.At().Labels()) samples := expandSeries(t, res.At().Iterator()) + for si := range samples { + if value.IsStaleNaN(samples[si].v) { + // We have to hack a bit otherwise not work, because Nan != Nan. + samples[si].v = hackyStaleMarker + } + } testutil.Equals(t, tcase.expected[i].samples, samples, "samples for series %v does not match", i) i++ } @@ -344,6 +550,126 @@ func TestQuerier_Select(t *testing.T) { } } +type mockedOneSeriesSeriesSet struct { + done bool + samples []sample +} + +func (s *mockedOneSeriesSeriesSet) Next() bool { + if s.done { + s.done = true + return false + } + return true +} + +func (s *mockedOneSeriesSeriesSet) At() storage.Series { return &mockedSeries{samples: s.samples} } +func (s *mockedOneSeriesSeriesSet) Err() error { return nil } + +type mockedSeries struct { + samples []sample +} + +func (s *mockedSeries) Labels() labels.Labels { return labels.FromStrings("a", "b") } +func (s *mockedSeries) Iterator() storage.SeriesIterator { + return &mockedSeriesIterator{cur: -1, samples: s.samples} +} + +type mockedSeriesIterator struct { + cur int + samples []sample +} + +func (s *mockedSeriesIterator) Seek(t int64) bool { + s.cur = sort.Search(len(s.samples), func(n int) bool { + return s.samples[n].t >= t + }) + return s.cur < len(s.samples) +} + +func (s *mockedSeriesIterator) At() (t int64, v float64) { + sample := s.samples[s.cur] + return sample.t, sample.v +} + +func (s *mockedSeriesIterator) Next() bool { + s.cur++ + return s.cur < len(s.samples) +} + +func (s *mockedSeriesIterator) Err() error { return nil } + +type mockedQueryable struct { + samples []sample +} + +func (q *mockedQueryable) Querier(context.Context, int64, int64) (storage.Querier, error) { + return q, nil +} + +func (q *mockedQueryable) Select(*storage.SelectParams, ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { + return &mockedOneSeriesSeriesSet{samples: q.samples}, nil, nil +} + +func (q mockedQueryable) LabelValues(string) ([]string, storage.Warnings, error) { + panic("implement me") +} +func (q mockedQueryable) LabelNames() ([]string, storage.Warnings, error) { panic("implement me") } +func (q mockedQueryable) Close() error { return nil } + +func TestQuerierUnderstoodByPromQL_Rate(t *testing.T) { + logger := log.NewLogfmtLogger(os.Stderr) + + for _, tcase := range []struct { + name string + input []sample + mint, maxt time.Time + expectedRate5m []sample + expectedRate30m []sample + }{ + { + name: "realistic data", + mint: timestamp.Time(1582329611249), + maxt: timestamp.Time(1582331381245), + input: expectedRealSeriesReplica0, + }, + //{ + // name: "realistic data with stale marker", + // mint: timestamp.Time(1587691200000), // 04/24/2020 01:20:00 GMT + // maxt: timestamp.Time(1587693600000), // 04/24/2020 02:00:00 GMT + // input: expectedRealSeriesWithStaleMarkerReplica0, + //}, + //{ + // // Regression test against https://github.com/thanos-io/thanos/issues/2401. + // name: "deduplicated realistic data with stale marker", + // mint: timestamp.Time(1587691200000), // 04/24/2020 01:20:00 GMT + // maxt: timestamp.Time(1587693600000), // 04/24/2020 02:00:00 GMT + // input: expectedRealSeriesWithStaleMarkerDeduplicated, + //}, + } { + t.Run(tcase.name, func(t *testing.T) { + e := promql.NewEngine(promql.EngineOpts{Logger: logger, MaxConcurrent: 1, Timeout: 5 * time.Second, MaxSamples: math.MinInt64}) + t.Run("Rate=5mStep=1800s", func(t *testing.T) { + q, err := e.NewRangeQuery(&mockedQueryable{samples: tcase.input}, `rate({a="b"}[5m])`, tcase.mint, tcase.maxt, 1800*time.Second) + testutil.Ok(t, err) + defer q.Close() + + r := q.Exec(context.Background()) + testutil.Ok(t, r.Err) + fmt.Println(r.String()) + }) + t.Run("Rate=30mStep=1800s", func(t *testing.T) { + q, err := e.NewRangeQuery(&mockedQueryable{samples: tcase.input}, `rate({a="b"}[30m])`, tcase.mint, tcase.maxt, 1800*time.Second) + testutil.Ok(t, err) + defer q.Close() + + r := q.Exec(context.Background()) + testutil.Ok(t, r.Err) + fmt.Println(r.String()) + }) + }) + } +} func TestSortReplicaLabel(t *testing.T) { defer leaktest.CheckTimeout(t, 10*time.Second)() diff --git a/pkg/query/testdata/issue2401-seriesresponses2.json b/pkg/query/testdata/issue2401-seriesresponses2.json new file mode 100644 index 00000000000..82faab987a5 --- /dev/null +++ b/pkg/query/testdata/issue2401-seriesresponses2.json @@ -0,0 +1,480 @@ +{ + "series": { + "labels": [ + { + "name": "__name__", + "value": "gitlab_transaction_cache_read_hit_count_total" + }, + { + "name": "action", + "value": "widget.json" + }, + { + "name": "controller", + "value": "Projects::MergeRequests::ContentController" + }, + { + "name": "env", + "value": "gprd" + }, + { + "name": "environment", + "value": "gprd" + }, + { + "name": "fqdn", + "value": "web-08-sv-gprd.c.gitlab-production.internal" + }, + { + "name": "instance", + "value": "web-08-sv-gprd.c.gitlab-production.internal:8083" + }, + { + "name": "job", + "value": "gitlab-rails" + }, + { + "name": "monitor", + "value": "app" + }, + { + "name": "provider", + "value": "gcp" + }, + { + "name": "region", + "value": "us-east" + }, + { + "name": "replica", + "value": "01" + }, + { + "name": "shard", + "value": "default" + }, + { + "name": "stage", + "value": "main" + }, + { + "name": "tier", + "value": "sv" + }, + { + "name": "type", + "value": "web" + } + ], + "chunks": [ + { + "minTime": "1587679207139", + "maxTime": "1587680992139", + "raw": { + "data": "AHjG+/CQtVxBEeOIAAAAAJh16jw3Qrd0qbmVwqUJ1Jnrch/6qASJAaagF8QOIoBHUBtKDt5ALqgMsQCHIHP0A8CAxNAKwh7eQG8oB8ED96AdlAY4gEVQeboBBUBmSAUhA18gPWQCRI/FUApCAwFA1OgHUQHzoBV0HsSAYtAZkgF3QOPIB0UB66ASdD1YgNyQCWoHF0AgMAHwBeX/FAXTAB8HiiARCAGIDJL/QgEAgBiAO1A44gLOQCaJ/tUApSAyVAJmgbVQGXoBfoAUg8Uv9iARqAFIDN0Aw6Bu5AGygNxQCcIfHIAPgFdv+KA1mAD4BR0DeiA6ZAPMg+9QCDIDJEAmCB3FAKAgMfQCao/sEAmqAwxAJ+gclQDioDK0AvqDypAI8gKXQBKIHWkBUSANZAK+h8ygBCApC/4IA8IAQgFCQOmoBN4AOgLFv+SD20A=" + } + }, + { + "minTime": "1587681007139", + "maxTime": "1587682792139", + "raw": { + "data": "AHjG2MyStVxBE+SMAAAAAJh16FaPACE9V/wcyuHgBCNtNuhQRINqP/FUAUFAO/gBKALHf7kB3uAEoAaKAWSgBkQADA6tv/6AIDAAMA8tAFAQHKkAUxANLQBHUHy9AETQDZUAV9AdWgA+AIbf8UA+uAD4AiiB7egBmQAZAECv+iAYnABkASRAcGQBNEAxtAEzQ/b0ARZANDQG50AQtAMtQBJkAO5A6/QDBkAUZADBQGgUAydAF7QffYAUgDW3+xANigBSAP0gPuIA66AfigCgIHcqAZTACEARe/4IDqkAIQBXUANhALeR/ekAWNAM3QAzEARZAcDQBIkA0lAHCQPH0AfRAPQQBaUBzJACtQBA0A79AEBgBSD7Ff7EAVeAFIBvyALQgO+IAvqAHMgHVoAryB98gDmoB+iAI2gOlwAhAETv+CAYRACEAVFD84QBSoAVgGKX+tAFFgBWA+kgDvIBiCAPageUIAiaAeCgC5IDbSAGAA==" + } + }, + { + "minTime": "1587682807139", + "maxTime": "1587684592139", + "raw": { + "data": "AHjGtaiUtVxBFbqwAAAAAJh16kxVxN+SoPygSCGPoI0juKD+oZGg6afBIECg3cAHQOm/5IZfAB0FBR81B30BbQwdBAG5D+30BiKAKSAKYBxL/igcEv8GATmAUIDNb/IgFCgByDxZAI2gCoQG0IGxEAjaAPZAaAgFzQ+XIBc0Bk6AeZA6SgDDgBCARi/4IDDYAQgFCQfuQAfAOBf8UB7MAHwCgoA4EDqSAQFAeogEER+VoB2EB6KAdJA/CgHkQGIoBQUHkqAQpAZCgElQOfICyEAWKAclD66gEeQBroDfUA3KBzZAJKgDRQFfIPXUAj6AMOAEoDE7/cgbbgBKAUpAYCgELT+WIBA0BqsAHwCSX/FAGrAB8DnCAbWAE4Akr/agO2gBOASdAFdACEHpV/wQCfQAhAfIgHwQPcoBBkAbKAzpAKwh/sQD9IDzYAXgHcv9KB7aAF4AmUAjaA3xAJag7VA=" + } + }, + { + "minTime": "1587684607139", + "maxTime": "1587686392139", + "raw": { + "data": "AHjGkoSWtVxBF2kcAAAAAJh16k0VM5zL7qjwJsSJyhSOBn7bqRmuhGa7EaLqTa3MXrqT9XQjPdh7LqJLkZ6KhukbIhA06wg10Kij/EIE4k/10KqlHQmXcy+/ogCGMmjo3bkZ5egu0KyiyE5EgtkK0jA0MA2Jf/SoAQNAGKoAZNAO6IAThAGHoAQFAB/ID3xACtoA19ADXoB1BAC9IAz1AC1IHwxACGIAwxAC1oB2FABTIAXZAHgoAKBACEoD0JADjoA2xABgIAQJAOuIAcFABcoAuqAGIPx6/0IASyAGIAwFACVQAhABDX/BAOBQAhADvoA2FACMoD7RACJoAzqAFIAce/2IB+OAFIAR9AGOwAhACK3/BA+HwAhADYoAwKAFIASS/2IAM2AFIBytAC/wAlAHe3+5ABlQAlACjoDzeAGIAde/0IA3+AGIAM5AC9IAHJAOIoAXhAHuwAxACM3+hD+CwAxACVg=" + } + }, + { + "minTime": "1587686407139", + "maxTime": "1587688192139", + "raw": { + "data": "AHjG79+XtVxBGQjEAAAAAJh17EdrqRPuZfCUUOAD5lV/xR94AP5F+HdSeS6DZupEa5l/9ROULtNZRYXE36ag8iHroOYjpKBjIL6h9qBBIPUnj6BFIIghgKCDo5cgkKGHoLu4HfkKBLkGxIENQ4ygbNARqDgEBooEWR8lACkCk3+xBi8AKQJUgNOAF4dUv9KBCoAXg8VArqAr0/MoEvQGIgqRDpKArkD7oPsQIcj8ZApMAIQcvf8EDuQAhDgSBV0GOoD7QJW3If4CgGHQG7oBe0D4qAaRAZygDuQCQIPE4ATgFAv9qA4WAE4A60AzkAHQONX/JAJ5AB0BjSAL5D2AgDYQCGIDaYAHgH6v/KByaAB4BG0BksAHwCWX/FAF1AB8H0aAKdAKYgMZQCQIHqoARgDUv96AvOAEYAXUAWMAIR9H3/BAKLACEBsyAIBAKIgcCQDtoDxUAyA=" + } + }, + { + "minTime": "1587688207139", + "maxTime": "1587689992139", + "raw": { + "data": "AHjGzLuZtVxBGp6wAAAAAJh15F/rQlEbpDTcy+Ao4sAHzB7/iiPQAfDjcTfXqBJIfMglyO1oHDACMF5v96H9wAjAp07nQTBDRUFFQJxHFUHeQxxAlW8791QDM6k610E11Jg8AJ8yuZv9qFGAE+pPvdCsNSruRnhKFXRqiDfQtKcIQ76NKQnKCkcTPw3YjldSdvAB3Qrfb/kjfgA6Q5zH9upFHADEzX+h0K3uAGJGHIzxlCPI+xCUoLlPEIfhGFocZwO/Y0CPIDJQasgcqACodAv+yA/IAKgVJB/CBM0eQoFZQbMgRaADYcDv+aAsIANgURBmyBxU+nIDZQKcg4uADoEPv+SHJYAOgNVBeqA5IAMgWS/6I8cgAyB+kGlQADAmN//Q5DAAMC1oMCQIYgJZtQ/4t1JvboTn03XYj1cyeG6FWTAAMXt//SawADyL8K6k9dIt0JwqWAA=" + } + }, + { + "minTime": "1587690007139", + "maxTime": "1587691792139", + "raw": { + "data": "AHjGqZebtVxBHDKkAAAAAJh16k/10Kx1IeiucTfscAIQY+/4IEzACEP0QXNA1UcOQQxDe0EdgAGeR3/9BxYABhtCCAoHwjv6D0wAXDLL/qggMAF8Dv31Aa6BjEB7IMIQKwh35A76DB0BCoFLR5agYNBlKBH0BtodXQOQg9NA3qfGECbID8QZygRRD1KBSUHBIHUR++gd9B9qBDkOcoGZQcmgYlAXTef/I0BAUDPUB0EH7UByEDp0Bs0PEEB8EDwkBd0HZEBB0DR0A/kBB0ffUBCoAPgZrf8UBq4APgFeg7/ACUBv7/cgbJACUBAEAv4AUh+ff7EBTYAUgGIgZTAAUBb7/8g7hAAUBgEAkEDL0+7kA80BkUAgUDBUBZUH4UB80D/kB5UP50BNEDfUBbUH6EBn0AwkDZ0B/YAmgETf2kfYYAmgPIA=" + } + }, + { + "minTime": "1587691807139", + "maxTime": "1587693592139", + "raw": { + "data": "AHXGhvOctVxBHcnIAAAAAJh16k0NNlzL66iPJySI+5Ge3oXtBFI9xCMJ3JB7IXhHIoTCAENx3+Df8HUiNgBDoTj3YjrdR5LmXzFF4TBUSByM/NQl6M6Qpic+QfCFiR4iE/cS/uoJDsP7dCuvQYo2YAUhBv9ifyAFIVUaDF793JygAAAAByv0r/kAAAAAAADqgSQCagAAAAAACACoAqwAAAAAAAgAxAAAAAAAEACwHy0AAAAAAC/9oAIWAAAAAABAAsAOTAAAAAAAQAPIAAAAAACAD4AMWAAAAAABf8UAD3gAAAAAAgA+APnIAAAAAAIABlgAAAAAAgAKcAAAAAACAB/gAAAAAAIABcAAAAAAAgAKqAAAAAAEAFQAfQAAAAAAC/7IACBAAAAAABABUADOAAAAAAAQAC6AAAAAABAAWIAAAAAAEAPoIAAAAAAQACUAAAAAABAAbaAAAAAAEAAJgAAAAAAgA6AAWIAAAAAAX/JAA4GAAAAAAIAOgADNAAAAAACAAQUAAAAAAQAhAAduAAAAAAL/ggAH6AAAAAAEAIQAfmAAAAAABAAF6AAAAAAEABfoAAAAAAQAB5AAAAAABAAKUAAAAAAEADi4AAAAAAQAD1AAAAAABAAEwAAAAAAEABpIAAAAAAQAD2AAAAAABAP7lAAAAAAEAAV0AAAAAAQADRAAAAAABAADAAAAAAAEAAUcAAAAAAQAHMwAAAAACAEIAA9AAAAAABf8EAA2UAAAAAAgBCAAJyAAAAAAIAHigAAAAAAgAB5AAAAAACAAXYAAAAAAIAAegAAAAAAgACDgAAAAACAA5kAAAAAAIABYIAAAAAAgACCAAAAAACAAGQAAAAAAIAPtgAAAAABADEAAWQAAAAAAv9CAAauAAAAAAQAxAAHsAAAAAAEAB/0AAAAAAQAB0gAAAAABAAPcAAAAAAEAAaQAAAAAAQAA3QAAAAABAA73AAAAAAEAAL4AAAAAAQAD0gAAAAAA" + } + }, + { + "minTime": "1587686407139", + "maxTime": "1587688192139", + "raw": { + "data": "AHjG79+XtVxBGQjEAAAAAJh17EdrqRPuZfCUUOAD5lV/xR94AP5F+HdSeS6DZupEa5l/9ROULtNZRYXE36ag8iHroOYjpKBjIL6h9qBBIPUnj6BFIIghgKCDo5cgkKGHoLu4HfkKBLkGxIENQ4ygbNARqDgEBooEWR8lACkCk3+xBi8AKQJUgNOAF4dUv9KBCoAXg8VArqAr0/MoEvQGIgqRDpKArkD7oPsQIcj8ZApMAIQcvf8EDuQAhDgSBV0GOoD7QJW3If4CgGHQG7oBe0D4qAaRAZygDuQCQIPE4ATgFAv9qA4WAE4A60AzkAHQONX/JAJ5AB0BjSAL5D2AgDYQCGIDaYAHgH6v/KByaAB4BG0BksAHwCWX/FAF1AB8H0aAKdAKYgMZQCQIHqoARgDUv96AvOAEYAXUAWMAIR9H3/BAKLACEBsyAIBAKIgcCQDtoDxUAyA=" + } + }, + { + "minTime": "1587688207139", + "maxTime": "1587689992139", + "raw": { + "data": "AHjGzLuZtVxBGp6wAAAAAJh15F/rQlEbpDTcy+Ao4sAHzB7/iiPQAfDjcTfXqBJIfMglyO1oHDACMF5v96H9wAjAp07nQTBDRUFFQJxHFUHeQxxAlW8791QDM6k610E11Jg8AJ8yuZv9qFGAE+pPvdCsNSruRnhKFXRqiDfQtKcIQ76NKQnKCkcTPw3YjldSdvAB3Qrfb/kjfgA6Q5zH9upFHADEzX+h0K3uAGJGHIzxlCPI+xCUoLlPEIfhGFocZwO/Y0CPIDJQasgcqACodAv+yA/IAKgVJB/CBM0eQoFZQbMgRaADYcDv+aAsIANgURBmyBxU+nIDZQKcg4uADoEPv+SHJYAOgNVBeqA5IAMgWS/6I8cgAyB+kGlQADAmN//Q5DAAMC1oMCQIYgJZtQ/4t1JvboTn03XYj1cyeG6FWTAAMXt//SawADyL8K6k9dIt0JwqWAA=" + } + }, + { + "minTime": "1587690007139", + "maxTime": "1587691792139", + "raw": { + "data": "AHjGqZebtVxBHDKkAAAAAJh16k/10Kx1IeiucTfscAIQY+/4IEzACEP0QXNA1UcOQQxDe0EdgAGeR3/9BxYABhtCCAoHwjv6D0wAXDLL/qggMAF8Dv31Aa6BjEB7IMIQKwh35A76DB0BCoFLR5agYNBlKBH0BtodXQOQg9NA3qfGECbID8QZygRRD1KBSUHBIHUR++gd9B9qBDkOcoGZQcmgYlAXTef/I0BAUDPUB0EH7UByEDp0Bs0PEEB8EDwkBd0HZEBB0DR0A/kBB0ffUBCoAPgZrf8UBq4APgFeg7/ACUBv7/cgbJACUBAEAv4AUh+ff7EBTYAUgGIgZTAAUBb7/8g7hAAUBgEAkEDL0+7kA80BkUAgUDBUBZUH4UB80D/kB5UP50BNEDfUBbUH6EBn0AwkDZ0B/YAmgETf2kfYYAmgPIA=" + } + }, + { + "minTime": "1587691807139", + "maxTime": "1587693592139", + "raw": { + "data": "AHXGhvOctVxBHcnIAAAAAJh16k0NNlzL66iPJySI+5Ge3oXtBFI9xCMJ3JB7IXhHIoTCAENx3+Df8HUiNgBDoTj3YjrdR5LmXzFF4TBUSByM/NQl6M6Qpic+QfCFiR4iE/cS/uoJDsP7dCuvQYo2YAUhBv9ifyAFIVUaDF793JygAAAAByv0r/kAAAAAAADqgSQCagAAAAAACACoAqwAAAAAAAgAxAAAAAAAEACwHy0AAAAAAC/9oAIWAAAAAABAAsAOTAAAAAAAQAPIAAAAAACAD4AMWAAAAAABf8UAD3gAAAAAAgA+APnIAAAAAAIABlgAAAAAAgAKcAAAAAACAB/gAAAAAAIABcAAAAAAAgAKqAAAAAAEAFQAfQAAAAAAC/7IACBAAAAAABABUADOAAAAAAAQAC6AAAAAABAAWIAAAAAAEAPoIAAAAAAQACUAAAAAABAAbaAAAAAAEAAJgAAAAAAgA6AAWIAAAAAAX/JAA4GAAAAAAIAOgADNAAAAAACAAQUAAAAAAQAhAAduAAAAAAL/ggAH6AAAAAAEAIQAfmAAAAAABAAF6AAAAAAEABfoAAAAAAQAB5AAAAAABAAKUAAAAAAEADi4AAAAAAQAD1AAAAAABAAEwAAAAAAEABpIAAAAAAQAD2AAAAAABAP7lAAAAAAEAAV0AAAAAAQADRAAAAAABAADAAAAAAAEAAUcAAAAAAQAHMwAAAAACAEIAA9AAAAAABf8EAA2UAAAAAAgBCAAJyAAAAAAIAHigAAAAAAgAB5AAAAAACAAXYAAAAAAIAAegAAAAAAgACDgAAAAACAA5kAAAAAAIABYIAAAAAAgACCAAAAAACAAGQAAAAAAIAPtgAAAAABADEAAWQAAAAAAv9CAAauAAAAAAQAxAAHsAAAAAAEAB/0AAAAAAQAB0gAAAAABAAPcAAAAAAEAAaQAAAAAAQAA3QAAAAABAA73AAAAAAEAAL4AAAAAAQAD0gAAAAAA" + } + }, + { + "minTime": "1587679207139", + "maxTime": "1587680992139", + "raw": { + "data": "AHjG+/CQtVxBEeOIAAAAAJh16jw3Qrd0qbmVwqUJ1Jnrch/6qASJAaagF8QOIoBHUBtKDt5ALqgMsQCHIHP0A8CAxNAKwh7eQG8oB8ED96AdlAY4gEVQeboBBUBmSAUhA18gPWQCRI/FUApCAwFA1OgHUQHzoBV0HsSAYtAZkgF3QOPIB0UB66ASdD1YgNyQCWoHF0AgMAHwBeX/FAXTAB8HiiARCAGIDJL/QgEAgBiAO1A44gLOQCaJ/tUApSAyVAJmgbVQGXoBfoAUg8Uv9iARqAFIDN0Aw6Bu5AGygNxQCcIfHIAPgFdv+KA1mAD4BR0DeiA6ZAPMg+9QCDIDJEAmCB3FAKAgMfQCao/sEAmqAwxAJ+gclQDioDK0AvqDypAI8gKXQBKIHWkBUSANZAK+h8ygBCApC/4IA8IAQgFCQOmoBN4AOgLFv+SD20A=" + } + }, + { + "minTime": "1587681007139", + "maxTime": "1587682792139", + "raw": { + "data": "AHjG2MyStVxBE+SMAAAAAJh16FaPACE9V/wcyuHgBCNtNuhQRINqP/FUAUFAO/gBKALHf7kB3uAEoAaKAWSgBkQADA6tv/6AIDAAMA8tAFAQHKkAUxANLQBHUHy9AETQDZUAV9AdWgA+AIbf8UA+uAD4AiiB7egBmQAZAECv+iAYnABkASRAcGQBNEAxtAEzQ/b0ARZANDQG50AQtAMtQBJkAO5A6/QDBkAUZADBQGgUAydAF7QffYAUgDW3+xANigBSAP0gPuIA66AfigCgIHcqAZTACEARe/4IDqkAIQBXUANhALeR/ekAWNAM3QAzEARZAcDQBIkA0lAHCQPH0AfRAPQQBaUBzJACtQBA0A79AEBgBSD7Ff7EAVeAFIBvyALQgO+IAvqAHMgHVoAryB98gDmoB+iAI2gOlwAhAETv+CAYRACEAVFD84QBSoAVgGKX+tAFFgBWA+kgDvIBiCAPageUIAiaAeCgC5IDbSAGAA==" + } + }, + { + "minTime": "1587682807139", + "maxTime": "1587684592139", + "raw": { + "data": "AHjGtaiUtVxBFbqwAAAAAJh16kxVxN+SoPygSCGPoI0juKD+oZGg6afBIECg3cAHQOm/5IZfAB0FBR81B30BbQwdBAG5D+30BiKAKSAKYBxL/igcEv8GATmAUIDNb/IgFCgByDxZAI2gCoQG0IGxEAjaAPZAaAgFzQ+XIBc0Bk6AeZA6SgDDgBCARi/4IDDYAQgFCQfuQAfAOBf8UB7MAHwCgoA4EDqSAQFAeogEER+VoB2EB6KAdJA/CgHkQGIoBQUHkqAQpAZCgElQOfICyEAWKAclD66gEeQBroDfUA3KBzZAJKgDRQFfIPXUAj6AMOAEoDE7/cgbbgBKAUpAYCgELT+WIBA0BqsAHwCSX/FAGrAB8DnCAbWAE4Akr/agO2gBOASdAFdACEHpV/wQCfQAhAfIgHwQPcoBBkAbKAzpAKwh/sQD9IDzYAXgHcv9KB7aAF4AmUAjaA3xAJag7VA=" + } + }, + { + "minTime": "1587684607139", + "maxTime": "1587686392139", + "raw": { + "data": "AHjGkoSWtVxBF2kcAAAAAJh16k0VM5zL7qjwJsSJyhSOBn7bqRmuhGa7EaLqTa3MXrqT9XQjPdh7LqJLkZ6KhukbIhA06wg10Kij/EIE4k/10KqlHQmXcy+/ogCGMmjo3bkZ5egu0KyiyE5EgtkK0jA0MA2Jf/SoAQNAGKoAZNAO6IAThAGHoAQFAB/ID3xACtoA19ADXoB1BAC9IAz1AC1IHwxACGIAwxAC1oB2FABTIAXZAHgoAKBACEoD0JADjoA2xABgIAQJAOuIAcFABcoAuqAGIPx6/0IASyAGIAwFACVQAhABDX/BAOBQAhADvoA2FACMoD7RACJoAzqAFIAce/2IB+OAFIAR9AGOwAhACK3/BA+HwAhADYoAwKAFIASS/2IAM2AFIBytAC/wAlAHe3+5ABlQAlACjoDzeAGIAde/0IA3+AGIAM5AC9IAHJAOIoAXhAHuwAxACM3+hD+CwAxACVg=" + } + }, + { + "minTime": "1587679207139", + "maxTime": "1587680992139", + "raw": { + "data": "AHjG+/CQtVxBEeOIAAAAAJh16jw3Qrd0qbmVwqUJ1Jnrch/6qASJAaagF8QOIoBHUBtKDt5ALqgMsQCHIHP0A8CAxNAKwh7eQG8oB8ED96AdlAY4gEVQeboBBUBmSAUhA18gPWQCRI/FUApCAwFA1OgHUQHzoBV0HsSAYtAZkgF3QOPIB0UB66ASdD1YgNyQCWoHF0AgMAHwBeX/FAXTAB8HiiARCAGIDJL/QgEAgBiAO1A44gLOQCaJ/tUApSAyVAJmgbVQGXoBfoAUg8Uv9iARqAFIDN0Aw6Bu5AGygNxQCcIfHIAPgFdv+KA1mAD4BR0DeiA6ZAPMg+9QCDIDJEAmCB3FAKAgMfQCao/sEAmqAwxAJ+gclQDioDK0AvqDypAI8gKXQBKIHWkBUSANZAK+h8ygBCApC/4IA8IAQgFCQOmoBN4AOgLFv+SD20A=" + } + }, + { + "minTime": "1587681007139", + "maxTime": "1587682792139", + "raw": { + "data": "AHjG2MyStVxBE+SMAAAAAJh16FaPACE9V/wcyuHgBCNtNuhQRINqP/FUAUFAO/gBKALHf7kB3uAEoAaKAWSgBkQADA6tv/6AIDAAMA8tAFAQHKkAUxANLQBHUHy9AETQDZUAV9AdWgA+AIbf8UA+uAD4AiiB7egBmQAZAECv+iAYnABkASRAcGQBNEAxtAEzQ/b0ARZANDQG50AQtAMtQBJkAO5A6/QDBkAUZADBQGgUAydAF7QffYAUgDW3+xANigBSAP0gPuIA66AfigCgIHcqAZTACEARe/4IDqkAIQBXUANhALeR/ekAWNAM3QAzEARZAcDQBIkA0lAHCQPH0AfRAPQQBaUBzJACtQBA0A79AEBgBSD7Ff7EAVeAFIBvyALQgO+IAvqAHMgHVoAryB98gDmoB+iAI2gOlwAhAETv+CAYRACEAVFD84QBSoAVgGKX+tAFFgBWA+kgDvIBiCAPageUIAiaAeCgC5IDbSAGAA==" + } + }, + { + "minTime": "1587682807139", + "maxTime": "1587684592139", + "raw": { + "data": "AHjGtaiUtVxBFbqwAAAAAJh16kxVxN+SoPygSCGPoI0juKD+oZGg6afBIECg3cAHQOm/5IZfAB0FBR81B30BbQwdBAG5D+30BiKAKSAKYBxL/igcEv8GATmAUIDNb/IgFCgByDxZAI2gCoQG0IGxEAjaAPZAaAgFzQ+XIBc0Bk6AeZA6SgDDgBCARi/4IDDYAQgFCQfuQAfAOBf8UB7MAHwCgoA4EDqSAQFAeogEER+VoB2EB6KAdJA/CgHkQGIoBQUHkqAQpAZCgElQOfICyEAWKAclD66gEeQBroDfUA3KBzZAJKgDRQFfIPXUAj6AMOAEoDE7/cgbbgBKAUpAYCgELT+WIBA0BqsAHwCSX/FAGrAB8DnCAbWAE4Akr/agO2gBOASdAFdACEHpV/wQCfQAhAfIgHwQPcoBBkAbKAzpAKwh/sQD9IDzYAXgHcv9KB7aAF4AmUAjaA3xAJag7VA=" + } + }, + { + "minTime": "1587684607139", + "maxTime": "1587686392139", + "raw": { + "data": "AHjGkoSWtVxBF2kcAAAAAJh16k0VM5zL7qjwJsSJyhSOBn7bqRmuhGa7EaLqTa3MXrqT9XQjPdh7LqJLkZ6KhukbIhA06wg10Kij/EIE4k/10KqlHQmXcy+/ogCGMmjo3bkZ5egu0KyiyE5EgtkK0jA0MA2Jf/SoAQNAGKoAZNAO6IAThAGHoAQFAB/ID3xACtoA19ADXoB1BAC9IAz1AC1IHwxACGIAwxAC1oB2FABTIAXZAHgoAKBACEoD0JADjoA2xABgIAQJAOuIAcFABcoAuqAGIPx6/0IASyAGIAwFACVQAhABDX/BAOBQAhADvoA2FACMoD7RACJoAzqAFIAce/2IB+OAFIAR9AGOwAhACK3/BA+HwAhADYoAwKAFIASS/2IAM2AFIBytAC/wAlAHe3+5ABlQAlACjoDzeAGIAde/0IA3+AGIAM5AC9IAHJAOIoAXhAHuwAxACM3+hD+CwAxACVg=" + } + }, + { + "minTime": "1587686407139", + "maxTime": "1587688192139", + "raw": { + "data": "AHjG79+XtVxBGQjEAAAAAJh17EdrqRPuZfCUUOAD5lV/xR94AP5F+HdSeS6DZupEa5l/9ROULtNZRYXE36ag8iHroOYjpKBjIL6h9qBBIPUnj6BFIIghgKCDo5cgkKGHoLu4HfkKBLkGxIENQ4ygbNARqDgEBooEWR8lACkCk3+xBi8AKQJUgNOAF4dUv9KBCoAXg8VArqAr0/MoEvQGIgqRDpKArkD7oPsQIcj8ZApMAIQcvf8EDuQAhDgSBV0GOoD7QJW3If4CgGHQG7oBe0D4qAaRAZygDuQCQIPE4ATgFAv9qA4WAE4A60AzkAHQONX/JAJ5AB0BjSAL5D2AgDYQCGIDaYAHgH6v/KByaAB4BG0BksAHwCWX/FAF1AB8H0aAKdAKYgMZQCQIHqoARgDUv96AvOAEYAXUAWMAIR9H3/BAKLACEBsyAIBAKIgcCQDtoDxUAyA=" + } + }, + { + "minTime": "1587688207139", + "maxTime": "1587689992139", + "raw": { + "data": "AHjGzLuZtVxBGp6wAAAAAJh15F/rQlEbpDTcy+Ao4sAHzB7/iiPQAfDjcTfXqBJIfMglyO1oHDACMF5v96H9wAjAp07nQTBDRUFFQJxHFUHeQxxAlW8791QDM6k610E11Jg8AJ8yuZv9qFGAE+pPvdCsNSruRnhKFXRqiDfQtKcIQ76NKQnKCkcTPw3YjldSdvAB3Qrfb/kjfgA6Q5zH9upFHADEzX+h0K3uAGJGHIzxlCPI+xCUoLlPEIfhGFocZwO/Y0CPIDJQasgcqACodAv+yA/IAKgVJB/CBM0eQoFZQbMgRaADYcDv+aAsIANgURBmyBxU+nIDZQKcg4uADoEPv+SHJYAOgNVBeqA5IAMgWS/6I8cgAyB+kGlQADAmN//Q5DAAMC1oMCQIYgJZtQ/4t1JvboTn03XYj1cyeG6FWTAAMXt//SawADyL8K6k9dIt0JwqWAA=" + } + }, + { + "minTime": "1587690007139", + "maxTime": "1587691792139", + "raw": { + "data": "AHjGqZebtVxBHDKkAAAAAJh16k/10Kx1IeiucTfscAIQY+/4IEzACEP0QXNA1UcOQQxDe0EdgAGeR3/9BxYABhtCCAoHwjv6D0wAXDLL/qggMAF8Dv31Aa6BjEB7IMIQKwh35A76DB0BCoFLR5agYNBlKBH0BtodXQOQg9NA3qfGECbID8QZygRRD1KBSUHBIHUR++gd9B9qBDkOcoGZQcmgYlAXTef/I0BAUDPUB0EH7UByEDp0Bs0PEEB8EDwkBd0HZEBB0DR0A/kBB0ffUBCoAPgZrf8UBq4APgFeg7/ACUBv7/cgbJACUBAEAv4AUh+ff7EBTYAUgGIgZTAAUBb7/8g7hAAUBgEAkEDL0+7kA80BkUAgUDBUBZUH4UB80D/kB5UP50BNEDfUBbUH6EBn0AwkDZ0B/YAmgETf2kfYYAmgPIA=" + } + }, + { + "minTime": "1587691807139", + "maxTime": "1587693592139", + "raw": { + "data": "AHXGhvOctVxBHcnIAAAAAJh16k0NNlzL66iPJySI+5Ge3oXtBFI9xCMJ3JB7IXhHIoTCAENx3+Df8HUiNgBDoTj3YjrdR5LmXzFF4TBUSByM/NQl6M6Qpic+QfCFiR4iE/cS/uoJDsP7dCuvQYo2YAUhBv9ifyAFIVUaDF793JygAAAAByv0r/kAAAAAAADqgSQCagAAAAAACACoAqwAAAAAAAgAxAAAAAAAEACwHy0AAAAAAC/9oAIWAAAAAABAAsAOTAAAAAAAQAPIAAAAAACAD4AMWAAAAAABf8UAD3gAAAAAAgA+APnIAAAAAAIABlgAAAAAAgAKcAAAAAACAB/gAAAAAAIABcAAAAAAAgAKqAAAAAAEAFQAfQAAAAAAC/7IACBAAAAAABABUADOAAAAAAAQAC6AAAAAABAAWIAAAAAAEAPoIAAAAAAQACUAAAAAABAAbaAAAAAAEAAJgAAAAAAgA6AAWIAAAAAAX/JAA4GAAAAAAIAOgADNAAAAAACAAQUAAAAAAQAhAAduAAAAAAL/ggAH6AAAAAAEAIQAfmAAAAAABAAF6AAAAAAEABfoAAAAAAQAB5AAAAAABAAKUAAAAAAEADi4AAAAAAQAD1AAAAAABAAEwAAAAAAEABpIAAAAAAQAD2AAAAAABAP7lAAAAAAEAAV0AAAAAAQADRAAAAAABAADAAAAAAAEAAUcAAAAAAQAHMwAAAAACAEIAA9AAAAAABf8EAA2UAAAAAAgBCAAJyAAAAAAIAHigAAAAAAgAB5AAAAAACAAXYAAAAAAIAAegAAAAAAgACDgAAAAACAA5kAAAAAAIABYIAAAAAAgACCAAAAAACAAGQAAAAAAIAPtgAAAAABADEAAWQAAAAAAv9CAAauAAAAAAQAxAAHsAAAAAAEAB/0AAAAAAQAB0gAAAAABAAPcAAAAAAEAAaQAAAAAAQAA3QAAAAABAA73AAAAAAEAAL4AAAAAAQAD0gAAAAAA" + } + } + ] + } +} +{ + "series": { + "labels": [ + { + "name": "__name__", + "value": "gitlab_transaction_cache_read_hit_count_total" + }, + { + "name": "action", + "value": "widget.json" + }, + { + "name": "controller", + "value": "Projects::MergeRequests::ContentController" + }, + { + "name": "env", + "value": "gprd" + }, + { + "name": "environment", + "value": "gprd" + }, + { + "name": "fqdn", + "value": "web-08-sv-gprd.c.gitlab-production.internal" + }, + { + "name": "instance", + "value": "web-08-sv-gprd.c.gitlab-production.internal:8083" + }, + { + "name": "job", + "value": "gitlab-rails" + }, + { + "name": "monitor", + "value": "app" + }, + { + "name": "provider", + "value": "gcp" + }, + { + "name": "region", + "value": "us-east" + }, + { + "name": "replica", + "value": "02" + }, + { + "name": "shard", + "value": "default" + }, + { + "name": "stage", + "value": "main" + }, + { + "name": "tier", + "value": "sv" + }, + { + "name": "type", + "value": "web" + } + ], + "chunks": [ + { + "minTime": "1587679205791", + "maxTime": "1587680990791", + "raw": { + "data": "AHi+5vCQtVxBEeNUAAAAAJh16g3QnqpwuZfCUqVFobj//mgHw6Ffqkjcy+LwAdELv+SbRAB3IzslCnoxtC+J3NDto/1DqOJn1boVWKSJzK411JhroT9UmnIn8ofuhH26k7NzJ38AGdCtlv+ic2ADOBn4LqRbuhWXuZe70XVMJRaoAk5F+7v7iGRAEkb5CWTiHUnhuhOdT6cS/twAj0Kq+/3pCoAR5k99EJDhN5FZyM8fRZiE0QUad0RYUAMwll/nbzP74AZ6ke7oRgupF65i7dB8LqRLuRXgwARC1f+HQnCwARPdzL6iiVpvqJK4l/9dSOGAD+hWe3/FIfAB/MvVqasiEuRX8upEG6FaukMcy+nor6ZuisuB3+GgV5Bg6BR0OUwAhAm9/wQe/ACECPI9xQJMgr5AUSHa0C6oNJQJIny5AoKDMoANgUK/5oc+gA2BD0FHI8rA" + } + }, + { + "minTime": "1587681005874", + "maxTime": "1587682790791", + "raw": { + "data": "AHjkxMyStVxBE+S8AAAAAMV06FYXAKcobmXwGAFotN/qTPoAWhtIkDah/2bqTLXQrs0uDmV2cAG9iMm/5uhV2wAbH25GdyoVVHloTdO8IXSAGY1q/zoTGAGeJv3tBMUMuQWJHeUENQ/lBbIATnmt/tQNyAE4IWhmKCeo7Egv6GTIKq4GfnupMldCt/cy9noj6ZahuIkzkX0uAGY29/ndSZfADMz3MvcqbwirOJv2tBzUM3QdqAEo+Tf7kGzgBKHToJQnoaCGIWcjrCCowAZA2L/oheEAGbz/62gLCgZygGggL6g4+gLegaYgPah4agI6gZjACsBaL/WgGhACsHZUBAUDg4ANgIvf80fXYANgL8gbWgKUg5ygI4gHygdrACEBe7/gh7LACEBNUD4EBAEHakBa0DdEBcIAIn5Xf+EBE4AIgZKgJIg9WgK8gZggIah7OgIbADcDbb/KgMrADcG/EA3w" + } + }, + { + "minTime": "1587682805791", + "maxTime": "1587684590791", + "raw": { + "data": "AHi+oKiUtVxBFbqAAAAAAJh16kQ7iZ/a6k0tOp0Jw7qTDXMvq8AGx/b/mmdwAbHHci/juxEG6j5XYjJdCc26k31zJ/6PqD7oVhqR9uP/wboVbqKyAPJs7+i5l8ZgFCL9f2MxOAD4r9/+8jPE4AVg8kK6jLkLWnPUJOjQULk4m/B0FWQwpBy0dUQMpBDEN6QeVPHUHFQ7pAw0EzR3xBIUOqQXRwM/ZdSaO6E6tLdzJ5aIboThU2XIzxFCfIyFC4pwJFZIJxDiuJf1XUmIp/uhPNSkdiMVzL3Sh5JcbkZ4uhH0GGjeU1+hEEYMhVW8/+QoCvIGNIBjoCkoOzoDDoBJIHKoC7IBzoemID1IGFICEQAjB0i/3oCEQAjAPFAwOAF4DK3+lHz+AF4COoGhwAbAVO/5oOwQAbAKZAQmAEoGx3+5AQWAEoeXgA==" + } + }, + { + "minTime": "1587684605791", + "maxTime": "1587686390791", + "raw": { + "data": "AHi+/YOWtVxBF2gkAAAAAJh17EerqRTuZfH0e1NrQzUWtwN/IoHkgxyA4upNxgA3mL2/5upHbABvQjfdiMeADuok3/JyM+hgA6FJRtUAGwit/zQX8AGzrSE0gAyPUv+iExgAziP/dSajACnQrF7/YmagBSI1zL4DABsTS/5po8AGxtEMeADeRntd/zQqMAGxcSc3gAyCOv+iFLgAyMIQ38AF7DP/7/q6kR8AF9CcC6k+GADeZfQ7/mhzQAbL6Q5IAdiC3+NyM9tgB2FFR3MAGwpV/zTu0AGwqSDugBCLSv+Dib9ngBCC2IYRACUEkv9yORQAlA6kFMgBGHj3+9ApoARgnye4wAhByL/ggSUAIQytBH4AZj2l/nQTSAGYFyhgkALQQ2/1OB32jAC0GXoDbgAiBq7/whwyACIExQbPADMBq3+dHQ8AMwIOg++AKIHtv7CHDYAogfFB+EAFwPhf9U/2QAXApyDMoARgOu/3oEmgBGHdEDtQAFBqN/+QH3AAUC9I/BgA6BX7/kgzKADoDfQK/ACMA33+9DnUAIwNWgzCAGYFyv87ef/D4AZgIMA=" + } + }, + { + "minTime": "1587686405791", + "maxTime": "1587688190791", + "raw": { + "data": "AHi+2t+XtVxBGQhgAAAAAKB17EZd/4dSLeACOZPbdSaDABvQr97/mkogA3kV5rqTyYAQ6E+2/4OpPlgBDmR+upMtSvdCNN1IiXEf+6kyN0K8lOQ5l9/QxoANi4X/NPggA2FUjqwAZDmv+jkZ97ABkOMjFUI4nPkJij6UNY4G/iUCDQdlAl3MvlKLKhMJyMAGw9b/miZQAbyL+ihSwAHGSv/rqTHYADmI5l9DgA6Prf8k12ADoniF5ABnEv8d/0QNoAM6kVboVYMAA8x+9//dx8cAA9RYuhXxwAnIrf7XIz7mAE4SVG1QAzCoX+dOLQAzBtIveADIMu/6IXeADNx/+HQD5dCthSFuZfo0aSAFJlV/sQ4oAUifuRnpsAGw+F/zRtcAGwY6GtgAuc1v+qFzgAuNnQX0AF8Tfb9/1QMeAC4Jah0cAGQTu/6I4vABkF4Q2iADILNf9EDXgAyfdIF+gr6GVoPyjtUAIQMK/4IWdACEBdQLFwN+zoEo6FYqiggA2dW/5uZP7gA3qT23Qr0UwgA=" + } + }, + { + "minTime": "1587688205791", + "maxTime": "1587689990791", + "raw": { + "data": "AHi+t7uZtVxBGp6wAAAAAJh15Ffm6kSboTo0wHMv+qKFABk3S/6IgMAGQ/HE37mgp0AFQ9y/7IInABUdBQN+ADoKZf8kPPgA6BaIIcncaGiILagQqOfIPmhnCBfIIpABG83+jf+HUjzgAjoVtqQRG85kd7qR2uhGy6kyuAFeRniN/rQmsAKwaSNfQkif1Qj6NqQfcADwtd/5cTP2YAH7EcrqTpXQrb4AMjMv+iR2ADOZPldSLGADJ1v+joVrsAGTxuRf8UIlHodSb+nu5jeupEa6FYikzcDv2WAIYEBv76A1YAhg1NA/SHfUB0oGxQZOgWmAGY9qv86BeoAZgwhAsaHd0BsIFYQZsgaKADoD2v+SfaIAOgSBBwiBFIAJhxq/7oDEgAmBOkGVIFMR+ugQhBsCBMUOJoEzQY6gQdAdDah/xbqTe3Qr6YAMltv+iIuADOY/FFHQnVwArOG/1uRnisAKwkSCCgBmMev86FLA" + } + }, + { + "minTime": "1587690005791", + "maxTime": "1587691790791", + "raw": { + "data": "AHi+lJebtVxBHDJAAAAAAJh16k9eABuhWj3/pJ1AA0c7iZ9WgPkF46Fe+lKRsuZHW6ky90JzaRzkV/LqTRYAW6D5b/U7DcwAt1JnrmR1wAD1I77/+6E4GAAZBOBn6rsRUupHynt0JxqXbmV1qNxDEmqRKuRnjqGKQciMeQqKfMQmSMmQq7ib8VQSpAvkMBQXJHnEFXQ5tBz0/ZQfBD4EHkR+VBp0NUQUdAtm8/+BoCfoHsoCtoPToDtoHyQATAe2/7oeeQATAQ1A1OAGoCbX+ZBz2AGoC1IBXwAbAxa/5oCRwAbH01AQGADoH53/JAXKADoBJoOhwApAY6/2IGqwApAOdAVSADIfq3/RAVWADIB3oGGQAZAV6/6IOKQAZAaZAO+ADYGGX/NASaADZ+KIDJwAhAIO/4IGCwAhAXdB+qAGYCAX+dAxSAGYDyof5wADAWy//oGUwADARhB6JAbhAIxA9ZAR2ADIBXX/RH1mADID8A==" + } + }, + { + "minTime": "1587691805791", + "maxTime": "1587693590791", + "raw": { + "data": "AHW+8fKctVxBHcmQAAAAAJh16k1VOpzK7V1JiroVxKThyK9d1Jr6XroV9aTxES5l8XRkE2NEHgBnch/gN/nQCOQAzAZUgCegBmAWq/zoHCoAZgFEQGIwADAOhf/0HisAAwCDIDbIAOgExv+SBwiADoAtkAmcAHQH33/JAKvAB0Ph6AS2ACoA0b/sgOWgAqADFAGkgBKQD+IAVUAF6AMti9+7k6AAAAAA5X5F/RYAAAAAAB1QOIAhgAAAAAAAgAyAAAAAAACA+igAAAAAAIALIAAAAAAAgBzMAAAAAACAB1AAAAAAAIAMZAAAAAABABkADnAAAAAAAv+iAPmYAAAAAAQAZAAOIAAAAAAEABcAAAAAAAgA6AB8gAAAAAAX/JAALAAAAAAAIAOgAL0AAAAAACAD/AAAAAAAQAfAARcAAAAAAL/igAZaAAAAAAEAHwACrAAAAAABAAWIAAAAAAEAPsYAAAAAAQACugAAAAABAAYwAAAAAAEAAO4AAAAAAQABfAAAAAABAAI8AAAAAAEADpgAAAAAAQACWAAAAAABAAdcAAAAAAIATgAFFAAAAAAF/tQAecAAAAAACAE4AAuAAAAAAAgAE/AAAAAACAAx0AAAAAAIABYgAAAAAAgAcbAAAAAACAAY8AAAAAAIAA/AAAAAABAD0ABogAAAAAAv8SAAfQAAAAAAQA9AP7VAAAAAAEAASIAAAAAAQAAwwAAAAABAANNAAAAAAEAAeEAAAAAAQAH/wAAAAABAAHrAAAAAAEAA2sAAAAAAQAA1QAAAAABAA70AAAAAAEAAOIAAAAAAgAyAAJ4AAAAAAX/RAAMDAAAAAAIAMgACfgAAAAACAA5GAAAAAAIABYIAAAAABABsAAOUAAAAAAv+aAAKSAAAAAAQAbAB8MAAAAAAEAATcAAAAAAQAD+QAAAAABAAEKAAAAAAEABxUAAAAAAQABUAAAAAABAANEAAAAAAIAZgACmgAAAAAF/nQAA3QAAAAACAGYAHggAAAAAAgACqAAAAAACAAeWAAAAAAA==" + } + }, + { + "minTime": "1587686405791", + "maxTime": "1587688190791", + "raw": { + "data": "AHi+2t+XtVxBGQhgAAAAAKB17EZd/4dSLeACOZPbdSaDABvQr97/mkogA3kV5rqTyYAQ6E+2/4OpPlgBDmR+upMtSvdCNN1IiXEf+6kyN0K8lOQ5l9/QxoANi4X/NPggA2FUjqwAZDmv+jkZ97ABkOMjFUI4nPkJij6UNY4G/iUCDQdlAl3MvlKLKhMJyMAGw9b/miZQAbyL+ihSwAHGSv/rqTHYADmI5l9DgA6Prf8k12ADoniF5ABnEv8d/0QNoAM6kVboVYMAA8x+9//dx8cAA9RYuhXxwAnIrf7XIz7mAE4SVG1QAzCoX+dOLQAzBtIveADIMu/6IXeADNx/+HQD5dCthSFuZfo0aSAFJlV/sQ4oAUifuRnpsAGw+F/zRtcAGwY6GtgAuc1v+qFzgAuNnQX0AF8Tfb9/1QMeAC4Jah0cAGQTu/6I4vABkF4Q2iADILNf9EDXgAyfdIF+gr6GVoPyjtUAIQMK/4IWdACEBdQLFwN+zoEo6FYqiggA2dW/5uZP7gA3qT23Qr0UwgA=" + } + }, + { + "minTime": "1587688205791", + "maxTime": "1587689990791", + "raw": { + "data": "AHi+t7uZtVxBGp6wAAAAAJh15Ffm6kSboTo0wHMv+qKFABk3S/6IgMAGQ/HE37mgp0AFQ9y/7IInABUdBQN+ADoKZf8kPPgA6BaIIcncaGiILagQqOfIPmhnCBfIIpABG83+jf+HUjzgAjoVtqQRG85kd7qR2uhGy6kyuAFeRniN/rQmsAKwaSNfQkif1Qj6NqQfcADwtd/5cTP2YAH7EcrqTpXQrb4AMjMv+iR2ADOZPldSLGADJ1v+joVrsAGTxuRf8UIlHodSb+nu5jeupEa6FYikzcDv2WAIYEBv76A1YAhg1NA/SHfUB0oGxQZOgWmAGY9qv86BeoAZgwhAsaHd0BsIFYQZsgaKADoD2v+SfaIAOgSBBwiBFIAJhxq/7oDEgAmBOkGVIFMR+ugQhBsCBMUOJoEzQY6gQdAdDah/xbqTe3Qr6YAMltv+iIuADOY/FFHQnVwArOG/1uRnisAKwkSCCgBmMev86FLA" + } + }, + { + "minTime": "1587690005791", + "maxTime": "1587691790791", + "raw": { + "data": "AHi+lJebtVxBHDJAAAAAAJh16k9eABuhWj3/pJ1AA0c7iZ9WgPkF46Fe+lKRsuZHW6ky90JzaRzkV/LqTRYAW6D5b/U7DcwAt1JnrmR1wAD1I77/+6E4GAAZBOBn6rsRUupHynt0JxqXbmV1qNxDEmqRKuRnjqGKQciMeQqKfMQmSMmQq7ib8VQSpAvkMBQXJHnEFXQ5tBz0/ZQfBD4EHkR+VBp0NUQUdAtm8/+BoCfoHsoCtoPToDtoHyQATAe2/7oeeQATAQ1A1OAGoCbX+ZBz2AGoC1IBXwAbAxa/5oCRwAbH01AQGADoH53/JAXKADoBJoOhwApAY6/2IGqwApAOdAVSADIfq3/RAVWADIB3oGGQAZAV6/6IOKQAZAaZAO+ADYGGX/NASaADZ+KIDJwAhAIO/4IGCwAhAXdB+qAGYCAX+dAxSAGYDyof5wADAWy//oGUwADARhB6JAbhAIxA9ZAR2ADIBXX/RH1mADID8A==" + } + }, + { + "minTime": "1587691805791", + "maxTime": "1587693590791", + "raw": { + "data": "AHW+8fKctVxBHcmQAAAAAJh16k1VOpzK7V1JiroVxKThyK9d1Jr6XroV9aTxES5l8XRkE2NEHgBnch/gN/nQCOQAzAZUgCegBmAWq/zoHCoAZgFEQGIwADAOhf/0HisAAwCDIDbIAOgExv+SBwiADoAtkAmcAHQH33/JAKvAB0Ph6AS2ACoA0b/sgOWgAqADFAGkgBKQD+IAVUAF6AMti9+7k6AAAAAA5X5F/RYAAAAAAB1QOIAhgAAAAAAAgAyAAAAAAACA+igAAAAAAIALIAAAAAAAgBzMAAAAAACAB1AAAAAAAIAMZAAAAAABABkADnAAAAAAAv+iAPmYAAAAAAQAZAAOIAAAAAAEABcAAAAAAAgA6AB8gAAAAAAX/JAALAAAAAAAIAOgAL0AAAAAACAD/AAAAAAAQAfAARcAAAAAAL/igAZaAAAAAAEAHwACrAAAAAABAAWIAAAAAAEAPsYAAAAAAQACugAAAAABAAYwAAAAAAEAAO4AAAAAAQABfAAAAAABAAI8AAAAAAEADpgAAAAAAQACWAAAAAABAAdcAAAAAAIATgAFFAAAAAAF/tQAecAAAAAACAE4AAuAAAAAAAgAE/AAAAAACAAx0AAAAAAIABYgAAAAAAgAcbAAAAAACAAY8AAAAAAIAA/AAAAAABAD0ABogAAAAAAv8SAAfQAAAAAAQA9AP7VAAAAAAEAASIAAAAAAQAAwwAAAAABAANNAAAAAAEAAeEAAAAAAQAH/wAAAAABAAHrAAAAAAEAA2sAAAAAAQAA1QAAAAABAA70AAAAAAEAAOIAAAAAAgAyAAJ4AAAAAAX/RAAMDAAAAAAIAMgACfgAAAAACAA5GAAAAAAIABYIAAAAABABsAAOUAAAAAAv+aAAKSAAAAAAQAbAB8MAAAAAAEAATcAAAAAAQAD+QAAAAABAAEKAAAAAAEABxUAAAAAAQABUAAAAAABAANEAAAAAAIAZgACmgAAAAAF/nQAA3QAAAAACAGYAHggAAAAAAgACqAAAAAACAAeWAAAAAAA==" + } + }, + { + "minTime": "1587679205791", + "maxTime": "1587680990791", + "raw": { + "data": "AHi+5vCQtVxBEeNUAAAAAJh16g3QnqpwuZfCUqVFobj//mgHw6Ffqkjcy+LwAdELv+SbRAB3IzslCnoxtC+J3NDto/1DqOJn1boVWKSJzK411JhroT9UmnIn8ofuhH26k7NzJ38AGdCtlv+ic2ADOBn4LqRbuhWXuZe70XVMJRaoAk5F+7v7iGRAEkb5CWTiHUnhuhOdT6cS/twAj0Kq+/3pCoAR5k99EJDhN5FZyM8fRZiE0QUad0RYUAMwll/nbzP74AZ6ke7oRgupF65i7dB8LqRLuRXgwARC1f+HQnCwARPdzL6iiVpvqJK4l/9dSOGAD+hWe3/FIfAB/MvVqasiEuRX8upEG6FaukMcy+nor6ZuisuB3+GgV5Bg6BR0OUwAhAm9/wQe/ACECPI9xQJMgr5AUSHa0C6oNJQJIny5AoKDMoANgUK/5oc+gA2BD0FHI8rA" + } + }, + { + "minTime": "1587681005874", + "maxTime": "1587682790791", + "raw": { + "data": "AHjkxMyStVxBE+S8AAAAAMV06FYXAKcobmXwGAFotN/qTPoAWhtIkDah/2bqTLXQrs0uDmV2cAG9iMm/5uhV2wAbH25GdyoVVHloTdO8IXSAGY1q/zoTGAGeJv3tBMUMuQWJHeUENQ/lBbIATnmt/tQNyAE4IWhmKCeo7Egv6GTIKq4GfnupMldCt/cy9noj6ZahuIkzkX0uAGY29/ndSZfADMz3MvcqbwirOJv2tBzUM3QdqAEo+Tf7kGzgBKHToJQnoaCGIWcjrCCowAZA2L/oheEAGbz/62gLCgZygGggL6g4+gLegaYgPah4agI6gZjACsBaL/WgGhACsHZUBAUDg4ANgIvf80fXYANgL8gbWgKUg5ygI4gHygdrACEBe7/gh7LACEBNUD4EBAEHakBa0DdEBcIAIn5Xf+EBE4AIgZKgJIg9WgK8gZggIah7OgIbADcDbb/KgMrADcG/EA3w" + } + }, + { + "minTime": "1587682805791", + "maxTime": "1587684590791", + "raw": { + "data": "AHi+oKiUtVxBFbqAAAAAAJh16kQ7iZ/a6k0tOp0Jw7qTDXMvq8AGx/b/mmdwAbHHci/juxEG6j5XYjJdCc26k31zJ/6PqD7oVhqR9uP/wboVbqKyAPJs7+i5l8ZgFCL9f2MxOAD4r9/+8jPE4AVg8kK6jLkLWnPUJOjQULk4m/B0FWQwpBy0dUQMpBDEN6QeVPHUHFQ7pAw0EzR3xBIUOqQXRwM/ZdSaO6E6tLdzJ5aIboThU2XIzxFCfIyFC4pwJFZIJxDiuJf1XUmIp/uhPNSkdiMVzL3Sh5JcbkZ4uhH0GGjeU1+hEEYMhVW8/+QoCvIGNIBjoCkoOzoDDoBJIHKoC7IBzoemID1IGFICEQAjB0i/3oCEQAjAPFAwOAF4DK3+lHz+AF4COoGhwAbAVO/5oOwQAbAKZAQmAEoGx3+5AQWAEoeXgA==" + } + }, + { + "minTime": "1587684605791", + "maxTime": "1587686390791", + "raw": { + "data": "AHi+/YOWtVxBF2gkAAAAAJh17EerqRTuZfH0e1NrQzUWtwN/IoHkgxyA4upNxgA3mL2/5upHbABvQjfdiMeADuok3/JyM+hgA6FJRtUAGwit/zQX8AGzrSE0gAyPUv+iExgAziP/dSajACnQrF7/YmagBSI1zL4DABsTS/5po8AGxtEMeADeRntd/zQqMAGxcSc3gAyCOv+iFLgAyMIQ38AF7DP/7/q6kR8AF9CcC6k+GADeZfQ7/mhzQAbL6Q5IAdiC3+NyM9tgB2FFR3MAGwpV/zTu0AGwqSDugBCLSv+Dib9ngBCC2IYRACUEkv9yORQAlA6kFMgBGHj3+9ApoARgnye4wAhByL/ggSUAIQytBH4AZj2l/nQTSAGYFyhgkALQQ2/1OB32jAC0GXoDbgAiBq7/whwyACIExQbPADMBq3+dHQ8AMwIOg++AKIHtv7CHDYAogfFB+EAFwPhf9U/2QAXApyDMoARgOu/3oEmgBGHdEDtQAFBqN/+QH3AAUC9I/BgA6BX7/kgzKADoDfQK/ACMA33+9DnUAIwNWgzCAGYFyv87ef/D4AZgIMA=" + } + }, + { + "minTime": "1587679205791", + "maxTime": "1587680990791", + "raw": { + "data": "AHi+5vCQtVxBEeNUAAAAAJh16g3QnqpwuZfCUqVFobj//mgHw6Ffqkjcy+LwAdELv+SbRAB3IzslCnoxtC+J3NDto/1DqOJn1boVWKSJzK411JhroT9UmnIn8ofuhH26k7NzJ38AGdCtlv+ic2ADOBn4LqRbuhWXuZe70XVMJRaoAk5F+7v7iGRAEkb5CWTiHUnhuhOdT6cS/twAj0Kq+/3pCoAR5k99EJDhN5FZyM8fRZiE0QUad0RYUAMwll/nbzP74AZ6ke7oRgupF65i7dB8LqRLuRXgwARC1f+HQnCwARPdzL6iiVpvqJK4l/9dSOGAD+hWe3/FIfAB/MvVqasiEuRX8upEG6FaukMcy+nor6ZuisuB3+GgV5Bg6BR0OUwAhAm9/wQe/ACECPI9xQJMgr5AUSHa0C6oNJQJIny5AoKDMoANgUK/5oc+gA2BD0FHI8rA" + } + }, + { + "minTime": "1587681005874", + "maxTime": "1587682790791", + "raw": { + "data": "AHjkxMyStVxBE+S8AAAAAMV06FYXAKcobmXwGAFotN/qTPoAWhtIkDah/2bqTLXQrs0uDmV2cAG9iMm/5uhV2wAbH25GdyoVVHloTdO8IXSAGY1q/zoTGAGeJv3tBMUMuQWJHeUENQ/lBbIATnmt/tQNyAE4IWhmKCeo7Egv6GTIKq4GfnupMldCt/cy9noj6ZahuIkzkX0uAGY29/ndSZfADMz3MvcqbwirOJv2tBzUM3QdqAEo+Tf7kGzgBKHToJQnoaCGIWcjrCCowAZA2L/oheEAGbz/62gLCgZygGggL6g4+gLegaYgPah4agI6gZjACsBaL/WgGhACsHZUBAUDg4ANgIvf80fXYANgL8gbWgKUg5ygI4gHygdrACEBe7/gh7LACEBNUD4EBAEHakBa0DdEBcIAIn5Xf+EBE4AIgZKgJIg9WgK8gZggIah7OgIbADcDbb/KgMrADcG/EA3w" + } + }, + { + "minTime": "1587682805791", + "maxTime": "1587684590791", + "raw": { + "data": "AHi+oKiUtVxBFbqAAAAAAJh16kQ7iZ/a6k0tOp0Jw7qTDXMvq8AGx/b/mmdwAbHHci/juxEG6j5XYjJdCc26k31zJ/6PqD7oVhqR9uP/wboVbqKyAPJs7+i5l8ZgFCL9f2MxOAD4r9/+8jPE4AVg8kK6jLkLWnPUJOjQULk4m/B0FWQwpBy0dUQMpBDEN6QeVPHUHFQ7pAw0EzR3xBIUOqQXRwM/ZdSaO6E6tLdzJ5aIboThU2XIzxFCfIyFC4pwJFZIJxDiuJf1XUmIp/uhPNSkdiMVzL3Sh5JcbkZ4uhH0GGjeU1+hEEYMhVW8/+QoCvIGNIBjoCkoOzoDDoBJIHKoC7IBzoemID1IGFICEQAjB0i/3oCEQAjAPFAwOAF4DK3+lHz+AF4COoGhwAbAVO/5oOwQAbAKZAQmAEoGx3+5AQWAEoeXgA==" + } + }, + { + "minTime": "1587684605791", + "maxTime": "1587686390791", + "raw": { + "data": "AHi+/YOWtVxBF2gkAAAAAJh17EerqRTuZfH0e1NrQzUWtwN/IoHkgxyA4upNxgA3mL2/5upHbABvQjfdiMeADuok3/JyM+hgA6FJRtUAGwit/zQX8AGzrSE0gAyPUv+iExgAziP/dSajACnQrF7/YmagBSI1zL4DABsTS/5po8AGxtEMeADeRntd/zQqMAGxcSc3gAyCOv+iFLgAyMIQ38AF7DP/7/q6kR8AF9CcC6k+GADeZfQ7/mhzQAbL6Q5IAdiC3+NyM9tgB2FFR3MAGwpV/zTu0AGwqSDugBCLSv+Dib9ngBCC2IYRACUEkv9yORQAlA6kFMgBGHj3+9ApoARgnye4wAhByL/ggSUAIQytBH4AZj2l/nQTSAGYFyhgkALQQ2/1OB32jAC0GXoDbgAiBq7/whwyACIExQbPADMBq3+dHQ8AMwIOg++AKIHtv7CHDYAogfFB+EAFwPhf9U/2QAXApyDMoARgOu/3oEmgBGHdEDtQAFBqN/+QH3AAUC9I/BgA6BX7/kgzKADoDfQK/ACMA33+9DnUAIwNWgzCAGYFyv87ef/D4AZgIMA=" + } + }, + { + "minTime": "1587686405791", + "maxTime": "1587688190791", + "raw": { + "data": "AHi+2t+XtVxBGQhgAAAAAKB17EZd/4dSLeACOZPbdSaDABvQr97/mkogA3kV5rqTyYAQ6E+2/4OpPlgBDmR+upMtSvdCNN1IiXEf+6kyN0K8lOQ5l9/QxoANi4X/NPggA2FUjqwAZDmv+jkZ97ABkOMjFUI4nPkJij6UNY4G/iUCDQdlAl3MvlKLKhMJyMAGw9b/miZQAbyL+ihSwAHGSv/rqTHYADmI5l9DgA6Prf8k12ADoniF5ABnEv8d/0QNoAM6kVboVYMAA8x+9//dx8cAA9RYuhXxwAnIrf7XIz7mAE4SVG1QAzCoX+dOLQAzBtIveADIMu/6IXeADNx/+HQD5dCthSFuZfo0aSAFJlV/sQ4oAUifuRnpsAGw+F/zRtcAGwY6GtgAuc1v+qFzgAuNnQX0AF8Tfb9/1QMeAC4Jah0cAGQTu/6I4vABkF4Q2iADILNf9EDXgAyfdIF+gr6GVoPyjtUAIQMK/4IWdACEBdQLFwN+zoEo6FYqiggA2dW/5uZP7gA3qT23Qr0UwgA=" + } + }, + { + "minTime": "1587688205791", + "maxTime": "1587689990791", + "raw": { + "data": "AHi+t7uZtVxBGp6wAAAAAJh15Ffm6kSboTo0wHMv+qKFABk3S/6IgMAGQ/HE37mgp0AFQ9y/7IInABUdBQN+ADoKZf8kPPgA6BaIIcncaGiILagQqOfIPmhnCBfIIpABG83+jf+HUjzgAjoVtqQRG85kd7qR2uhGy6kyuAFeRniN/rQmsAKwaSNfQkif1Qj6NqQfcADwtd/5cTP2YAH7EcrqTpXQrb4AMjMv+iR2ADOZPldSLGADJ1v+joVrsAGTxuRf8UIlHodSb+nu5jeupEa6FYikzcDv2WAIYEBv76A1YAhg1NA/SHfUB0oGxQZOgWmAGY9qv86BeoAZgwhAsaHd0BsIFYQZsgaKADoD2v+SfaIAOgSBBwiBFIAJhxq/7oDEgAmBOkGVIFMR+ugQhBsCBMUOJoEzQY6gQdAdDah/xbqTe3Qr6YAMltv+iIuADOY/FFHQnVwArOG/1uRnisAKwkSCCgBmMev86FLA" + } + }, + { + "minTime": "1587690005791", + "maxTime": "1587691790791", + "raw": { + "data": "AHi+lJebtVxBHDJAAAAAAJh16k9eABuhWj3/pJ1AA0c7iZ9WgPkF46Fe+lKRsuZHW6ky90JzaRzkV/LqTRYAW6D5b/U7DcwAt1JnrmR1wAD1I77/+6E4GAAZBOBn6rsRUupHynt0JxqXbmV1qNxDEmqRKuRnjqGKQciMeQqKfMQmSMmQq7ib8VQSpAvkMBQXJHnEFXQ5tBz0/ZQfBD4EHkR+VBp0NUQUdAtm8/+BoCfoHsoCtoPToDtoHyQATAe2/7oeeQATAQ1A1OAGoCbX+ZBz2AGoC1IBXwAbAxa/5oCRwAbH01AQGADoH53/JAXKADoBJoOhwApAY6/2IGqwApAOdAVSADIfq3/RAVWADIB3oGGQAZAV6/6IOKQAZAaZAO+ADYGGX/NASaADZ+KIDJwAhAIO/4IGCwAhAXdB+qAGYCAX+dAxSAGYDyof5wADAWy//oGUwADARhB6JAbhAIxA9ZAR2ADIBXX/RH1mADID8A==" + } + }, + { + "minTime": "1587691805791", + "maxTime": "1587693590791", + "raw": { + "data": "AHW+8fKctVxBHcmQAAAAAJh16k1VOpzK7V1JiroVxKThyK9d1Jr6XroV9aTxES5l8XRkE2NEHgBnch/gN/nQCOQAzAZUgCegBmAWq/zoHCoAZgFEQGIwADAOhf/0HisAAwCDIDbIAOgExv+SBwiADoAtkAmcAHQH33/JAKvAB0Ph6AS2ACoA0b/sgOWgAqADFAGkgBKQD+IAVUAF6AMti9+7k6AAAAAA5X5F/RYAAAAAAB1QOIAhgAAAAAAAgAyAAAAAAACA+igAAAAAAIALIAAAAAAAgBzMAAAAAACAB1AAAAAAAIAMZAAAAAABABkADnAAAAAAAv+iAPmYAAAAAAQAZAAOIAAAAAAEABcAAAAAAAgA6AB8gAAAAAAX/JAALAAAAAAAIAOgAL0AAAAAACAD/AAAAAAAQAfAARcAAAAAAL/igAZaAAAAAAEAHwACrAAAAAABAAWIAAAAAAEAPsYAAAAAAQACugAAAAABAAYwAAAAAAEAAO4AAAAAAQABfAAAAAABAAI8AAAAAAEADpgAAAAAAQACWAAAAAABAAdcAAAAAAIATgAFFAAAAAAF/tQAecAAAAAACAE4AAuAAAAAAAgAE/AAAAAACAAx0AAAAAAIABYgAAAAAAgAcbAAAAAACAAY8AAAAAAIAA/AAAAAABAD0ABogAAAAAAv8SAAfQAAAAAAQA9AP7VAAAAAAEAASIAAAAAAQAAwwAAAAABAANNAAAAAAEAAeEAAAAAAQAH/wAAAAABAAHrAAAAAAEAA2sAAAAAAQAA1QAAAAABAA70AAAAAAEAAOIAAAAAAgAyAAJ4AAAAAAX/RAAMDAAAAAAIAMgACfgAAAAACAA5GAAAAAAIABYIAAAAABABsAAOUAAAAAAv+aAAKSAAAAAAQAbAB8MAAAAAAEAATcAAAAAAQAD+QAAAAABAAEKAAAAAAEABxUAAAAAAQABUAAAAAABAANEAAAAAAIAZgACmgAAAAAF/nQAA3QAAAAACAGYAHggAAAAAAgACqAAAAAACAAeWAAAAAAA==" + } + } + ] + } +}