Skip to content

Summary counter breaks under high load #189

Closed
@DifferentialOrange

Description

@DifferentialOrange

I tried to monitor event loop with

local function monitor(collector)
    local time_before
    while true do
        time_before = clock.monotonic()
        fiber.yield()
        collector:observer(clock.monotonic() - time_before)
    end
end

local function init()
    local collector = require('metrics').summary('tnt_fiber_event_loop', 'event loop time',
        { [0.5] = 0.01, [0.9] = 0.01, [0.99] = 0.01, })

    fiber.create(monitor, collector)
end

after half a minute I've got

{
   "label_pairs":{
      "alias":"tnt_router"
   },
   "timestamp":1612882820817429,
   "metric_name":"tnt_fiber_event_loop_count",
   "value":7766500
},
{
   "label_pairs":{
      "alias":"tnt_router"
   },
   "timestamp":1612882820817429,
   "metric_name":"tnt_fiber_event_loop_sum",
   "value":7.6474393837925
},
{
   "label_pairs":{
      "quantile":0.5,
      "alias":"tnt_router"
   },
   "timestamp":1612882820821171,
   "metric_name":"tnt_fiber_event_loop",
   "value":8.4099883679301e-07
},
{
   "label_pairs":{
      "quantile":0.9,
      "alias":"tnt_router"
   },
   "timestamp":1612882820873518,
   "metric_name":"tnt_fiber_event_loop",
   "value":"inf"
},
{
   "label_pairs":{
      "quantile":0.99,
      "alias":"tnt_router"
   },
   "timestamp":1612882820893734,
   "metric_name":"tnt_fiber_event_loop",
   "value":"inf"
}

Also observation count in collector wasn't equal to the one in structure

help: event loop time
    observations:
      '': 7997000
    name: tnt_fiber_event_loop_count
    label_pairs:
      '': &6 []
  registry: *1
  objectives:
    0.5: 0.01
    0.9: 0.01
    0.99: 0.01
  sum_collector:
    registry: *1
    help: event loop time
    observations:
      '': 8.0931331436805
    name: tnt_fiber_event_loop_sum
    label_pairs:
      '': *6
  help: event loop time
  observations:
    '':
      b: 'cdata<double [?]>: 0x4113c3e0'
      compress_cnt: 500
      __max_samples: 500
      b_len: 0
      stream:
        f: 'function: 0x4113c100'
        l_len: 141219
        l_cap: 194581
        l: 'cdata<struct 913 [?]>: 0x460de020'
        n: 8008162
      sorted: true
  name: tnt_fiber_event_loop
  label_pairs:
    '': []

Metadata

Metadata

Labels

bugSomething isn't working

Type

No type

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions