Skip to content

Commit

Permalink
influxdb input: Add shard counter
Browse files Browse the repository at this point in the history 
closes #1221
  • Loading branch information
@sparrc
sparrc committed May 18, 2016
1 parent e94e501 commit b065573
Show file tree
Hide file tree
Showing 3 changed files with 90 additions and 64 deletions.
124 changes: 62 additions & 62 deletions plugins/inputs/influxdb/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,16 +22,78 @@ InfluxDB-formatted endpoints. See below for more information.

### Measurements & Fields

- influxdb
- n_shards
- influxdb_database
- influxdb_httpd
- influxdb_measurement
- influxdb_memstats
- heap_inuse
- heap_released
- mspan_inuse
- total_alloc
- sys
- mallocs
- frees
- heap_idle
- pause_total_ns
- lookups
- heap_sys
- mcache_sys
- next_gc
- gcc_pu_fraction
- other_sys
- alloc
- stack_inuse
- stack_sys
- buck_hash_sys
- gc_sys
- num_gc
- heap_alloc
- heap_objects
- mspan_sys
- mcache_inuse
- last_gc
- influxdb_shard
- influxdb_subscriber
- influxdb_tsm1_cache
- influxdb_tsm1_wal
- influxdb_write

### Example Output:

```
telegraf -config ~/ws/telegraf.conf -input-filter influxdb -test
* Plugin: influxdb, Collection 1
> influxdb_database,database=_internal,host=tyrion,url=http://localhost:8086/debug/vars numMeasurements=10,numSeries=29 1463590500247354636
> influxdb_httpd,bind=:8086,host=tyrion,url=http://localhost:8086/debug/vars req=7,reqActive=1,reqDurationNs=14227734 1463590500247354636
> influxdb_measurement,database=_internal,host=tyrion,measurement=database,url=http://localhost:8086/debug/vars numSeries=1 1463590500247354636
> influxdb_measurement,database=_internal,host=tyrion,measurement=httpd,url=http://localhost:8086/debug/vars numSeries=1 1463590500247354636
> influxdb_measurement,database=_internal,host=tyrion,measurement=measurement,url=http://localhost:8086/debug/vars numSeries=10 1463590500247354636
> influxdb_measurement,database=_internal,host=tyrion,measurement=runtime,url=http://localhost:8086/debug/vars numSeries=1 1463590500247354636
> influxdb_measurement,database=_internal,host=tyrion,measurement=shard,url=http://localhost:8086/debug/vars numSeries=4 1463590500247354636
> influxdb_measurement,database=_internal,host=tyrion,measurement=subscriber,url=http://localhost:8086/debug/vars numSeries=1 1463590500247354636
> influxdb_measurement,database=_internal,host=tyrion,measurement=tsm1_cache,url=http://localhost:8086/debug/vars numSeries=4 1463590500247354636
> influxdb_measurement,database=_internal,host=tyrion,measurement=tsm1_filestore,url=http://localhost:8086/debug/vars numSeries=2 1463590500247354636
> influxdb_measurement,database=_internal,host=tyrion,measurement=tsm1_wal,url=http://localhost:8086/debug/vars numSeries=4 1463590500247354636
> influxdb_measurement,database=_internal,host=tyrion,measurement=write,url=http://localhost:8086/debug/vars numSeries=1 1463590500247354636
> influxdb_memstats,host=tyrion,url=http://localhost:8086/debug/vars alloc=7642384i,buck_hash_sys=1463471i,frees=1169558i,gc_sys=653312i,gcc_pu_fraction=0.00003825652361068311,heap_alloc=7642384i,heap_idle=9912320i,heap_inuse=9125888i,heap_objects=48276i,heap_released=0i,heap_sys=19038208i,last_gc=1463590480877651621i,lookups=90i,mallocs=1217834i,mcache_inuse=4800i,mcache_sys=16384i,mspan_inuse=70920i,mspan_sys=81920i,next_gc=11679787i,num_gc=141i,other_sys=1244233i,pause_total_ns=24034027i,stack_inuse=884736i,stack_sys=884736i,sys=23382264i,total_alloc=679012200i 1463590500277918755
> influxdb_shard,database=_internal,engine=tsm1,host=tyrion,id=4,path=/Users/sparrc/.influxdb/data/_internal/monitor/4,retentionPolicy=monitor,url=http://localhost:8086/debug/vars fieldsCreate=65,seriesCreate=26,writePointsOk=7274,writeReq=280 1463590500247354636
> influxdb_subscriber,host=tyrion,url=http://localhost:8086/debug/vars pointsWritten=7274 1463590500247354636
> influxdb_tsm1_cache,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/data/_internal/monitor/1,retentionPolicy=monitor,url=http://localhost:8086/debug/vars WALCompactionTimeMs=0,cacheAgeMs=2809192,cachedBytes=0,diskBytes=0,memBytes=0,snapshotCount=0 1463590500247354636
> influxdb_tsm1_cache,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/data/_internal/monitor/2,retentionPolicy=monitor,url=http://localhost:8086/debug/vars WALCompactionTimeMs=0,cacheAgeMs=2809184,cachedBytes=0,diskBytes=0,memBytes=0,snapshotCount=0 1463590500247354636
> influxdb_tsm1_cache,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/data/_internal/monitor/3,retentionPolicy=monitor,url=http://localhost:8086/debug/vars WALCompactionTimeMs=0,cacheAgeMs=2809180,cachedBytes=0,diskBytes=0,memBytes=42368,snapshotCount=0 1463590500247354636
> influxdb_tsm1_cache,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/data/_internal/monitor/4,retentionPolicy=monitor,url=http://localhost:8086/debug/vars WALCompactionTimeMs=0,cacheAgeMs=2799155,cachedBytes=0,diskBytes=0,memBytes=331216,snapshotCount=0 1463590500247354636
> influxdb_tsm1_filestore,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/data/_internal/monitor/1,retentionPolicy=monitor,url=http://localhost:8086/debug/vars diskBytes=37892 1463590500247354636
> influxdb_tsm1_filestore,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/data/_internal/monitor/2,retentionPolicy=monitor,url=http://localhost:8086/debug/vars diskBytes=52907 1463590500247354636
> influxdb_tsm1_wal,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/wal/_internal/monitor/1,retentionPolicy=monitor,url=http://localhost:8086/debug/vars currentSegmentDiskBytes=0,oldSegmentsDiskBytes=0 1463590500247354636
> influxdb_tsm1_wal,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/wal/_internal/monitor/2,retentionPolicy=monitor,url=http://localhost:8086/debug/vars currentSegmentDiskBytes=0,oldSegmentsDiskBytes=0 1463590500247354636
> influxdb_tsm1_wal,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/wal/_internal/monitor/3,retentionPolicy=monitor,url=http://localhost:8086/debug/vars currentSegmentDiskBytes=0,oldSegmentsDiskBytes=65651 1463590500247354636
> influxdb_tsm1_wal,database=_internal,host=tyrion,path=/Users/sparrc/.influxdb/wal/_internal/monitor/4,retentionPolicy=monitor,url=http://localhost:8086/debug/vars currentSegmentDiskBytes=495687,oldSegmentsDiskBytes=0 1463590500247354636
> influxdb_write,host=tyrion,url=http://localhost:8086/debug/vars pointReq=7274,pointReqLocal=7274,req=280,subWriteOk=280,writeOk=280 1463590500247354636
> influxdb_shard,host=tyrion n_shards=4i 1463590500247354636
```

### InfluxDB-formatted endpoints

The influxdb plugin can collect InfluxDB-formatted data from JSON endpoints.
Expand All @@ -46,65 +108,3 @@ With a configuration of:
"http://192.168.2.1:8086/debug/vars"
]
```

And if 127.0.0.1 responds with this JSON:

```json
{
"k1": {
"name": "fruit",
"tags": {
"kind": "apple"
},
"values": {
"inventory": 371,
"sold": 112
}
},
"k2": {
"name": "fruit",
"tags": {
"kind": "banana"
},
"values": {
"inventory": 1000,
"sold": 403
}
}
}
```

And if 192.168.2.1 responds like so:

```json
{
"k3": {
"name": "transactions",
"tags": {},
"values": {
"total": 100,
"balance": 184.75
}
}
}
```

Then the collected metrics will be:

```
influxdb_fruit,url='http://127.0.0.1:8086/debug/vars',kind='apple' inventory=371.0,sold=112.0
influxdb_fruit,url='http://127.0.0.1:8086/debug/vars',kind='banana' inventory=1000.0,sold=403.0
influxdb_transactions,url='http://192.168.2.1:8086/debug/vars' total=100.0,balance=184.75
```

There are two important details to note about the collected metrics:

1. Even though the values in JSON are being displayed as integers,
the metrics are reported as floats.
JSON encoders usually don't print the fractional part for round floats.
Because you cannot change the type of an existing field in InfluxDB,
we assume all numbers are floats.

2. The top-level keys' names (in the example above, `"k1"`, `"k2"`, and `"k3"`)
are not considered when recording the metrics.
16 changes: 16 additions & 0 deletions plugins/inputs/influxdb/influxdb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,9 @@ func (i *InfluxDB) gatherURL(
acc telegraf.Accumulator,
url string,
) error {
shardCounter := 0
now := time.Now()

resp, err := client.Get(url)
if err != nil {
return err
Expand Down Expand Up @@ -201,6 +204,10 @@ func (i *InfluxDB) gatherURL(
continue
}

if p.Name == "shard" {
shardCounter++
}

// If the object was a point, but was not fully initialized,
// ignore it and move on.
if p.Name == "" || p.Tags == nil || p.Values == nil || len(p.Values) == 0 {
Expand All @@ -214,9 +221,18 @@ func (i *InfluxDB) gatherURL(
"influxdb_"+p.Name,
p.Values,
p.Tags,
now,
)
}

acc.AddFields("influxdb",
map[string]interface{}{
"n_shards": shardCounter,
},
nil,
now,
)

return nil
}

Expand Down
14 changes: 12 additions & 2 deletions plugins/inputs/influxdb/influxdb_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ func TestBasic(t *testing.T) {
var acc testutil.Accumulator
require.NoError(t, plugin.Gather(&acc))

require.Len(t, acc.Metrics, 2)
require.Len(t, acc.Metrics, 3)
fields := map[string]interface{}{
// JSON will truncate floats to integer representations.
// Since there's no distinction in JSON, we can't assume it's an int.
Expand All @@ -50,6 +50,11 @@ func TestBasic(t *testing.T) {
"url": fakeServer.URL + "/endpoint",
}
acc.AssertContainsTaggedFields(t, "influxdb_bar", fields, tags)

acc.AssertContainsTaggedFields(t, "influxdb",
map[string]interface{}{
"n_shards": 0,
}, map[string]string{})
}

func TestInfluxDB(t *testing.T) {
Expand All @@ -69,7 +74,7 @@ func TestInfluxDB(t *testing.T) {
var acc testutil.Accumulator
require.NoError(t, plugin.Gather(&acc))

require.Len(t, acc.Metrics, 33)
require.Len(t, acc.Metrics, 34)

fields := map[string]interface{}{
"heap_inuse": int64(18046976),
Expand Down Expand Up @@ -104,6 +109,11 @@ func TestInfluxDB(t *testing.T) {
"url": fakeInfluxServer.URL + "/endpoint",
}
acc.AssertContainsTaggedFields(t, "influxdb_memstats", fields, tags)

acc.AssertContainsTaggedFields(t, "influxdb",
map[string]interface{}{
"n_shards": 2,
}, map[string]string{})
}

func TestErrorHandling(t *testing.T) {
Expand Down

0 comments on commit b065573

Please sign in to comment.