Docker input panic error on invalid label

[dahendel]

Trying to deploy telegraf agent in my environment which is managed by rancher, I get the below panic error. I am building telegraf from source since my container is based on Alpine Linux.

I pasted the stack trace as well.

panic: descriptor Desc{fqName: "docker_mem_inactive_anon", help: "Telegraf collected point 'docker_mem_inactive_anon'", constLabels: {}, variableLabels: [cont_id cont_image cont_name host io.rancher.container.system]} is invalid: "io.rancher.container.system" is not a valid label name

goroutine 109 [running]:
github.com/prometheus/client_golang/prometheus.MustRegister(0x7f987f293378, 0xc820636120)
/go/src/github.com/prometheus/client_golang/prometheus/registry.go:119 +0x6d
github.com/influxdata/telegraf/plugins/outputs/prometheus_client.(_PrometheusClient).Write(0xc820127540, 0xc8203d6000, 0x80, 0x80, 0x0, 0x0)
/go/src/github.com/influxdata/telegraf/plugins/outputs/prometheus_client/prometheus_client.go:96 +0x7dc
github.com/influxdata/telegraf/internal/models.(_RunningOutput).write(0xc820086080, 0xc8203d6000, 0x80, 0x80, 0x0, 0x0)
/go/src/github.com/influxdata/telegraf/internal/models/running_output.go:128 +0xd1
github.com/influxdata/telegraf/internal/models.(_RunningOutput).Write(0xc820086080, 0x0, 0x0)
/go/src/github.com/influxdata/telegraf/internal/models/running_output.go:103 +0xcc
github.com/influxdata/telegraf/agent.(_Agent).flush.func1(0xc8205f2120, 0xc820086080)
/go/src/github.com/influxdata/telegraf/agent/agent.go:257 +0x7a
created by github.com/influxdata/telegraf/agent.(*Agent).flush
/go/src/github.com/influxdata/telegraf/agent/agent.go:262 +0xc0

goroutine 1 [semacquire]:
sync.runtime_Semacquire(0xc82055619c)
/usr/lib/go/src/runtime/sema.go:43 +0x26
sync.(_WaitGroup).Wait(0xc820556190)
/usr/lib/go/src/sync/waitgroup.go:126 +0xb4
github.com/influxdata/telegraf/agent.(_Agent).gatherParallel(0xc82002a038, 0xc8200186c0, 0x0, 0x0)
/go/src/github.com/influxdata/telegraf/agent/agent.go:150 +0x145
github.com/influxdata/telegraf/agent.(*Agent).Run(0xc82002a038, 0xc8200184e0, 0x0, 0x0)
/go/src/github.com/influxdata/telegraf/agent/agent.go:378 +0xa84
main.main()
/go/src/github.com/influxdata/telegraf/cmd/telegraf/telegraf.go:269 +0x25f6

goroutine 17 [syscall, locked to thread]:
runtime.goexit()
/usr/lib/go/src/runtime/asm_amd64.s:1721 +0x1

goroutine 5 [syscall]:
os/signal.loop()
/usr/lib/go/src/os/signal/signal_unix.go:22 +0x18
created by os/signal.init.1
/usr/lib/go/src/os/signal/signal_unix.go:28 +0x37

goroutine 78 [semacquire]:
sync.runtime_Syncsemacquire(0xc820578b80)
/usr/lib/go/src/runtime/sema.go:237 +0x201
sync.(_Cond).Wait(0xc820578b70)
/usr/lib/go/src/sync/cond.go:62 +0x9b
io.(_pipe).read(0xc820578b40, 0xc82058ea00, 0x200, 0x200, 0x0, 0x0, 0x0)
/usr/lib/go/src/io/pipe.go:52 +0x2d2
io.(_PipeReader).Read(0xc82002a950, 0xc82058ea00, 0x200, 0x200, 0x0, 0x0, 0x0)
/usr/lib/go/src/io/pipe.go:134 +0x50
encoding/json.(_Decoder).refill(0xc820179040, 0x0, 0x0)
/usr/lib/go/src/encoding/json/stream.go:152 +0x287
encoding/json.(_Decoder).readValue(0xc820179040, 0x1, 0x0, 0x0)
/usr/lib/go/src/encoding/json/stream.go:128 +0x41b
encoding/json.(_Decoder).Decode(0xc820179040, 0xd86f80, 0xc8204d7200, 0x0, 0x0)
/usr/lib/go/src/encoding/json/stream.go:57 +0x159
github.com/fsouza/go-dockerclient.(_Client).Stats(0xc82016e6c0, 0xc8205c0200, 0x40, 0xc820019aa0, 0x0, 0xc820019b00, 0x12a05f200, 0x0, 0x0)
/go/src/github.com/fsouza/go-dockerclient/container.go:817 +0x497
github.com/influxdata/telegraf/plugins/inputs/docker.(_Docker).gatherContainer.func1(0xc8202021c0, 0xc8205c0200, 0x40, 0xc820019aa0, 0x1398e00, 0xc820019b00, 0x12a05f200)
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:218 +0x9a
created by github.com/influxdata/telegraf/plugins/inputs/docker.(*Docker).gatherContainer
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:222 +0x4dc

goroutine 40 [IO wait]:
net.runtime_pollWait(0x7f987f292a20, 0x72, 0xc820012170)
/usr/lib/go/src/runtime/netpoll.go:157 +0x60
net.(_pollDesc).Wait(0xc8201fa450, 0x72, 0x0, 0x0)
/usr/lib/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(_pollDesc).WaitRead(0xc8201fa450, 0x0, 0x0)
/usr/lib/go/src/net/fd_poll_runtime.go:78 +0x36
net.(_netFD).accept(0xc8201fa3f0, 0x0, 0x7f987f239720, 0xc82024c220)
/usr/lib/go/src/net/fd_unix.go:408 +0x27c
net.(_TCPListener).AcceptTCP(0xc82002a0c0, 0xc820049e10, 0x0, 0x0)
/usr/lib/go/src/net/tcpsock_posix.go:254 +0x4d
net/http.tcpKeepAliveListener.Accept(0xc82002a0c0, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go/src/net/http/server.go:2135 +0x41
net/http.(_Server).Serve(0xc8200183c0, 0x7f987f2396e8, 0xc82002a0c0, 0x0, 0x0)
/usr/lib/go/src/net/http/server.go:1887 +0xb3
net/http.(_Server).ListenAndServe(0xc8200183c0, 0x0, 0x0)
/usr/lib/go/src/net/http/server.go:1877 +0x136
created by github.com/influxdata/telegraf/plugins/outputs/prometheus_client.(*PrometheusClient).Start
/go/src/github.com/influxdata/telegraf/plugins/outputs/prometheus_client/prometheus_client.go:34 +0x14a

goroutine 41 [select, locked to thread]:
runtime.gopark(0x139e6e8, 0xc820521f28, 0x1165be0, 0x6, 0x18, 0x2)
/usr/lib/go/src/runtime/proc.go:185 +0x163
runtime.selectgoImpl(0xc820521f28, 0x0, 0x18)
/usr/lib/go/src/runtime/select.go:392 +0xa64
runtime.selectgo(0xc820521f28)
/usr/lib/go/src/runtime/select.go:212 +0x12
runtime.ensureSigM.func1()
/usr/lib/go/src/runtime/signal1_unix.go:227 +0x353
runtime.goexit()
/usr/lib/go/src/runtime/asm_amd64.s:1721 +0x1

goroutine 42 [chan receive]:
main.main.func2(0xc820018540, 0xc8200184e0, 0xc820124540)
/go/src/github.com/influxdata/telegraf/cmd/telegraf/telegraf.go:241 +0x47
created by main.main
/go/src/github.com/influxdata/telegraf/cmd/telegraf/telegraf.go:251 +0x1ec3

goroutine 45 [select]:
github.com/influxdata/telegraf/agent.(_Agent).flusher(0xc82002a038, 0xc8200184e0, 0xc8200186c0, 0x0, 0x0)
/go/src/github.com/influxdata/telegraf/agent/agent.go:277 +0x340
github.com/influxdata/telegraf/agent.(_Agent).Run.func1(0xc82013cb20, 0xc82002a038, 0xc8200184e0, 0xc8200186c0)
/go/src/github.com/influxdata/telegraf/agent/agent.go:355 +0x7f
created by github.com/influxdata/telegraf/agent.(*Agent).Run
/go/src/github.com/influxdata/telegraf/agent/agent.go:359 +0x938

goroutine 31 [chan receive]:
github.com/influxdata/telegraf/plugins/inputs/docker.(_Docker).gatherContainer(0xc8202021c0, 0xc8205c0200, 0x40, 0xc8205a33c0, 0x11, 0xc820401890, 0x4, 0x56f169b0, 0xc8204019c0, 0xd, ...)
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:224 +0x509
github.com/influxdata/telegraf/plugins/inputs/docker.(_Docker).Gather.func1(0xc820401ed0, 0xc8202021c0, 0x7f987f239748, 0xc82012f3c0, 0xc8205c0200, 0x40, 0xc8205a33c0, 0x11, 0xc820401890, 0x4, ...)
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:104 +0xa5
created by github.com/influxdata/telegraf/plugins/inputs/docker.(*Docker).Gather
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:108 +0x4c6

goroutine 33 [chan receive]:
github.com/influxdata/telegraf/plugins/inputs/docker.(_Docker).gatherContainer(0xc8202021c0, 0xc8205c0380, 0x40, 0xc8205a3f00, 0x1d, 0xc8203fb2c0, 0x27, 0x56e98324, 0xc820401db0, 0xb, ...)
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:224 +0x509
github.com/influxdata/telegraf/plugins/inputs/docker.(_Docker).Gather.func1(0xc820401ed0, 0xc8202021c0, 0x7f987f239748, 0xc82012f3c0, 0xc8205c0380, 0x40, 0xc8205a3f00, 0x1d, 0xc8203fb2c0, 0x27, ...)
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:104 +0xa5
created by github.com/influxdata/telegraf/plugins/inputs/docker.(*Docker).Gather
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:108 +0x4c6

goroutine 100 [semacquire]:
sync.runtime_Syncsemacquire(0xc8204aa4c0)
/usr/lib/go/src/runtime/sema.go:237 +0x201
sync.(_Cond).Wait(0xc8204aa4b0)
/usr/lib/go/src/sync/cond.go:62 +0x9b
io.(_pipe).read(0xc8204aa480, 0xc82014be00, 0x200, 0x200, 0x0, 0x0, 0x0)
/usr/lib/go/src/io/pipe.go:52 +0x2d2
io.(_PipeReader).Read(0xc8200bc108, 0xc82014be00, 0x200, 0x200, 0x0, 0x0, 0x0)
/usr/lib/go/src/io/pipe.go:134 +0x50
encoding/json.(_Decoder).refill(0xc82024a680, 0x0, 0x0)
/usr/lib/go/src/encoding/json/stream.go:152 +0x287
encoding/json.(_Decoder).readValue(0xc82024a680, 0x1, 0x0, 0x0)
/usr/lib/go/src/encoding/json/stream.go:128 +0x41b
encoding/json.(_Decoder).Decode(0xc82024a680, 0xd86f80, 0xc8201c3500, 0x0, 0x0)
/usr/lib/go/src/encoding/json/stream.go:57 +0x159
github.com/fsouza/go-dockerclient.(_Client).Stats(0xc82016e6c0, 0xc8205c0480, 0x40, 0xc820147320, 0xc820024f00, 0xc820147380, 0x12a05f200, 0x0, 0x0)
/go/src/github.com/fsouza/go-dockerclient/container.go:817 +0x497
github.com/influxdata/telegraf/plugins/inputs/docker.(_Docker).gatherContainer.func1(0xc8202021c0, 0xc8205c0480, 0x40, 0xc820147320, 0x1398e00, 0xc820147380, 0x12a05f200)
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:218 +0x9a
created by github.com/influxdata/telegraf/plugins/inputs/docker.(*Docker).gatherContainer
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:222 +0x4dc

goroutine 89 [semacquire]:
sync.runtime_Semacquire(0xc820401edc)
/usr/lib/go/src/runtime/sema.go:43 +0x26
sync.(_WaitGroup).Wait(0xc820401ed0)
/usr/lib/go/src/sync/waitgroup.go:126 +0xb4
github.com/influxdata/telegraf/plugins/inputs/docker.(_Docker).Gather(0xc8202021c0, 0x7f987f239748, 0xc82012f3c0, 0x0, 0x0)
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:110 +0x501
github.com/influxdata/telegraf/agent.(_Agent).gatherParallel.func1(0xc820556190, 0xc8200186c0, 0xc82002a038, 0x0, 0xc8201186f0)
/go/src/github.com/influxdata/telegraf/agent/agent.go:139 +0x432
created by github.com/influxdata/telegraf/agent.(_Agent).gatherParallel
/go/src/github.com/influxdata/telegraf/agent/agent.go:143 +0x442

goroutine 29 [IO wait]:
net.runtime_pollWait(0x7f987f292ae0, 0x72, 0xc820012170)
/usr/lib/go/src/runtime/netpoll.go:157 +0x60
net.(_pollDesc).Wait(0xc82051a8b0, 0x72, 0x0, 0x0)
/usr/lib/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(_pollDesc).WaitRead(0xc82051a8b0, 0x0, 0x0)
/usr/lib/go/src/net/fd_poll_runtime.go:78 +0x36
net.(_netFD).Read(0xc82051a850, 0xc820588000, 0x1000, 0x1000, 0x0, 0x7f987f325050, 0xc820012170)
/usr/lib/go/src/net/fd_unix.go:232 +0x23a
net.(_conn).Read(0xc8200bc088, 0xc820588000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/lib/go/src/net/net.go:172 +0xe4
net/http.noteEOFReader.Read(0x7f987f32aa18, 0xc8200bc088, 0xc8201d2058, 0xc820588000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/lib/go/src/net/http/transport.go:1370 +0x67
net/http.(_noteEOFReader).Read(0xc820573680, 0xc820588000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
:126 +0xd0
bufio.(_Reader).fill(0xc8201471a0)
/usr/lib/go/src/bufio/bufio.go:97 +0x1e9
bufio.(_Reader).Peek(0xc8201471a0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go/src/bufio/bufio.go:132 +0xcc
net/http.(_persistConn).readLoop(0xc8201d2000)
/usr/lib/go/src/net/http/transport.go:876 +0xf7
created by net/http.(*Transport).dialConn
/usr/lib/go/src/net/http/transport.go:685 +0xc78

goroutine 75 [semacquire]:
sync.runtime_Syncsemacquire(0xc820578ac0)
/usr/lib/go/src/runtime/sema.go:237 +0x201
sync.(_Cond).Wait(0xc820578ab0)
/usr/lib/go/src/sync/cond.go:62 +0x9b
io.(_pipe).read(0xc820578a80, 0xc82058e800, 0x200, 0x200, 0x0, 0x0, 0x0)
/usr/lib/go/src/io/pipe.go:52 +0x2d2
io.(_PipeReader).Read(0xc82002a940, 0xc82058e800, 0x200, 0x200, 0x0, 0x0, 0x0)
/usr/lib/go/src/io/pipe.go:134 +0x50
encoding/json.(_Decoder).refill(0xc820178ea0, 0x0, 0x0)
/usr/lib/go/src/encoding/json/stream.go:152 +0x287
encoding/json.(_Decoder).readValue(0xc820178ea0, 0x1, 0x0, 0x0)
/usr/lib/go/src/encoding/json/stream.go:128 +0x41b
encoding/json.(_Decoder).Decode(0xc820178ea0, 0xd86f80, 0xc8204d6f00, 0x0, 0x0)
/usr/lib/go/src/encoding/json/stream.go:57 +0x159
github.com/fsouza/go-dockerclient.(_Client).Stats(0xc82016e6c0, 0xc8205c02c0, 0x40, 0xc820019920, 0xc820025f00, 0xc820019980, 0x12a05f200, 0x0, 0x0)
/go/src/github.com/fsouza/go-dockerclient/container.go:817 +0x497
github.com/influxdata/telegraf/plugins/inputs/docker.(_Docker).gatherContainer.func1(0xc8202021c0, 0xc8205c02c0, 0x40, 0xc820019920, 0xc8200bc000, 0xc820019980, 0x12a05f200)
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:218 +0x9a
created by github.com/influxdata/telegraf/plugins/inputs/docker.(*Docker).gatherContainer
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:222 +0x4dc

goroutine 76 [IO wait]:
net.runtime_pollWait(0x7f987f292720, 0x72, 0xc820012170)
/usr/lib/go/src/runtime/netpoll.go:157 +0x60
net.(_pollDesc).Wait(0xc820124a70, 0x72, 0x0, 0x0)
/usr/lib/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(_pollDesc).WaitRead(0xc820124a70, 0x0, 0x0)
/usr/lib/go/src/net/fd_poll_runtime.go:78 +0x36
net.(_netFD).Read(0xc820124a10, 0xc8205c3000, 0x1000, 0x1000, 0x0, 0x7f987f325050, 0xc820012170)
/usr/lib/go/src/net/fd_unix.go:232 +0x23a
net.(_conn).Read(0xc82002a960, 0xc8205c3000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/lib/go/src/net/net.go:172 +0xe4
bufio.(_Reader).fill(0xc820019ce0)
/usr/lib/go/src/bufio/bufio.go:97 +0x1e9
bufio.(_Reader).ReadSlice(0xc820019ce0, 0xc81ffd130a, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go/src/bufio/bufio.go:328 +0x21a
bufio.(_Reader).ReadLine(0xc820019ce0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go/src/bufio/bufio.go:357 +0x53
net/textproto.(_Reader).readLineSlice(0xc8201b7a30, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go/src/net/textproto/reader.go:55 +0x81
net/textproto.(_Reader).ReadLine(0xc8201b7a30, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go/src/net/textproto/reader.go:36 +0x40
net/http.ReadResponse(0xc820019ce0, 0xc8205a76c0, 0xb108f8f, 0x0, 0x0)
/usr/lib/go/src/net/http/response.go:128 +0xf6
github.com/fsouza/go-dockerclient.(_Client).stream(0xc82016e6c0, 0x115b4c0, 0x3, 0xc820019c20, 0x5f, 0x10100, 0x0, 0x0, 0x0, 0x7f987f293328, ...)
/go/src/github.com/fsouza/go-dockerclient/client.go:490 +0x85f
github.com/fsouza/go-dockerclient.(_Client).Stats.func2(0xc82016e6c0, 0xc8205c02c0, 0x40, 0xc820019920, 0xc820019900, 0xc820019980, 0x12a05f200, 0xc82002a948, 0xc8200199e0)
/go/src/github.com/fsouza/go-dockerclient/container.go:782 +0x26d
created by github.com/fsouza/go-dockerclient.(_Client).Stats
/go/src/github.com/fsouza/go-dockerclient/container.go:801 +0x2e5

goroutine 99 [chan receive]:
github.com/influxdata/telegraf/plugins/inputs/docker.(_Docker).gatherContainer(0xc8202021c0, 0xc8205c0480, 0x40, 0xc820401e40, 0xe, 0xc8205e02e0, 0x1b, 0x56e97c35, 0xc820401ec0, 0xb, ...)
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:224 +0x509
github.com/influxdata/telegraf/plugins/inputs/docker.(_Docker).Gather.func1(0xc820401ed0, 0xc8202021c0, 0x7f987f239748, 0xc82012f3c0, 0xc8205c0480, 0x40, 0xc820401e40, 0xe, 0xc8205e02e0, 0x1b, ...)
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:104 +0xa5
created by github.com/influxdata/telegraf/plugins/inputs/docker.(*Docker).Gather
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:108 +0x4c6

goroutine 98 [chan receive]:
github.com/influxdata/telegraf/plugins/inputs/docker.(_Docker).gatherContainer(0xc8202021c0, 0xc8205c0400, 0x40, 0xc8205e01e0, 0x15, 0xc820401dd0, 0xb, 0x56e9825c, 0xc820401e20, 0xb, ...)
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:224 +0x509
github.com/influxdata/telegraf/plugins/inputs/docker.(_Docker).Gather.func1(0xc820401ed0, 0xc8202021c0, 0x7f987f239748, 0xc82012f3c0, 0xc8205c0400, 0x40, 0xc8205e01e0, 0x15, 0xc820401dd0, 0xb, ...)
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:104 +0xa5
created by github.com/influxdata/telegraf/plugins/inputs/docker.(*Docker).Gather
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:108 +0x4c6

goroutine 79 [IO wait]:
net.runtime_pollWait(0x7f987f292960, 0x72, 0xc820012170)
/usr/lib/go/src/runtime/netpoll.go:157 +0x60
net.(_pollDesc).Wait(0xc820124ca0, 0x72, 0x0, 0x0)
/usr/lib/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(_pollDesc).WaitRead(0xc820124ca0, 0x0, 0x0)
/usr/lib/go/src/net/fd_poll_runtime.go:78 +0x36
net.(_netFD).Read(0xc820124c40, 0xc8205f1000, 0x1000, 0x1000, 0x0, 0x7f987f325050, 0xc820012170)
/usr/lib/go/src/net/fd_unix.go:232 +0x23a
net.(_conn).Read(0xc82002a968, 0xc8205f1000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/lib/go/src/net/net.go:172 +0xe4
bufio.(_Reader).fill(0xc820019e60)
/usr/lib/go/src/bufio/bufio.go:97 +0x1e9
bufio.(_Reader).ReadSlice(0xc820019e60, 0xc81ffd130a, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go/src/bufio/bufio.go:328 +0x21a
bufio.(_Reader).ReadLine(0xc820019e60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go/src/bufio/bufio.go:357 +0x53
net/textproto.(_Reader).readLineSlice(0xc82054da30, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go/src/net/textproto/reader.go:55 +0x81
net/textproto.(_Reader).ReadLine(0xc82054da30, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go/src/net/textproto/reader.go:36 +0x40
net/http.ReadResponse(0xc820019e60, 0xc8205a77a0, 0xb1305bb, 0x0, 0x0)
/usr/lib/go/src/net/http/response.go:128 +0xf6
github.com/fsouza/go-dockerclient.(_Client).stream(0xc82016e6c0, 0x115b4c0, 0x3, 0xc820019da0, 0x5f, 0x10100, 0x0, 0x0, 0x0, 0x7f987f293328, ...)
/go/src/github.com/fsouza/go-dockerclient/client.go:490 +0x85f
github.com/fsouza/go-dockerclient.(_Client).Stats.func2(0xc82016e6c0, 0xc8205c0200, 0x40, 0xc820019aa0, 0xc820019b00, 0xc820019b00, 0x12a05f200, 0xc82002a958, 0xc820019b60)
/go/src/github.com/fsouza/go-dockerclient/container.go:782 +0x26d
created by github.com/fsouza/go-dockerclient.(_Client).Stats
/go/src/github.com/fsouza/go-dockerclient/container.go:801 +0x2e5

goroutine 30 [select]:
net/http.(_persistConn).writeLoop(0xc8201d2000)
/usr/lib/go/src/net/http/transport.go:1009 +0x40c
created by net/http.(_Transport).dialConn
/usr/lib/go/src/net/http/transport.go:686 +0xc9d

goroutine 77 [select]:
github.com/fsouza/go-dockerclient.(_Client).Stats.func3(0xc8205c02c0, 0x40, 0xc820019920, 0xc820019900, 0xc820019980, 0x12a05f200, 0xc82002a940, 0xc820019a40)
/go/src/github.com/fsouza/go-dockerclient/container.go:807 +0xe3
created by github.com/fsouza/go-dockerclient.(_Client).Stats
/go/src/github.com/fsouza/go-dockerclient/container.go:813 +0x3a7

goroutine 32 [chan receive]:
github.com/influxdata/telegraf/plugins/inputs/docker.(_Docker).gatherContainer(0xc8202021c0, 0xc8205c02c0, 0x40, 0xc8205a3720, 0x18, 0xc8205a3740, 0x13, 0x56f16888, 0xc820401c30, 0xc, ...)
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:224 +0x509
github.com/influxdata/telegraf/plugins/inputs/docker.(_Docker).Gather.func1(0xc820401ed0, 0xc8202021c0, 0x7f987f239748, 0xc82012f3c0, 0xc8205c02c0, 0x40, 0xc8205a3720, 0x18, 0xc8205a3740, 0x13, ...)
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:104 +0xa5
created by github.com/influxdata/telegraf/plugins/inputs/docker.(*Docker).Gather
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:108 +0x4c6

goroutine 101 [IO wait]:
net.runtime_pollWait(0x7f987f2928a0, 0x72, 0xc820012170)
/usr/lib/go/src/runtime/netpoll.go:157 +0x60
net.(_pollDesc).Wait(0xc82051aed0, 0x72, 0x0, 0x0)
/usr/lib/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(_pollDesc).WaitRead(0xc82051aed0, 0x0, 0x0)
/usr/lib/go/src/net/fd_poll_runtime.go:78 +0x36
net.(_netFD).Read(0xc82051ae70, 0xc8205df000, 0x1000, 0x1000, 0x0, 0x7f987f325050, 0xc820012170)
/usr/lib/go/src/net/fd_unix.go:232 +0x23a
net.(_conn).Read(0xc8200bc138, 0xc8205df000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/lib/go/src/net/net.go:172 +0xe4
bufio.(_Reader).fill(0xc820147860)
/usr/lib/go/src/bufio/bufio.go:97 +0x1e9
bufio.(_Reader).ReadSlice(0xc820147860, 0xc81ffd6c0a, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go/src/bufio/bufio.go:328 +0x21a
bufio.(_Reader).ReadLine(0xc820147860, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go/src/bufio/bufio.go:357 +0x53
net/textproto.(_Reader).readLineSlice(0xc8201b5a30, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go/src/net/textproto/reader.go:55 +0x81
net/textproto.(_Reader).ReadLine(0xc8201b5a30, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go/src/net/textproto/reader.go:36 +0x40
net/http.ReadResponse(0xc820147860, 0xc8200c6460, 0xb1087a2, 0x0, 0x0)
/usr/lib/go/src/net/http/response.go:128 +0xf6
github.com/fsouza/go-dockerclient.(_Client).stream(0xc82016e6c0, 0x115b4c0, 0x3, 0xc8201477a0, 0x5f, 0x10100, 0x0, 0x0, 0x0, 0x7f987f293328, ...)
/go/src/github.com/fsouza/go-dockerclient/client.go:490 +0x85f
github.com/fsouza/go-dockerclient.(_Client).Stats.func2(0xc82016e6c0, 0xc8205c0480, 0x40, 0xc820147320, 0xc820147300, 0xc820147380, 0x12a05f200, 0xc8200bc110, 0xc8201473e0)
/go/src/github.com/fsouza/go-dockerclient/container.go:782 +0x26d
created by github.com/fsouza/go-dockerclient.(_Client).Stats
/go/src/github.com/fsouza/go-dockerclient/container.go:801 +0x2e5

goroutine 102 [select]:
github.com/fsouza/go-dockerclient.(_Client).Stats.func3(0xc8205c0480, 0x40, 0xc820147320, 0xc820147300, 0xc820147380, 0x12a05f200, 0xc8200bc108, 0xc820147440)
/go/src/github.com/fsouza/go-dockerclient/container.go:807 +0xe3
created by github.com/fsouza/go-dockerclient.(_Client).Stats
/go/src/github.com/fsouza/go-dockerclient/container.go:813 +0x3a7

goroutine 103 [semacquire]:
sync.runtime_Syncsemacquire(0xc8204aa580)
/usr/lib/go/src/runtime/sema.go:237 +0x201
sync.(_Cond).Wait(0xc8204aa570)
/usr/lib/go/src/sync/cond.go:62 +0x9b
io.(_pipe).read(0xc8204aa540, 0xc8205e4000, 0x200, 0x200, 0x0, 0x0, 0x0)
/usr/lib/go/src/io/pipe.go:52 +0x2d2
io.(_PipeReader).Read(0xc8200bc118, 0xc8205e4000, 0x200, 0x200, 0x0, 0x0, 0x0)
/usr/lib/go/src/io/pipe.go:134 +0x50
encoding/json.(_Decoder).refill(0xc82024a820, 0x0, 0x0)
/usr/lib/go/src/encoding/json/stream.go:152 +0x287
encoding/json.(_Decoder).readValue(0xc82024a820, 0x1, 0x0, 0x0)
/usr/lib/go/src/encoding/json/stream.go:128 +0x41b
encoding/json.(_Decoder).Decode(0xc82024a820, 0xd86f80, 0xc8201c3800, 0x0, 0x0)
/usr/lib/go/src/encoding/json/stream.go:57 +0x159
github.com/fsouza/go-dockerclient.(_Client).Stats(0xc82016e6c0, 0xc8205c0380, 0x40, 0xc8201474a0, 0xc820025700, 0xc820147500, 0x12a05f200, 0x0, 0x0)
/go/src/github.com/fsouza/go-dockerclient/container.go:817 +0x497
github.com/influxdata/telegraf/plugins/inputs/docker.(_Docker).gatherContainer.func1(0xc8202021c0, 0xc8205c0380, 0x40, 0xc8201474a0, 0x1398e00, 0xc820147500, 0x12a05f200)
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:218 +0x9a
created by github.com/influxdata/telegraf/plugins/inputs/docker.(*Docker).gatherContainer
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:222 +0x4dc

goroutine 104 [IO wait]:
net.runtime_pollWait(0x7f987f2927e0, 0x72, 0xc820012170)
/usr/lib/go/src/runtime/netpoll.go:157 +0x60
net.(_pollDesc).Wait(0xc82051b020, 0x72, 0x0, 0x0)
/usr/lib/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(_pollDesc).WaitRead(0xc82051b020, 0x0, 0x0)
/usr/lib/go/src/net/fd_poll_runtime.go:78 +0x36
net.(_netFD).Read(0xc82051afc0, 0xc8205ef000, 0x1000, 0x1000, 0x0, 0x7f987f325050, 0xc820012170)
/usr/lib/go/src/net/fd_unix.go:232 +0x23a
net.(_conn).Read(0xc8200bc140, 0xc8205ef000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/lib/go/src/net/net.go:172 +0xe4
bufio.(_Reader).fill(0xc8201479e0)
/usr/lib/go/src/bufio/bufio.go:97 +0x1e9
bufio.(_Reader).ReadSlice(0xc8201479e0, 0xc81ffd6c0a, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go/src/bufio/bufio.go:328 +0x21a
bufio.(_Reader).ReadLine(0xc8201479e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go/src/bufio/bufio.go:357 +0x53
net/textproto.(_Reader).readLineSlice(0xc8201b1a30, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go/src/net/textproto/reader.go:55 +0x81
net/textproto.(_Reader).ReadLine(0xc8201b1a30, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go/src/net/textproto/reader.go:36 +0x40
net/http.ReadResponse(0xc8201479e0, 0xc8200c6540, 0xb125519, 0x0, 0x0)
/usr/lib/go/src/net/http/response.go:128 +0xf6
github.com/fsouza/go-dockerclient.(_Client).stream(0xc82016e6c0, 0x115b4c0, 0x3, 0xc820147920, 0x5f, 0x10100, 0x0, 0x0, 0x0, 0x7f987f293328, ...)
/go/src/github.com/fsouza/go-dockerclient/client.go:490 +0x85f
github.com/fsouza/go-dockerclient.(_Client).Stats.func2(0xc82016e6c0, 0xc8205c0380, 0x40, 0xc8201474a0, 0xc820147500, 0xc820147500, 0x12a05f200, 0xc8200bc120, 0xc820147560)
/go/src/github.com/fsouza/go-dockerclient/container.go:782 +0x26d
created by github.com/fsouza/go-dockerclient.(_Client).Stats
/go/src/github.com/fsouza/go-dockerclient/container.go:801 +0x2e5

goroutine 105 [select]:
github.com/fsouza/go-dockerclient.(_Client).Stats.func3(0xc8205c0380, 0x40, 0xc8201474a0, 0xc820147400, 0xc820147500, 0x12a05f200, 0xc8200bc118, 0xc8201475c0)
/go/src/github.com/fsouza/go-dockerclient/container.go:807 +0xe3
created by github.com/fsouza/go-dockerclient.(_Client).Stats
/go/src/github.com/fsouza/go-dockerclient/container.go:813 +0x3a7

goroutine 80 [select]:
github.com/fsouza/go-dockerclient.(_Client).Stats.func3(0xc8205c0200, 0x40, 0xc820019aa0, 0xc820019a00, 0xc820019b00, 0x12a05f200, 0xc82002a950, 0xc820019bc0)
/go/src/github.com/fsouza/go-dockerclient/container.go:807 +0xe3
created by github.com/fsouza/go-dockerclient.(_Client).Stats
/go/src/github.com/fsouza/go-dockerclient/container.go:813 +0x3a7

goroutine 106 [semacquire]:
sync.runtime_Syncsemacquire(0xc8204aa640)
/usr/lib/go/src/runtime/sema.go:237 +0x201
sync.(_Cond).Wait(0xc8204aa630)
/usr/lib/go/src/sync/cond.go:62 +0x9b
io.(_pipe).read(0xc8204aa600, 0xc8205e4200, 0x200, 0x200, 0x0, 0x0, 0x0)
/usr/lib/go/src/io/pipe.go:52 +0x2d2
io.(_PipeReader).Read(0xc8200bc128, 0xc8205e4200, 0x200, 0x200, 0x0, 0x0, 0x0)
/usr/lib/go/src/io/pipe.go:134 +0x50
encoding/json.(_Decoder).refill(0xc82024a9c0, 0x0, 0x0)
/usr/lib/go/src/encoding/json/stream.go:152 +0x287
encoding/json.(_Decoder).readValue(0xc82024a9c0, 0x1, 0x0, 0x0)
/usr/lib/go/src/encoding/json/stream.go:128 +0x41b
encoding/json.(_Decoder).Decode(0xc82024a9c0, 0xd86f80, 0xc8201c3b00, 0x0, 0x0)
/usr/lib/go/src/encoding/json/stream.go:57 +0x159
github.com/fsouza/go-dockerclient.(_Client).Stats(0xc82016e6c0, 0xc8205c0400, 0x40, 0xc820147620, 0xc820029700, 0xc820147680, 0x12a05f200, 0x0, 0x0)
/go/src/github.com/fsouza/go-dockerclient/container.go:817 +0x497
github.com/influxdata/telegraf/plugins/inputs/docker.(_Docker).gatherContainer.func1(0xc8202021c0, 0xc8205c0400, 0x40, 0xc820147620, 0x7f987f1f8100, 0xc820147680, 0x12a05f200)
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:218 +0x9a
created by github.com/influxdata/telegraf/plugins/inputs/docker.(*Docker).gatherContainer
/go/src/github.com/influxdata/telegraf/plugins/inputs/docker/docker.go:222 +0x4dc

goroutine 107 [IO wait]:
net.runtime_pollWait(0x7f987f292660, 0x72, 0xc820012170)
/usr/lib/go/src/runtime/netpoll.go:157 +0x60
net.(_pollDesc).Wait(0xc82051b170, 0x72, 0x0, 0x0)
/usr/lib/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(_pollDesc).WaitRead(0xc82051b170, 0x0, 0x0)
/usr/lib/go/src/net/fd_poll_runtime.go:78 +0x36
net.(_netFD).Read(0xc82051b110, 0xc8205f5000, 0x1000, 0x1000, 0x0, 0x7f987f325050, 0xc820012170)
/usr/lib/go/src/net/fd_unix.go:232 +0x23a
net.(_conn).Read(0xc8200bc148, 0xc8205f5000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/lib/go/src/net/net.go:172 +0xe4
bufio.(_Reader).fill(0xc820147b60)
/usr/lib/go/src/bufio/bufio.go:97 +0x1e9
bufio.(_Reader).ReadSlice(0xc820147b60, 0xc81ffd6c0a, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go/src/bufio/bufio.go:328 +0x21a
bufio.(_Reader).ReadLine(0xc820147b60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go/src/bufio/bufio.go:357 +0x53
net/textproto.(_Reader).readLineSlice(0xc8201b3a30, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go/src/net/textproto/reader.go:55 +0x81
net/textproto.(_Reader).ReadLine(0xc8201b3a30, 0x0, 0x0, 0x0, 0x0)
/usr/lib/go/src/net/textproto/reader.go:36 +0x40
net/http.ReadResponse(0xc820147b60, 0xc8200c6620, 0xb13d295, 0x0, 0x0)
/usr/lib/go/src/net/http/response.go:128 +0xf6
github.com/fsouza/go-dockerclient.(_Client).stream(0xc82016e6c0, 0x115b4c0, 0x3, 0xc820147aa0, 0x5f, 0x10100, 0x0, 0x0, 0x0, 0x7f987f293328, ...)
/go/src/github.com/fsouza/go-dockerclient/client.go:490 +0x85f
github.com/fsouza/go-dockerclient.(_Client).Stats.func2(0xc82016e6c0, 0xc8205c0400, 0x40, 0xc820147620, 0xc820147600, 0xc820147680, 0x12a05f200, 0xc8200bc130, 0xc8201476e0)
/go/src/github.com/fsouza/go-dockerclient/container.go:782 +0x26d
created by github.com/fsouza/go-dockerclient.(_Client).Stats
/go/src/github.com/fsouza/go-dockerclient/container.go:801 +0x2e5

goroutine 108 [select]:
github.com/fsouza/go-dockerclient.(_Client).Stats.func3(0xc8205c0400, 0x40, 0xc820147620, 0xc820147600, 0xc820147680, 0x12a05f200, 0xc8200bc128, 0xc820147740)
/go/src/github.com/fsouza/go-dockerclient/container.go:807 +0xe3
created by github.com/fsouza/go-dockerclient.(_Client).Stats
/go/src/github.com/fsouza/go-dockerclient/container.go:813 +0x3a7

[sparrc]

can you provide version and config file?

[dahendel]

bash-4.3# ./telegraf version
Telegraf - Version 0.11.1-47-g7f65ffc

# Telegraf Configuration

# Telegraf is entirely plugin driven. All metrics are gathered from the
# declared inputs, and sent to the declared outputs.

# Plugins must be declared in here to be active.
# To deactivate a plugin, comment out the name and any variables.

# Use 'telegraf -config telegraf.conf -test' to see what metrics a config
# file would generate.

# Global tags can be specified here in key="value" format.
[global_tags]
  # dc = "us-east-1" # will tag all metrics with dc=us-east-1
  # rack = "1a"

# Configuration for telegraf agent
[agent]
  ## Default data collection interval for all inputs
  interval = "10s"
  ## Rounds collection interval to 'interval'
  ## ie, if interval="10s" then always collect on :00, :10, :20, etc.
  round_interval = true

  ## Telegraf will cache metric_buffer_limit metrics for each output, and will
  ## flush this buffer on a successful write.
  metric_buffer_limit = 1000
  ## Flush the buffer whenever full, regardless of flush_interval.
  flush_buffer_when_full = true

  ## Collection jitter is used to jitter the collection by a random amount.
  ## Each plugin will sleep for a random time within jitter before collecting.
  ## This can be used to avoid many plugins querying things like sysfs at the
  ## same time, which can have a measurable effect on the system.
  collection_jitter = "0s"

  ## Default flushing interval for all outputs. You shouldn't set this below
  ## interval. Maximum flush_interval will be flush_interval + flush_jitter
  flush_interval = "10s"
  ## Jitter the flush interval by a random amount. This is primarily to avoid
  ## large write spikes for users running a large number of telegraf instances.
  ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
  flush_jitter = "0s"

  ## Run telegraf in debug mode
  debug = false
  ## Run telegraf in quiet mode
  quiet = false
  ## Override default hostname, if empty use os.Hostname()
  hostname = ""
  ## If set to true, do no set the "host" tag in the telegraf agent.
  omit_hostname = false


#
# OUTPUTS:
#


# Configuration for the Prometheus client to spawn
[[outputs.prometheus_client]]
  ## Address to listen on
  # listen = ":9126"

#
# INPUTS:
#

# Read metrics about cpu usage
[[inputs.cpu]]
  ## Whether to report per-cpu stats or not
  percpu = true
  ## Whether to report total system cpu stats or not
  totalcpu = true
  ## Comment this line if you want the raw CPU time metrics
  fielddrop = ["time_*"]

# Read metrics about disk usage by mount point
[[inputs.disk]]
  ## By default, telegraf gather stats for all mountpoints.
  ## Setting mountpoints will restrict the stats to the specified mountpoints.
  mount_points = ["paths","paths"]


  ## Ignore some mountpoints by filesystem type. For example (dev)tmpfs (usually
  ## present on /run, /var/run, /dev/shm or /dev).
  ignore_fs = ["tmpfs", "devtmpfs"]

# Read metrics about disk IO by device
[[inputs.diskio]]
  ## By default, telegraf will gather stats for all devices including
  ## disk partitions.
  ## Setting devices will restrict the stats to the specified devices.
  # devices = ["sda", "sdb"]
  ## Uncomment the following line if you do not need disk serial numbers.
  # skip_serial_number = true

# Read metrics about docker containers
[[inputs.docker]]
  ## Docker Endpoint
  ##   To use TCP, set endpoint = "tcp://[ip]:[port]"
  ##   To use environment variables (ie, docker-machine), set endpoint = "ENV"
  endpoint = "unix:///var/run/docker/docker.sock"
  ## Only collect metrics for these containers, collect all if empty
  container_names = []

# Read metrics about memory usage
[[inputs.mem]]
  # no configuration

[sparrc]

looks like there's two issues here: we're trying to submit invalid prometheus label names, and we're panicking if there is an error

[liuxiang999888]

C:\Users\Administrator>docker run -d -p 80:80 docker/getting-started
panic: Invalid standard handle identifier: 4294967286

goroutine 1 [running]:
github.com/docker/cli/vendor/github.com/Azure/go-ansiterm/winterm.GetStdFile(0xfffffff6, 0xc000050768, 0x1e8)
/go/src/github.com/docker/cli/vendor/github.com/Azure/go-ansiterm/winterm/ansi.go:173 +0x1f3
github.com/docker/cli/vendor/github.com/moby/term/windows.NewAnsiReader(0xfffffff6, 0xc00000000f, 0x20d5a80)
/go/src/github.com/docker/cli/vendor/github.com/moby/term/windows/ansi_reader.go:34 +0x36
github.com/docker/cli/vendor/github.com/moby/term.StdStreams(0xc000404a90, 0xc0004a3eb8, 0x1, 0x1, 0x0, 0x0)
/go/src/github.com/docker/cli/vendor/github.com/moby/term/term_windows.go:75 +0x1eb
github.com/docker/cli/cli/command.NewDockerCli(0x0, 0x0, 0x0, 0x19ba60c, 0x1b500e0, 0xc0002041b0)
/go/src/github.com/docker/cli/cli/command/cli.go:473 +0x1a7
main.main()
/go/src/github.com/docker/cli/cmd/docker/docker.go:291 +0x4b

[liuxiang999888]

why