Fix prometheus_client reload behavior

wip

fixes #2282

agent/agent.go

@@ -392,5 +392,6 @@ func (a *Agent) Run(shutdown chan struct{}) error {
 	}
 
 	wg.Wait()
+	a.Close()
 	return nil
 }

circle.yml

@@ -4,9 +4,9 @@ machine:
   post:
     - sudo service zookeeper stop
     - go version
-    - go version | grep 1.7.4 || sudo rm -rf /usr/local/go
-    - wget https://storage.googleapis.com/golang/go1.7.4.linux-amd64.tar.gz
-    - sudo tar -C /usr/local -xzf go1.7.4.linux-amd64.tar.gz
+    - sudo rm -rf /usr/local/go
+    - wget https://storage.googleapis.com/golang/go1.8rc2.linux-amd64.tar.gz
+    - sudo tar -C /usr/local -xzf go1.8rc2.linux-amd64.tar.gz
     - go version
 
 dependencies:

plugins/outputs/prometheus_client/prometheus_client.go

@@ -1,6 +1,7 @@
 package prometheus_client
 
 import (
+	"context"
 	"fmt"
 	"log"
 	"net/http"
@@ -24,6 +25,7 @@ type MetricWithExpiration struct {
 type PrometheusClient struct {
 	Listen             string
 	ExpirationInterval internal.Duration `toml:"expiration_interval"`
+	server             *http.Server
 
 	metrics map[string]*MetricWithExpiration
 
@@ -41,30 +43,25 @@ var sampleConfig = `
 func (p *PrometheusClient) Start() error {
 	p.metrics = make(map[string]*MetricWithExpiration)
 	prometheus.Register(p)
-	defer func() {
-		if r := recover(); r != nil {
-			// recovering from panic here because there is no way to stop a
-			// running http go server except by a kill signal. Since the server
-			// does not stop on SIGHUP, Start() will panic when the process
-			// is reloaded.
-		}
-	}()
+
 	if p.Listen == "" {
 		p.Listen = "localhost:9126"
 	}
 
-	http.Handle("/metrics", prometheus.Handler())
-	server := &http.Server{
-		Addr: p.Listen,
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", prometheus.Handler())
+
+	p.server = &http.Server{
+		Addr:    p.Listen,
+		Handler: mux,
 	}
 
-	go server.ListenAndServe()
+	go p.server.ListenAndServe()
 	return nil
 }
 
 func (p *PrometheusClient) Stop() {
-	// TODO: Use a listener for http.Server that counts active connections
-	//       that can be stopped and closed gracefully
+	// plugin gets cleaned up in Close() already.
 }
 
 func (p *PrometheusClient) Connect() error {
@@ -73,8 +70,9 @@ func (p *PrometheusClient) Connect() error {
 }
 
 func (p *PrometheusClient) Close() error {
-	// This service output does not need to close any of its connections
-	return nil
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	return p.server.Shutdown(ctx)
 }
 
 func (p *PrometheusClient) SampleConfig() string {