Support modules configuration (prometheus-community#146)

* support modules configuration

Signed-off-by: Ben Ye <ben.ye@bytedance.com>

* fallback default module if the param is missing

Signed-off-by: Ben Ye <ben.ye@bytedance.com>

* update readme and example config file

Signed-off-by: Ben Ye <ben.ye@bytedance.com>

* fix lint

Signed-off-by: Ben Ye <ben.ye@bytedance.com>

Author: Ben Ye

README.md

@@ -39,35 +39,37 @@ $ cat examples/data.json
 
 $ cat examples/config.yml
 ---
-metrics:
-- name: example_global_value
-  path: "{ .counter }"
-  help: Example of a top-level global value scrape in the json
-  labels:
-    environment: beta # static label
-    location: "planet-{.location}"          # dynamic label
-
-- name: example_value
-  type: object
-  help: Example of sub-level value scrapes from a json
-  path: '{.values[?(@.state == "ACTIVE")]}'
-  labels:
-    environment: beta # static label
-    id: '{.id}'          # dynamic label
-  values:
-    active: 1      # static value
-    count: '{.count}' # dynamic value
-    boolean: '{.some_boolean}'
-
-headers:
-  X-Dummy: my-test-header
+modules:
+  default:
+    metrics:
+    - name: example_global_value
+      path: "{ .counter }"
+      help: Example of a top-level global value scrape in the json
+      labels:
+        environment: beta # static label
+        location: "planet-{.location}"          # dynamic label
+
+    - name: example_value
+      type: object
+      help: Example of sub-level value scrapes from a json
+      path: '{.values[?(@.state == "ACTIVE")]}'
+      labels:
+        environment: beta # static label
+        id: '{.id}'          # dynamic label
+      values:
+        active: 1      # static value
+        count: '{.count}' # dynamic value
+        boolean: '{.some_boolean}'
+
+    headers:
+      X-Dummy: my-test-header
 
 $ python -m SimpleHTTPServer 8000 &
 Serving HTTP on 0.0.0.0 port 8000 ...
 
 $ ./json_exporter --config.file examples/config.yml &
 
-$ curl "http://localhost:7979/probe?target=http://localhost:8000/examples/data.json" | grep ^example
+$ curl "http://localhost:7979/probe?module=default&target=http://localhost:8000/examples/data.json" | grep ^example
 example_global_value{environment="beta",location="planet-mars"} 1234
 example_value_active{environment="beta",id="id-A"} 1
 example_value_active{environment="beta",id="id-C"} 1

cmd/main.go

@@ -16,6 +16,7 @@ package cmd
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"os"
 
@@ -86,9 +87,19 @@ func probeHandler(w http.ResponseWriter, r *http.Request, logger log.Logger, con
 	defer cancel()
 	r = r.WithContext(ctx)
 
+	module := r.URL.Query().Get("module")
+	if module == "" {
+		module = "default"
+	}
+	if _, ok := config.Modules[module]; !ok {
+		http.Error(w, fmt.Sprintf("Unknown module %q", module), http.StatusBadRequest)
+		level.Debug(logger).Log("msg", "Unknown module", "module", module)
+		return
+	}
+
 	registry := prometheus.NewPedanticRegistry()
 
-	metrics, err := exporter.CreateMetricsList(config)
+	metrics, err := exporter.CreateMetricsList(config.Modules[module])
 	if err != nil {
 		level.Error(logger).Log("msg", "Failed to create metrics list from config", "err", err)
 	}
@@ -102,7 +113,7 @@ func probeHandler(w http.ResponseWriter, r *http.Request, logger log.Logger, con
 		return
 	}
 
-	fetcher := exporter.NewJSONFetcher(ctx, logger, config, r.URL.Query())
+	fetcher := exporter.NewJSONFetcher(ctx, logger, config.Modules[module], r.URL.Query())
 	data, err := fetcher.FetchJSON(target)
 	if err != nil {
 		http.Error(w, "Failed to fetch JSON response. TARGET: "+target+", ERROR: "+err.Error(), http.StatusServiceUnavailable)

cmd/main_test.go

@@ -32,9 +32,9 @@ func TestFailIfSelfSignedCA(t *testing.T) {
 	}))
 	defer target.Close()
 
-	req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL, nil)
+	req := httptest.NewRequest("GET", "http://example.com/foo"+"?module=default&target="+target.URL, nil)
 	recorder := httptest.NewRecorder()
-	probeHandler(recorder, req, log.NewNopLogger(), config.Config{})
+	probeHandler(recorder, req, log.NewNopLogger(), config.Config{Modules: map[string]config.Module{"default": {}}})
 
 	resp := recorder.Result()
 	body, _ := ioutil.ReadAll(resp.Body)
@@ -45,13 +45,21 @@ func TestFailIfSelfSignedCA(t *testing.T) {
 }
 
 func TestSucceedIfSelfSignedCA(t *testing.T) {
-	c := config.Config{}
-	c.HTTPClientConfig.TLSConfig.InsecureSkipVerify = true
+	c := config.Config{
+		Modules: map[string]config.Module{
+			"default": {
+				HTTPClientConfig: pconfig.HTTPClientConfig{
+					TLSConfig: pconfig.TLSConfig{
+						InsecureSkipVerify: true,
+					},
+				},
+			}},
+	}
 	target := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 	}))
 	defer target.Close()
 
-	req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL, nil)
+	req := httptest.NewRequest("GET", "http://example.com/foo"+"?module=default&target="+target.URL, nil)
 	recorder := httptest.NewRecorder()
 	probeHandler(recorder, req, log.NewNopLogger(), c)
 
@@ -63,6 +71,29 @@ func TestSucceedIfSelfSignedCA(t *testing.T) {
 	}
 }
 
+func TestDefaultModule(t *testing.T) {
+	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	}))
+	defer target.Close()
+
+	req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL, nil)
+	recorder := httptest.NewRecorder()
+	probeHandler(recorder, req, log.NewNopLogger(), config.Config{Modules: map[string]config.Module{"default": {}}})
+
+	resp := recorder.Result()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("Default module test fails unexpectedly, expected 200, got %d", resp.StatusCode)
+	}
+
+	// Module doesn't exist.
+	recorder = httptest.NewRecorder()
+	probeHandler(recorder, req, log.NewNopLogger(), config.Config{Modules: map[string]config.Module{"foo": {}}})
+	resp = recorder.Result()
+	if resp.StatusCode != http.StatusBadRequest {
+		t.Fatalf("Default module test fails unexpectedly, expected 400, got %d", resp.StatusCode)
+	}
+}
+
 func TestFailIfTargetMissing(t *testing.T) {
 	req := httptest.NewRequest("GET", "http://example.com/foo", nil)
 	recorder := httptest.NewRecorder()
@@ -86,9 +117,9 @@ func TestDefaultAcceptHeader(t *testing.T) {
 	}))
 	defer target.Close()
 
-	req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL, nil)
+	req := httptest.NewRequest("GET", "http://example.com/foo"+"?module=default&target="+target.URL, nil)
 	recorder := httptest.NewRecorder()
-	probeHandler(recorder, req, log.NewNopLogger(), config.Config{})
+	probeHandler(recorder, req, log.NewNopLogger(), config.Config{Modules: map[string]config.Module{"default": {}}})
 
 	resp := recorder.Result()
 	body, _ := ioutil.ReadAll(resp.Body)
@@ -118,7 +149,7 @@ func TestCorrectResponse(t *testing.T) {
 			t.Fatalf("Failed to load config file %s", test.ConfigFile)
 		}
 
-		req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL+test.ServeFile, nil)
+		req := httptest.NewRequest("GET", "http://example.com/foo"+"?module=default&target="+target.URL+test.ServeFile, nil)
 		recorder := httptest.NewRecorder()
 		probeHandler(recorder, req, log.NewNopLogger(), c)
 
@@ -145,15 +176,21 @@ func TestBasicAuth(t *testing.T) {
 	}))
 	defer target.Close()
 
-	req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL, nil)
+	req := httptest.NewRequest("GET", "http://example.com/foo"+"?module=default&target="+target.URL, nil)
 	recorder := httptest.NewRecorder()
-	c := config.Config{}
-	auth := &pconfig.BasicAuth{
-		Username: username,
-		Password: pconfig.Secret(password),
+	c := config.Config{
+		Modules: map[string]config.Module{
+			"default": {
+				HTTPClientConfig: pconfig.HTTPClientConfig{
+					BasicAuth: &pconfig.BasicAuth{
+						Username: username,
+						Password: pconfig.Secret(password),
+					},
+				},
+			},
+		},
 	}
 
-	c.HTTPClientConfig.BasicAuth = auth
 	probeHandler(recorder, req, log.NewNopLogger(), c)
 
 	resp := recorder.Result()
@@ -175,11 +212,16 @@ func TestBearerToken(t *testing.T) {
 	}))
 	defer target.Close()
 
-	req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL, nil)
+	req := httptest.NewRequest("GET", "http://example.com/foo"+"?module=default&target="+target.URL, nil)
 	recorder := httptest.NewRecorder()
-	c := config.Config{}
+	c := config.Config{
+		Modules: map[string]config.Module{"default": {
+			HTTPClientConfig: pconfig.HTTPClientConfig{
+				BearerToken: pconfig.Secret(token),
+			},
+		}},
+	}
 
-	c.HTTPClientConfig.BearerToken = pconfig.Secret(token)
 	probeHandler(recorder, req, log.NewNopLogger(), c)
 
 	resp := recorder.Result()
@@ -206,10 +248,15 @@ func TestHTTPHeaders(t *testing.T) {
 	}))
 	defer target.Close()
 
-	req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL, nil)
+	req := httptest.NewRequest("GET", "http://example.com/foo"+"?module=default&target="+target.URL, nil)
 	recorder := httptest.NewRecorder()
-	c := config.Config{}
-	c.Headers = headers
+	c := config.Config{
+		Modules: map[string]config.Module{
+			"default": {
+				Headers: headers,
+			},
+		},
+	}
 
 	probeHandler(recorder, req, log.NewNopLogger(), c)
 
@@ -264,9 +311,15 @@ func TestBodyPostTemplate(t *testing.T) {
 			w.WriteHeader(http.StatusOK)
 		}))
 
-		req := httptest.NewRequest("POST", "http://example.com/foo"+"?target="+target.URL, strings.NewReader(test.Body.Content))
+		req := httptest.NewRequest("POST", "http://example.com/foo"+"?module=default&target="+target.URL, strings.NewReader(test.Body.Content))
 		recorder := httptest.NewRecorder()
-		c := config.Config{Body: test.Body}
+		c := config.Config{
+			Modules: map[string]config.Module{
+				"default": {
+					Body: test.Body,
+				},
+			},
+		}
 
 		probeHandler(recorder, req, log.NewNopLogger(), c)
 
@@ -351,15 +404,21 @@ func TestBodyPostQuery(t *testing.T) {
 			w.WriteHeader(http.StatusOK)
 		}))
 
-		req := httptest.NewRequest("POST", "http://example.com/foo"+"?target="+target.URL, strings.NewReader(test.Body.Content))
+		req := httptest.NewRequest("POST", "http://example.com/foo"+"?module=default&target="+target.URL, strings.NewReader(test.Body.Content))
 		q := req.URL.Query()
 		for k, v := range test.QueryParams {
 			q.Add(k, v)
 		}
 		req.URL.RawQuery = q.Encode()
 
 		recorder := httptest.NewRecorder()
-		c := config.Config{Body: test.Body}
+		c := config.Config{
+			Modules: map[string]config.Module{
+				"default": {
+					Body: test.Body,
+				},
+			},
+		}
 
 		probeHandler(recorder, req, log.NewNopLogger(), c)
 

config/config.go

@@ -46,8 +46,13 @@ const (
 	ValueTypeUntyped ValueType = "untyped"
 )
 
-// Config contains metrics and headers defining a configuration
+// Config contains multiple modules.
 type Config struct {
+	Modules map[string]Module `yaml:"modules"`
+}
+
+// Module contains metrics and headers defining a configuration
+type Module struct {
 	Headers          map[string]string        `yaml:"headers,omitempty"`
 	Metrics          []Metric                 `yaml:"metrics"`
 	HTTPClientConfig pconfig.HTTPClientConfig `yaml:"http_client_config,omitempty"`
@@ -71,15 +76,17 @@ func LoadConfig(configPath string) (Config, error) {
 	}
 
 	// Complete Defaults
-	for i := 0; i < len(config.Metrics); i++ {
-		if config.Metrics[i].Type == "" {
-			config.Metrics[i].Type = ValueScrape
-		}
-		if config.Metrics[i].Help == "" {
-			config.Metrics[i].Help = config.Metrics[i].Name
-		}
-		if config.Metrics[i].ValueType == "" {
-			config.Metrics[i].ValueType = ValueTypeUntyped
+	for _, module := range config.Modules {
+		for i := 0; i < len(module.Metrics); i++ {
+			if module.Metrics[i].Type == "" {
+				module.Metrics[i].Type = ValueScrape
+			}
+			if module.Metrics[i].Help == "" {
+				module.Metrics[i].Help = module.Metrics[i].Name
+			}
+			if module.Metrics[i].ValueType == "" {
+				module.Metrics[i].ValueType = ValueTypeUntyped
+			}
 		}
 	}