Add HCL parsing logic

martin-helmich · Oct 31, 2016 · e7ea105 · e7ea105
1 parent ab5f3c2
commit e7ea105
Show file tree

Hide file tree

Showing 4 changed files with 90 additions and 69 deletions.
diff --git a/config/loader.go b/config/loader.go
@@ -4,30 +4,36 @@ import (
 	"fmt"
 	"io/ioutil"
 )
+
 import "github.com/hashicorp/hcl"
 
-func LoadConfigFromFile(filename string) (*Config, error) {
+func LoadConfigFromFile(config *Config, filename string) error {
 	buf, err := ioutil.ReadFile(filename)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
-	config := Config{}
 	hclText := string(buf)
 
-	err = hcl.Decode(&config, hclText)
+	err = hcl.Decode(config, hclText)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	fmt.Println(config)
-	return &config, nil
+
+	return nil
 }
 
-func LoadConfigFromFlags(flags *StartupFlags) (*Config, error) {
-	config := Config{
-		Port: flags.ListenPort,
+func LoadConfigFromFlags(config *Config, flags *StartupFlags) error {
+	config.Port = flags.ListenPort
+	config.Namespaces = []NamespaceConfig{
+		NamespaceConfig{
+			Format:      flags.Format,
+			SourceFiles: flags.Filenames,
+			Name:        flags.Namespace,
+		},
 	}
 
-	return &config, nil
+	return nil
 }
diff --git a/config/structs.go b/config/structs.go
@@ -11,12 +11,12 @@ type StartupFlags struct {
 }
 
 type Config struct {
-	Port      int
-	Namespace []NamespaceConfig
+	Port       int
+	Namespaces []NamespaceConfig `hcl:"namespace"`
 }
 
 type NamespaceConfig struct {
-	Name       string
-	SourceFile []string
-	Format     string
+	Name        string   `hcl:",key"`
+	SourceFiles []string `hcl:"source_files"`
+	Format      string   `hcl:"format"`
 }
diff --git a/example-config.hcl b/example-config.hcl
@@ -1,6 +1,9 @@
 port = 4040
 
 namespace "nginx" {
-  sourceFile = "test.log"
+  source_files = [
+    "test.log",
+    "foo.log"
+  ]
   format = "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\""
 }
diff --git a/main.go b/main.go
@@ -38,29 +38,29 @@ type Metrics struct {
 }
 
 // Init initializes a metrics struct
-func (m *Metrics) Init(opts *config.StartupFlags) {
+func (m *Metrics) Init(cfg *config.NamespaceConfig) {
 	labels := []string{"method", "status"}
 
 	m.countTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
-		Namespace: opts.Namespace,
+		Namespace: cfg.Name,
 		Name:      "http_response_count_total",
 		Help:      "Amount of processed HTTP requests",
 	}, labels)
 
 	m.bytesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
-		Namespace: opts.Namespace,
+		Namespace: cfg.Name,
 		Name:      "http_response_size_bytes",
 		Help:      "Total amount of transferred bytes",
 	}, labels)
 
 	m.upstreamSeconds = prometheus.NewSummaryVec(prometheus.SummaryOpts{
-		Namespace: opts.Namespace,
+		Namespace: cfg.Name,
 		Name:      "http_upstream_time_seconds",
 		Help:      "Time needed by upstream servers to handle requests",
 	}, labels)
 
 	m.responseSeconds = prometheus.NewSummaryVec(prometheus.SummaryOpts{
-		Namespace: opts.Namespace,
+		Namespace: cfg.Name,
 		Name:      "http_response_time_seconds",
 		Help:      "Time needed by NGINX to handle requests",
 	}, labels)
@@ -73,72 +73,84 @@ func (m *Metrics) Init(opts *config.StartupFlags) {
 
 func main() {
 	var opts config.StartupFlags
+	var cfg = config.Config{
+		Port: 4040,
+	}
 
 	flag.IntVar(&opts.ListenPort, "listen-port", 4040, "HTTP port to listen on")
 	flag.StringVar(&opts.Format, "format", `$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for"`, "NGINX access log format")
 	flag.StringVar(&opts.Namespace, "namespace", "nginx", "namespace to use for metric names")
 	flag.StringVar(&opts.ConfigFile, "config-file", "", "Configuration file to read from")
 	flag.Parse()
 
+	opts.Filenames = flag.Args()
+
 	if opts.ConfigFile != "" {
-		fmt.Printf("loading configuration file %s", opts.ConfigFile)
-		config, err := config.LoadConfigFromFile(opts.ConfigFile)
-		if err != nil {
+		fmt.Printf("loading configuration file %s\n", opts.ConfigFile)
+		if err := config.LoadConfigFromFile(&cfg, opts.ConfigFile); err != nil {
 			panic(err)
 		}
+	} else if err := config.LoadConfigFromFlags(&cfg, &opts); err != nil {
+		panic(err)
 	}
 
-	opts.Filenames = flag.Args()
-	parser := gonx.NewParser(opts.Format)
-
-	metrics := Metrics{}
-	metrics.Init(&opts)
-
-	for _, f := range opts.Filenames {
-		t, err := tail.TailFile(f, tail.Config{
-			Follow: true,
-			ReOpen: true,
-			Poll:   true,
-		})
-		if err != nil {
-			panic(err)
-		}
+	fmt.Printf("using configuration %s\n", cfg)
 
-		go func() {
-			for line := range t.Lines {
-				entry, err := parser.ParseString(line.Text)
-				if err != nil {
-					fmt.Printf("error while parsing line '%s': %s", line.Text, err)
-					continue
-				}
+	for _, ns := range cfg.Namespaces {
+		fmt.Printf("starting listener for namespace %s\n", ns.Name)
+		go func(nsCfg *config.NamespaceConfig) {
+			parser := gonx.NewParser(nsCfg.Format)
 
-				method := "UNKNOWN"
-				status := "0"
+			metrics := Metrics{}
+			metrics.Init(nsCfg)
 
-				if request, err := entry.Field("request"); err == nil {
-					f := strings.Split(request, " ")
-					method = f[0]
-				}
-
-				if s, err := entry.Field("status"); err == nil {
-					status = s
-				}
-
-				metrics.countTotal.WithLabelValues(method, status).Inc()
-
-				if bytes, err := entry.FloatField("body_bytes_sent"); err == nil {
-					metrics.bytesTotal.WithLabelValues(method, status).Add(bytes)
-				}
-
-				if upstreamTime, err := entry.FloatField("upstream_response_time"); err == nil {
-					metrics.upstreamSeconds.WithLabelValues(method, status).Observe(upstreamTime)
+			for _, f := range nsCfg.SourceFiles {
+				t, err := tail.TailFile(f, tail.Config{
+					Follow: true,
+					ReOpen: true,
+					Poll:   true,
+				})
+				if err != nil {
+					panic(err)
 				}
 
-				if responseTime, err := entry.FloatField("request_time"); err == nil {
-					metrics.responseSeconds.WithLabelValues(method, status).Observe(responseTime)
-				}
+				go func() {
+					for line := range t.Lines {
+						entry, err := parser.ParseString(line.Text)
+						if err != nil {
+							fmt.Printf("error while parsing line '%s': %s", line.Text, err)
+							continue
+						}
+
+						method := "UNKNOWN"
+						status := "0"
+
+						if request, err := entry.Field("request"); err == nil {
+							f := strings.Split(request, " ")
+							method = f[0]
+						}
+
+						if s, err := entry.Field("status"); err == nil {
+							status = s
+						}
+
+						metrics.countTotal.WithLabelValues(method, status).Inc()
+
+						if bytes, err := entry.FloatField("body_bytes_sent"); err == nil {
+							metrics.bytesTotal.WithLabelValues(method, status).Add(bytes)
+						}
+
+						if upstreamTime, err := entry.FloatField("upstream_response_time"); err == nil {
+							metrics.upstreamSeconds.WithLabelValues(method, status).Observe(upstreamTime)
+						}
+
+						if responseTime, err := entry.FloatField("request_time"); err == nil {
+							metrics.responseSeconds.WithLabelValues(method, status).Observe(responseTime)
+						}
+					}
+				}()
 			}
-		}()
+		}(&ns)
 	}
 
 	listenAddr := fmt.Sprintf("%s:%d", "0.0.0.0", opts.ListenPort)