feat: provide configuration subset via env variables

cmd/sql_exporter/main.go

@@ -20,9 +20,8 @@ import (
 )
 
 const (
-	appName               string        = "sql_exporter"
-	envConfigFile         string        = "SQLEXPORTER_CONFIG"
-	envDebug              string        = "SQLEXPORTER_DEBUG"
+	appName string = "sql_exporter"
+
 	httpReadHeaderTimeout time.Duration = time.Duration(time.Second * 60)
 	debugMaxLevel         klog.Level    = 3
 )
@@ -47,10 +46,11 @@ func init() {
 }
 
 func main() {
-	if os.Getenv(envDebug) != "" {
+	if os.Getenv(cfg.EnvDebug) != "" {
 		runtime.SetBlockProfileRate(1)
 		runtime.SetMutexProfileFraction(1)
 	}
+
 	flag.Parse()
 
 	// Show version and exit.
@@ -79,7 +79,7 @@ func main() {
 	klog.ClampLevel(debugMaxLevel)
 
 	// Override the config.file default with the SQLEXPORTER_CONFIG environment variable if set.
-	if val, ok := os.LookupEnv(envConfigFile); ok {
+	if val, ok := os.LookupEnv(cfg.EnvConfigFile); ok {
 		*configFile = val
 	}
 

config/config.go

@@ -1,20 +1,28 @@
 package config
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
 
+	"github.com/sethvargo/go-envconfig"
 	"gopkg.in/yaml.v3"
 	"k8s.io/klog/v2"
 )
 
-const EnvDsnOverride = "SQLEXPORTER_TARGET_DSN"
-
 // MaxInt32 defines the maximum value of allowed integers
 // and serves to help us avoid overflow/wraparound issues.
 const MaxInt32 int = 1<<31 - 1
 
+// EnvPrefix is the prefix for environment variables.
+const (
+	EnvPrefix string = "SQLEXPORTER_"
+
+	EnvConfigFile string = EnvPrefix + "CONFIG"
+	EnvDebug      string = EnvPrefix + "DEBUG"
+)
+
 var (
 	EnablePing        bool
 	IgnoreMissingVals bool
@@ -49,9 +57,9 @@ func Load(configFile string) (*Config, error) {
 
 // Config is a collection of jobs and collectors.
 type Config struct {
-	Globals        *GlobalConfig      `yaml:"global,omitempty"`
-	CollectorFiles []string           `yaml:"collector_files,omitempty"`
-	Target         *TargetConfig      `yaml:"target,omitempty"`
+	Globals        *GlobalConfig      `yaml:"global,omitempty" env:", prefix=GLOBAL_"`
+	CollectorFiles []string           `yaml:"collector_files,omitempty" env:"COLLECTOR_FILES"`
+	Target         *TargetConfig      `yaml:"target,omitempty" env:", prefix=TARGET_"`
 	Jobs           []*JobConfig       `yaml:"jobs,omitempty"`
 	Collectors     []*CollectorConfig `yaml:"collectors,omitempty"`
 
@@ -63,32 +71,77 @@ type Config struct {
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface for Config.
 func (c *Config) UnmarshalYAML(unmarshal func(any) error) error {
-	type plain Config
-	if err := unmarshal((*plain)(c)); err != nil {
+	// unmarshalConfig does the actual unmarshalling
+	if err := c.unmarshalConfig(unmarshal); err != nil {
+		return err
+	}
+	// Populate global defaults.
+	if err := c.populateGlobalDefaults(); err != nil {
+		return err
+	}
+
+	// Load any externally defined collectors.
+	if err := c.loadCollectorFiles(); err != nil {
 		return err
 	}
 
+	// Process environment variables.
+	if err := c.processEnvConfig(); err != nil {
+		return err
+	}
+
+	// Check required fields
+	if err := c.checkRequiredFields(); err != nil {
+		return err
+	}
+
+	// Populate collector references for the target/jobs.
+	if err := c.populateCollectorReferences(); err != nil {
+		return err
+	}
+
+	return checkOverflow(c.XXX, "config")
+}
+
+// unmarshalConfig unmarshals the config, but does not populate global defaults, process environment variables, or check required fields.
+func (c *Config) unmarshalConfig(unmarshal func(any) error) error {
+	type plain Config
+	return unmarshal((*plain)(c))
+}
+
+// populateGlobalDefaults populates any unset global defaults.
+func (c *Config) populateGlobalDefaults() error {
 	if c.Globals == nil {
 		c.Globals = &GlobalConfig{}
 		// Force a dummy unmarshall to populate global defaults
-		if err := c.Globals.UnmarshalYAML(func(any) error { return nil }); err != nil {
-			return err
-		}
+		return c.Globals.UnmarshalYAML(func(any) error { return nil })
 	}
+	return nil
+}
+
+// processEnvConfig processes environment variables.
+func (c *Config) processEnvConfig() error {
+	return envconfig.ProcessWith(context.Background(), &envconfig.Config{
+		Target:           c,
+		Lookuper:         envconfig.PrefixLookuper(EnvPrefix, envconfig.OsLookuper()),
+		DefaultNoInit:    true,
+		DefaultOverwrite: true,
+		DefaultDelimiter: ";",
+	})
+}
 
+// checkRequiredFields checks that all required fields are present.
+func (c *Config) checkRequiredFields() error {
 	if (len(c.Jobs) == 0) == (c.Target == nil) {
 		return fmt.Errorf("exactly one of `jobs` and `target` must be defined")
 	}
+	return nil
+}
 
-	// Load any externally defined collectors.
-	if err := c.loadCollectorFiles(); err != nil {
-		return err
-	}
-
-	// Populate collector references for the target/jobs.
+// populateCollectorReferences populates collector references for the target/jobs.
+func (c *Config) populateCollectorReferences() error {
 	colls := make(map[string]*CollectorConfig)
 	for _, coll := range c.Collectors {
-		// Set the min interval to the global default if not explicitly set.
 		if coll.MinInterval < 0 {
 			coll.MinInterval = c.Globals.MinInterval
 		}
@@ -97,22 +150,23 @@ func (c *Config) UnmarshalYAML(unmarshal func(any) error) error {
 		}
 		colls[coll.Name] = coll
 	}
+
 	if c.Target != nil {
 		cs, err := resolveCollectorRefs(c.Target.CollectorRefs, colls, "target")
 		if err != nil {
 			return err
 		}
 		c.Target.collectors = cs
 	}
+
 	for _, j := range c.Jobs {
 		cs, err := resolveCollectorRefs(j.CollectorRefs, colls, fmt.Sprintf("job %q", j.Name))
 		if err != nil {
 			return err
 		}
 		j.collectors = cs
 	}
-
-	return checkOverflow(c.XXX, "config")
+	return nil
 }
 
 // YAML marshals the config into YAML format.

config/global_config.go

@@ -9,12 +9,12 @@ import (
 
 // GlobalConfig contains globally applicable defaults.
 type GlobalConfig struct {
-	MinInterval     model.Duration `yaml:"min_interval"`            // minimum interval between query executions, default is 0
-	ScrapeTimeout   model.Duration `yaml:"scrape_timeout"`          // per-scrape timeout, global
-	TimeoutOffset   model.Duration `yaml:"scrape_timeout_offset"`   // offset to subtract from timeout in seconds
-	MaxConnLifetime time.Duration  `yaml:"max_connection_lifetime"` // maximum amount of time a connection may be reused to any one target
-	MaxConns        int            `yaml:"max_connections"`         // maximum number of open connections to any one target
-	MaxIdleConns    int            `yaml:"max_idle_connections"`    // maximum number of idle connections to any one target
+	MinInterval     model.Duration `yaml:"min_interval" env:"MIN_INTERVAL"`                       // minimum interval between query executions, default is 0
+	ScrapeTimeout   model.Duration `yaml:"scrape_timeout" env:"SCRAPE_TIMEOUT"`                   // per-scrape timeout, global
+	TimeoutOffset   model.Duration `yaml:"scrape_timeout_offset" env:"SCRAPE_TIMEOUT_OFFSET"`     // offset to subtract from timeout in seconds
+	MaxConnLifetime time.Duration  `yaml:"max_connection_lifetime" env:"MAX_CONNECTION_LIFETIME"` // maximum amount of time a connection may be reused to any one target
+	MaxConns        int            `yaml:"max_connections" env:"MAX_CONNECTIONS"`                 // maximum number of open connections to any one target
+	MaxIdleConns    int            `yaml:"max_idle_connections" env:"MAX_IDLE_CONNECTIONS"`       // maximum number of idle connections to any one target
 
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]any `yaml:",inline" json:"-"`

config/target_config.go

@@ -17,11 +17,11 @@ import (
 
 // TargetConfig defines a DSN and a set of collectors to be executed on it.
 type TargetConfig struct {
-	Name          string   `yaml:"name,omitempty"`        // name of the target
-	DSN           Secret   `yaml:"data_source_name"`      // data source name to connect to
-	AwsSecretName string   `yaml:"aws_secret_name"`       // AWS secret name
-	CollectorRefs []string `yaml:"collectors"`            // names of collectors to execute on the target
-	EnablePing    *bool    `yaml:"enable_ping,omitempty"` // ping the target before executing the collectors
+	Name          string   `yaml:"name,omitempty" env:"NAME"`               // name of the target
+	DSN           Secret   `yaml:"data_source_name" env:"DSN"`              // data source name to connect to
+	AwsSecretName string   `yaml:"aws_secret_name" env:"AWS_SECRET_NAME"`   // AWS secret name
+	CollectorRefs []string `yaml:"collectors" env:"COLLECTORS"`             // names of collectors to execute on the target
+	EnablePing    *bool    `yaml:"enable_ping,omitempty" env:"ENABLE_PING"` // ping the target before executing the collectors
 
 	collectors []*CollectorConfig // resolved collector references
 

exporter.go

@@ -4,13 +4,13 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"os"
 	"strings"
 	"sync"
 
 	"github.com/burningalchemist/sql_exporter/config"
 	"github.com/prometheus/client_golang/prometheus"
 	dto "github.com/prometheus/client_model/go"
+
 	"google.golang.org/protobuf/proto"
 )
 
@@ -45,13 +45,10 @@ func NewExporter(configFile string) (Exporter, error) {
 		return nil, err
 	}
 
-	if val, ok := os.LookupEnv(config.EnvDsnOverride); ok {
-		config.DsnOverride = val
-	}
 	// Override the DSN if requested (and in single target mode).
 	if config.DsnOverride != "" {
 		if len(c.Jobs) > 0 {
-			return nil, fmt.Errorf("the config.data-source-name flag (value %q) only applies in single target mode", config.DsnOverride)
+			return nil, errors.New("the config.data-source-name flag only applies in single target mode")
 		}
 		c.Target.DSN = config.Secret(config.DsnOverride)
 	}

go.mod

@@ -1,6 +1,8 @@
 module github.com/burningalchemist/sql_exporter
 
-go 1.18
+go 1.21
+
+toolchain go1.21.5
 
 require (
 	github.com/ClickHouse/clickhouse-go v1.5.4
@@ -16,6 +18,7 @@ require (
 	github.com/prometheus/client_model v0.5.0
 	github.com/prometheus/common v0.45.0
 	github.com/prometheus/exporter-toolkit v0.11.0
+	github.com/sethvargo/go-envconfig v1.0.0
 	github.com/snowflakedb/gosnowflake v1.7.1
 	github.com/vertica/vertica-sql-go v1.3.3
 	github.com/xo/dburl v0.20.0

go.sum

@@ -287,6 +287,8 @@ github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
+github.com/sethvargo/go-envconfig v1.0.0 h1:1C66wzy4QrROf5ew4KdVw942CQDa55qmlYmw9FZxZdU=
+github.com/sethvargo/go-envconfig v1.0.0/go.mod h1:Lzc75ghUn5ucmcRGIdGQ33DKJrcjk4kihFYgSTBmjIc=
 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
 github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=