Detect changes to config and reload telegraf (copy of pr #8529) (#9485)

(cherry picked from commit 2a72295)

cmd/telegraf/telegraf.go

@@ -15,6 +15,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/influxdata/tail/watch"
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/agent"
 	"github.com/influxdata/telegraf/config"
@@ -27,6 +28,7 @@ import (
 	"github.com/influxdata/telegraf/plugins/outputs"
 	_ "github.com/influxdata/telegraf/plugins/outputs/all"
 	_ "github.com/influxdata/telegraf/plugins/processors/all"
+	"gopkg.in/tomb.v1"
 )
 
 type sliceFlags []string
@@ -53,7 +55,7 @@ var fTestWait = flag.Int("test-wait", 0, "wait up to this many seconds for servi
 
 var fConfigs sliceFlags
 var fConfigDirs sliceFlags
-
+var fWatchConfig = flag.String("watch-config", "", "Monitoring config changes [notify, poll]")
 var fVersion = flag.Bool("version", false, "display the version and exit")
 var fSampleConfig = flag.Bool("sample-config", false,
 	"print out full sample configuration")
@@ -115,6 +117,15 @@ func reloadLoop(
 		signals := make(chan os.Signal, 1)
 		signal.Notify(signals, os.Interrupt, syscall.SIGHUP,
 			syscall.SIGTERM, syscall.SIGINT)
+		if *fWatchConfig != "" {
+			for _, fConfig := range fConfigs {
+				if _, err := os.Stat(fConfig); err == nil {
+					go watchLocalConfig(signals, fConfig)
+				} else {
+					log.Printf("W! Cannot watch config %s: %s", fConfig, err)
+				}
+			}
+		}
 		go func() {
 			select {
 			case sig := <-signals:
@@ -136,6 +147,46 @@ func reloadLoop(
 	}
 }
 
+func watchLocalConfig(signals chan os.Signal, fConfig string) {
+	var mytomb tomb.Tomb
+	var watcher watch.FileWatcher
+	if *fWatchConfig == "poll" {
+		watcher = watch.NewPollingFileWatcher(fConfig)
+	} else {
+		watcher = watch.NewInotifyFileWatcher(fConfig)
+	}
+	changes, err := watcher.ChangeEvents(&mytomb, 0)
+	if err != nil {
+		log.Printf("E! Error watching config: %s\n", err)
+		return
+	}
+	log.Println("I! Config watcher started")
+	select {
+	case <-changes.Modified:
+		log.Println("I! Config file modified")
+	case <-changes.Deleted:
+		// deleted can mean moved. wait a bit a check existence
+		<-time.After(time.Second)
+		if _, err := os.Stat(fConfig); err == nil {
+			log.Println("I! Config file overwritten")
+		} else {
+			log.Println("W! Config file deleted")
+			if err := watcher.BlockUntilExists(&mytomb); err != nil {
+				log.Printf("E! Cannot watch for config: %s\n", err.Error())
+				return
+			}
+			log.Println("I! Config file appeared")
+		}
+	case <-changes.Truncated:
+		log.Println("I! Config file truncated")
+	case <-mytomb.Dying():
+		log.Println("I! Config watcher ended")
+		return
+	}
+	mytomb.Done()
+	signals <- syscall.SIGHUP
+}
+
 func runAgent(ctx context.Context,
 	inputFilters []string,
 	outputFilters []string,

go.mod

@@ -77,7 +77,7 @@ require (
 	github.com/influxdata/influxdb-observability/common v0.0.0-20210429174543-86ae73cafd31
 	github.com/influxdata/influxdb-observability/otel2influx v0.0.0-20210429174543-86ae73cafd31
 	github.com/influxdata/influxdb-observability/otlp v0.0.0-20210429174543-86ae73cafd31
-	github.com/influxdata/tail v1.0.1-0.20200707181643-03a791b270e4
+	github.com/influxdata/tail v1.0.1-0.20210707231403-b283181d1fa7
 	github.com/influxdata/toml v0.0.0-20190415235208-270119a8ce65
 	github.com/influxdata/wlog v0.0.0-20160411224016-7c63b0a71ef8
 	github.com/jackc/pgx/v4 v4.6.0
@@ -147,6 +147,7 @@ require (
 	gopkg.in/ldap.v3 v3.1.0
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22
 	gopkg.in/olivere/elastic.v5 v5.0.70
+	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7
 	gopkg.in/yaml.v2 v2.4.0
 	gotest.tools v2.2.0+incompatible
 	k8s.io/api v0.20.4

go.sum

@@ -864,8 +864,8 @@ github.com/influxdata/influxql v1.1.0/go.mod h1:KpVI7okXjK6PRi3Z5B+mtKZli+R1DnZg
 github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e/go.mod h1:4kt73NQhadE3daL3WhR5EJ/J2ocX0PZzwxQ0gXJ7oFE=
 github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19ybifQhZoQNF5D8=
 github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE=
-github.com/influxdata/tail v1.0.1-0.20200707181643-03a791b270e4 h1:K3A5vHPs/p8OjI4SL3l1+hs/98mhxTVDcV1Ap0c265E=
-github.com/influxdata/tail v1.0.1-0.20200707181643-03a791b270e4/go.mod h1:VeiWgI3qaGdJWust2fP27a6J+koITo/1c/UhxeOxgaM=
+github.com/influxdata/tail v1.0.1-0.20210707231403-b283181d1fa7 h1:0rQOs1VHLVFpAAOIR0mJEvVOIaMYFgYdreeVbgI9sII=
+github.com/influxdata/tail v1.0.1-0.20210707231403-b283181d1fa7/go.mod h1:VeiWgI3qaGdJWust2fP27a6J+koITo/1c/UhxeOxgaM=
 github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0=
 github.com/influxdata/toml v0.0.0-20190415235208-270119a8ce65 h1:vvyMtD5LTJc1W9sQKjDkAWdcg0478CszSdzlHtiAXCY=
 github.com/influxdata/toml v0.0.0-20190415235208-270119a8ce65/go.mod h1:zApaNFpP/bTpQItGZNNUMISDMDAnTXu9UqJ4yT3ocz8=

internal/usage.go

@@ -16,6 +16,9 @@ The commands & flags are:
   --aggregator-filter <filter>   filter the aggregators to enable, separator is :
   --config <file>                configuration file to load
   --config-directory <directory> directory containing additional *.conf files
+  --watch-config                 Telegraf will restart on local config changes. Monitor changes 
+                                 using either fs notifications or polling.  Valid values: 'inotify' or 'poll'. 
+                                 Monitoring is off by default.
   --plugin-directory             directory containing *.so files, this directory will be
                                  searched recursively. Any Plugin found will be loaded
                                  and namespaced.

internal/usage_windows.go

@@ -16,6 +16,9 @@ The commands & flags are:
   --aggregator-filter <filter>   filter the aggregators to enable, separator is :
   --config <file>                configuration file to load
   --config-directory <directory> directory containing additional *.conf files
+  --watch-config                 Telegraf will restart on local config changes. Monitor changes 
+                                 using either fs notifications or polling.  Valid values: 'inotify' or 'poll'. 
+                                 Monitoring is off by default.
   --debug                        turn on debug logging
   --input-filter <filter>        filter the inputs to enable, separator is :
   --input-list                   print available input plugins.