Adding a sync() function to filetarget (#293)

* Adding a sync() function to filetarget which runs on a timer to catch any events missed by fsnotify

* moving where we mark the final position to avoid a race condition exposed in testing, we want to stop the tailer before shutting down our tailer loop

* Fix changes from PR and refactor to remove circular imports and make config consistent

* Adding an "integration" type unit test which tests promtail as a whole

* fixing a bunch of imports I munged
moved tailer to its own file
refactoring filetarget to make synclogic easier to follow and reused as much as possible

.gitignore

@@ -7,3 +7,4 @@ cmd/loki/loki
 cmd/promtail/promtail
 /loki
 /promtail
+.idea/

cmd/promtail/main.go

@@ -4,16 +4,15 @@ import (
 	"flag"
 	"os"
 
+	"github.com/cortexproject/cortex/pkg/util"
+	"github.com/cortexproject/cortex/pkg/util/flagext"
 	"github.com/go-kit/kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/version"
 
-	"github.com/cortexproject/cortex/pkg/util"
-	"github.com/cortexproject/cortex/pkg/util/flagext"
-
 	"github.com/grafana/loki/pkg/helpers"
 	"github.com/grafana/loki/pkg/promtail"
-	"github.com/grafana/loki/pkg/promtail/api"
+	"github.com/grafana/loki/pkg/promtail/config"
 )
 
 func init() {
@@ -23,7 +22,7 @@ func init() {
 func main() {
 	var (
 		configFile = "docs/promtail-local-config.yaml"
-		config     api.Config
+		config     config.Config
 	)
 	flag.StringVar(&configFile, "config.file", "promtail.yml", "The config file.")
 	flagext.RegisterFlags(&config)

pkg/promtail/config/config.go

@@ -0,0 +1,47 @@
+package config
+
+import (
+	"flag"
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/weaveworks/common/server"
+	"gopkg.in/yaml.v2"
+
+	"github.com/grafana/loki/pkg/promtail/client"
+	"github.com/grafana/loki/pkg/promtail/positions"
+	"github.com/grafana/loki/pkg/promtail/scrape"
+	"github.com/grafana/loki/pkg/promtail/targets"
+)
+
+// Config for promtail, describing what files to watch.
+type Config struct {
+	ServerConfig    server.Config    `yaml:"server,omitempty"`
+	ClientConfig    client.Config    `yaml:"client,omitempty"`
+	PositionsConfig positions.Config `yaml:"positions,omitempty"`
+	ScrapeConfig    []scrape.Config  `yaml:"scrape_configs,omitempty"`
+	TargetConfig    targets.Config   `yaml:"target_config,omitempty"`
+}
+
+// RegisterFlags registers flags.
+func (c *Config) RegisterFlags(f *flag.FlagSet) {
+	c.ServerConfig.RegisterFlags(f)
+	c.ClientConfig.RegisterFlags(f)
+	c.PositionsConfig.RegisterFlags(f)
+	c.TargetConfig.RegisterFlags(f)
+}
+
+// LoadConfig loads config from a file.
+func LoadConfig(filename string) (*Config, error) {
+	buf, err := ioutil.ReadFile(filepath.Clean(filename))
+	if err != nil {
+		return nil, err
+	}
+
+	var cfg Config
+	if err := yaml.UnmarshalStrict(buf, &cfg); err != nil {
+		return nil, err
+	}
+
+	return &cfg, nil
+}

pkg/promtail/promtail.go

@@ -4,8 +4,8 @@ import (
 	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/weaveworks/common/server"
 
-	"github.com/grafana/loki/pkg/promtail/api"
 	"github.com/grafana/loki/pkg/promtail/client"
+	"github.com/grafana/loki/pkg/promtail/config"
 	"github.com/grafana/loki/pkg/promtail/positions"
 	"github.com/grafana/loki/pkg/promtail/targets"
 )
@@ -19,7 +19,7 @@ type Promtail struct {
 }
 
 // New makes a new Promtail.
-func New(cfg api.Config) (*Promtail, error) {
+func New(cfg config.Config) (*Promtail, error) {
 	positions, err := positions.New(util.Logger, cfg.PositionsConfig)
 	if err != nil {
 		return nil, err
@@ -30,7 +30,7 @@ func New(cfg api.Config) (*Promtail, error) {
 		return nil, err
 	}
 
-	tms, err := targets.NewTargetManagers(util.Logger, positions, client, cfg.ScrapeConfig)
+	tms, err := targets.NewTargetManagers(util.Logger, positions, client, cfg.ScrapeConfig, &cfg.TargetConfig)
 	if err != nil {
 		return nil, err
 	}