PMB-77 Improved loggers

cli/mongodb-backup-admin/main.go

@@ -312,7 +312,7 @@ func processCliArgs(args []string) (string, *cliOptions, error) {
 		ServerAddr: defaultServerAddr,
 	}
 	if *opts.configFile != "" {
-		err := loadOptionsFromFile(defaultConfigFile, yamlOpts)
+		loadOptionsFromFile(defaultConfigFile, yamlOpts)
 	}
 
 	cmd, err := app.Parse(args)

cli/mongodb-backup-agent/main.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"math/rand"
 	"os"
 	"os/signal"
@@ -16,8 +15,8 @@ import (
 
 	"github.com/alecthomas/kingpin"
 	"github.com/globalsign/mgo"
-	"github.com/kr/pretty"
 	"github.com/percona/mongodb-backup/grpc/client"
+	"github.com/percona/mongodb-backup/internal/logger"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
@@ -36,10 +35,12 @@ type cliOptions struct {
 	DSN           string `yaml:"dsn"`
 	Debug         bool   `yaml:"debug"`
 	KeyFile       string `yaml:"key_file"`
+	LogFile       string `yaml:"log_file"`
 	PIDFile       string `yaml:"pid_file"`
 	Quiet         bool   `yaml:"quiet"`
 	ServerAddress string `yaml:"server_address"`
 	TLS           bool   `yaml:"tls"`
+	UseSysLog     bool   `yaml:"use_syslog"`
 
 	// MongoDB connection options
 	MongodbConnOptions client.ConnectionOptions `yaml:"mongodb_conn_options"`
@@ -52,14 +53,25 @@ const (
 	sampleConfigFile = "config.sample.yml"
 )
 
+var (
+	log = logrus.New()
+)
+
 func main() {
-	log := logrus.New()
 	opts, err := processCliArgs()
 	if err != nil {
 		log.Fatalf("Cannot parse command line arguments: %s", err)
 	}
 
-	pretty.Println(opts)
+	if opts.UseSysLog {
+		log = logger.NewSyslogLogger()
+	} else {
+		log = logger.NewDefaultLogger(opts.LogFile)
+	}
+
+	if opts.Debug {
+		log.SetLevel(logrus.DebugLevel)
+	}
 
 	if opts.generateSampleConfig {
 		if err := writeSampleConfig(sampleConfigFile, opts); err != nil {
@@ -69,7 +81,6 @@ func main() {
 		return
 	}
 
-	log.SetLevel(logrus.InfoLevel)
 	if opts.Quiet {
 		log.SetLevel(logrus.ErrorLevel)
 	}
@@ -146,6 +157,8 @@ func processCliArgs() (*cliOptions, error) {
 	app.Flag("key-file", "Key file").StringVar(&opts.KeyFile)
 	app.Flag("debug", "Enable debug log level").BoolVar(&opts.Debug)
 	app.Flag("quiet", "Quiet mode. Log only errors").BoolVar(&opts.Quiet)
+	app.Flag("log-file", "Log file").StringVar(&opts.LogFile)
+	app.Flag("use-syslog", "Use syslog instead of Stderr or file").BoolVar(&opts.UseSysLog)
 
 	app.Flag("mongodb-host", "MongoDB host").StringVar(&opts.MongodbConnOptions.Host)
 	app.Flag("mongodb-port", "MongoDB port").StringVar(&opts.MongodbConnOptions.Port)
@@ -160,23 +173,15 @@ func processCliArgs() (*cliOptions, error) {
 
 	//TODO: Remove defaults. These values are only here to test during development
 	// These params should be Required()
-	yamlOpts := &cliOptions{
-		MongodbConnOptions: client.ConnectionOptions{
-			Host:           "127.0.0.1",
-			Port:           os.Getenv("TEST_MONGODB_S1_PRIMARY_PORT"),
-			User:           os.Getenv("TEST_MONGODB_USERNAME"),
-			Password:       os.Getenv("TEST_MONGODB_PASSWORD"),
-			ReplicasetName: os.Getenv("TEST_MONGODB_S1_RS"),
-		},
-	}
+	yamlOpts := &cliOptions{}
 
 	if opts.configFile != "" {
 		loadOptionsFromFile(opts.configFile, yamlOpts)
 	}
 
 	mergeOptions(opts, yamlOpts)
 	validateOptions(yamlOpts)
-	return yamlOpts, nil
+	return opts, nil
 }
 
 func loadOptionsFromFile(filename string, opts *cliOptions) error {
@@ -251,21 +256,6 @@ func mergeOptions(opts, yamlOpts *cliOptions) {
 	if opts.DSN != "" {
 		yamlOpts.DSN = opts.DSN
 	}
-	app.Flag("server-address", "MongoDB backup server address").Default("127.0.0.1:10000").StringVar(&opts.ServerAddress)
-	app.Flag("backup-dir", "Directory where to store the backups").Default("/tmp").StringVar(&opts.BackupDir)
-	app.Flag("tls", "Use TLS").BoolVar(&opts.TLS)
-	app.Flag("pid-file", "pid file").StringVar(&opts.PIDFile)
-	app.Flag("ca-file", "CA file").StringVar(&opts.CAFile)
-	app.Flag("cert-file", "Cert file").StringVar(&opts.CertFile)
-	app.Flag("key-file", "Key file").StringVar(&opts.KeyFile)
-	app.Flag("debug", "Enable debug log level").BoolVar(&opts.Debug)
-	app.Flag("quiet", "Quiet mode. Log only errors").BoolVar(&opts.Quiet)
-
-	app.Flag("mongodb-host", "MongoDB host").StringVar(&opts.MongodbConnOptions.Host)
-	app.Flag("mongodb-port", "MongoDB port").StringVar(&opts.MongodbConnOptions.Port)
-	app.Flag("mongodb-user", "MongoDB username").StringVar(&opts.MongodbConnOptions.User)
-	app.Flag("mongodb-password", "MongoDB password").StringVar(&opts.MongodbConnOptions.Password)
-	app.Flag("replicaset", "Replicaset name").StringVar(&opts.MongodbConnOptions.ReplicasetName)
 }
 
 func expandDirs(opts *cliOptions) {
@@ -333,3 +323,19 @@ func writePidFile(pidFile string) error {
 	// or the pid in it doesn't belong to the user running this app.
 	return ioutil.WriteFile(pidFile, []byte(fmt.Sprintf("%d", os.Getpid())), 0664)
 }
+
+func getDefaultLogger() *logrus.Logger {
+	logger := &logrus.Logger{
+		Out: os.Stderr,
+		Formatter: &logrus.TextFormatter{
+			FullTimestamp:          true,
+			DisableLevelTruncation: true,
+		},
+		Hooks: make(logrus.LevelHooks),
+		Level: logrus.InfoLevel,
+	}
+	logger.SetLevel(logrus.StandardLogger().Level)
+	logger.Out = logrus.StandardLogger().Out
+
+	return logger
+}

cli/mongodb-backupd/main.go

@@ -15,6 +15,7 @@ import (
 	"github.com/alecthomas/kingpin"
 	"github.com/percona/mongodb-backup/grpc/api"
 	"github.com/percona/mongodb-backup/grpc/server"
+	"github.com/percona/mongodb-backup/internal/logger"
 	apipb "github.com/percona/mongodb-backup/proto/api"
 	pb "github.com/percona/mongodb-backup/proto/messages"
 	"github.com/pkg/errors"
@@ -35,16 +36,14 @@ type cliOptions struct {
 	CertFile             string `yaml:"cert_file"`
 	KeyFile              string `yaml:"key_file"`
 	GrpcPort             int    `yaml:"grpc_port"`
-	ApiPort              int    `yaml:"api_port"`
+	APIPort              int    `yaml:"api_port"`
 	ShutdownTimeout      int    `yaml:"shutdown_timeout"`
+	LogFile              string `yaml:"log_file"`
+	UseSysLog            bool   `yaml:"sys_log_url"`
 	Debug                bool   `yaml:"debug"`
 	EnableClientsLogging bool   `yaml:"enable_clients_logging"`
 }
 
-var (
-	log = logrus.New()
-)
-
 const (
 	defaultGrpcPort        = 10000
 	defaultAPIPort         = 10001
@@ -53,19 +52,30 @@ const (
 	defaultDebugMode       = true
 )
 
+var (
+	log = logrus.New()
+)
+
 func main() {
 	opts, err := processCliParams()
 	if err != nil {
 		log.Fatalf("Cannot parse command line arguments: %s", err)
 	}
+
+	if opts.UseSysLog {
+		log = logger.NewSyslogLogger()
+	} else {
+		log = logger.NewDefaultLogger(opts.LogFile)
+	}
+
 	if opts.Debug {
 		log.SetLevel(logrus.DebugLevel)
 	}
 
 	log.Infof("Starting clients gRPC net listener on port: %d", opts.GrpcPort)
 	lis, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", opts.GrpcPort))
 
-	apilis, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", opts.ApiPort))
+	apilis, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", opts.APIPort))
 
 	if err != nil {
 		log.Fatalf("failed to listen: %v", err)
@@ -162,8 +172,10 @@ func processCliParams() (*cliOptions, error) {
 	app.Flag("cert-file", "Cert file for gRPC client connections").StringVar(&opts.CertFile)
 	app.Flag("key-file", "Key file for gRPC client connections").StringVar(&opts.KeyFile)
 	app.Flag("grpc-port", "Listening port for client connections").IntVar(&opts.GrpcPort)
-	app.Flag("api-port", "Listening por for API client connecions").IntVar(&opts.ApiPort)
+	app.Flag("api-port", "Listening por for API client connecions").IntVar(&opts.APIPort)
 	app.Flag("shutdown-timeout", "Server shutdown timeout").IntVar(&opts.ShutdownTimeout)
+	app.Flag("log-file", "Write logs to file").StringVar(&opts.LogFile)
+	app.Flag("use-syslog", "Also send the logs to the local syslog server").BoolVar(&opts.UseSysLog)
 	app.Flag("debug", "Enable debug log level").BoolVar(&opts.Debug)
 	app.Flag("enable-clients-logging", "Enable showing logs comming from agents on the server side").BoolVar(&opts.EnableClientsLogging)
 
@@ -174,13 +186,13 @@ func processCliParams() (*cliOptions, error) {
 
 	yamlOpts := &cliOptions{
 		GrpcPort:             defaultGrpcPort,
-		ApiPort:              defaultAPIPort,
+		APIPort:              defaultAPIPort,
 		ShutdownTimeout:      defaultShutdownTimeout,
 		Debug:                defaultDebugMode,
 		EnableClientsLogging: defaultClientsLogging,
 	}
 	if opts.configFile != "" {
-		err = loadOptionsFromFile(expandHomeDir(opts.configFile), yamlOpts)
+		loadOptionsFromFile(expandHomeDir(opts.configFile), yamlOpts)
 	}
 	mergeOptions(opts, yamlOpts)
 	expandDirs(yamlOpts)
@@ -216,15 +228,18 @@ func mergeOptions(opts, yamlOpts *cliOptions) {
 	if opts.GrpcPort != 0 {
 		yamlOpts.GrpcPort = opts.GrpcPort
 	}
-	if opts.ApiPort != 0 {
-		yamlOpts.ApiPort = opts.ApiPort
+	if opts.APIPort != 0 {
+		yamlOpts.APIPort = opts.APIPort
 	}
 	if opts.ShutdownTimeout != 0 {
 		yamlOpts.ShutdownTimeout = opts.ShutdownTimeout
 	}
 	if opts.EnableClientsLogging != false {
 		yamlOpts.EnableClientsLogging = opts.EnableClientsLogging
 	}
+	if opts.UseSysLog == true {
+		yamlOpts.UseSysLog = true
+	}
 	if opts.Debug != false {
 		yamlOpts.Debug = opts.Debug
 	}

internal/logger/logger.go

@@ -0,0 +1,57 @@
+package logger
+
+import (
+	"fmt"
+	"io"
+	"log/syslog"
+	"os"
+
+	"github.com/sirupsen/logrus"
+	lSyslog "github.com/sirupsen/logrus/hooks/syslog"
+)
+
+func NewDefaultLogger(filename string) *logrus.Logger {
+	if filename == "" {
+		return newDefaultLogger(os.Stderr)
+	}
+
+	fh, err := os.Create(filename)
+	if err != nil {
+		fmt.Printf("Cannot create lof file %s: %s", filename, err)
+		fmt.Println("Using Stderr for error output")
+		return newDefaultLogger(os.Stderr)
+	}
+
+	return newDefaultLogger(fh)
+}
+
+func NewSyslogLogger() *logrus.Logger {
+	w, err := os.Create(os.DevNull)
+	if err != nil {
+		w = os.Stderr
+	}
+
+	log := newDefaultLogger(w)
+	if hook, err := lSyslog.NewSyslogHook("", "", syslog.LOG_INFO, ""); err != nil {
+		fmt.Printf("Cannot hook into local syslog: %s", err)
+	} else {
+		log.AddHook(hook)
+	}
+	return log
+}
+
+func newDefaultLogger(w io.Writer) *logrus.Logger {
+	logger := &logrus.Logger{
+		Out: w,
+		Formatter: &logrus.TextFormatter{
+			FullTimestamp:          true,
+			DisableLevelTruncation: true,
+		},
+		Hooks: make(logrus.LevelHooks),
+		Level: logrus.InfoLevel,
+	}
+	logger.SetLevel(logrus.StandardLogger().Level)
+	logger.Out = logrus.StandardLogger().Out
+
+	return logger
+}