From 29f2ba014b59f9793ca828272f54921d05ba6f99 Mon Sep 17 00:00:00 2001 From: qybdyx Date: Fri, 26 Jan 2024 14:09:41 +0800 Subject: [PATCH] log: support maxBackups in config.toml --- log/async_file_writer.go | 104 +++++++++++++++++++++++++++++++++- log/async_file_writer_test.go | 26 ++++++++- log/handler.go | 4 +- log/logger.go | 4 +- node/config.go | 1 + node/node.go | 7 ++- 6 files changed, 138 insertions(+), 8 deletions(-) diff --git a/log/async_file_writer.go b/log/async_file_writer.go index b4e7a60f30..cf00f7dd7f 100644 --- a/log/async_file_writer.go +++ b/log/async_file_writer.go @@ -3,13 +3,18 @@ package log import ( "errors" "fmt" + "io/fs" "os" "path/filepath" + "sort" + "strings" "sync" "sync/atomic" "time" ) +const backupTimeFormat = "2006-01-02_15" + type TimeTicker struct { stop chan struct{} C <-chan time.Time @@ -69,9 +74,11 @@ type AsyncFileWriter struct { buf chan []byte stop chan struct{} timeTicker *TimeTicker + + maxBackups int } -func NewAsyncFileWriter(filePath string, maxBytesSize int64, rotateHours uint) *AsyncFileWriter { +func NewAsyncFileWriter(filePath string, maxBytesSize int64, maxBackups int, rotateHours uint) *AsyncFileWriter { absFilePath, err := filepath.Abs(filePath) if err != nil { panic(fmt.Sprintf("get file path of logger error. filePath=%s, err=%s", filePath, err)) @@ -81,6 +88,7 @@ func NewAsyncFileWriter(filePath string, maxBytesSize int64, rotateHours uint) * filePath: absFilePath, buf: make(chan []byte, maxBytesSize), stop: make(chan struct{}), + maxBackups: maxBackups, timeTicker: NewTimeTicker(rotateHours), } } @@ -111,6 +119,8 @@ func (w *AsyncFileWriter) initLogFile() error { return err } + _ = w.clearBackups() + return nil } @@ -222,5 +232,95 @@ func (w *AsyncFileWriter) flushAndClose() error { } func (w *AsyncFileWriter) timeFilePath(filePath string) string { - return filePath + "." + time.Now().Format("2006-01-02_15") + return filePath + "." + time.Now().Format(backupTimeFormat) +} + +func (w *AsyncFileWriter) dir() string { + return filepath.Dir(w.filePath) +} + +// oldLogFiles returns the list of backup log files stored in the same +// directory as the current log file, sorted by ModTime +func (w *AsyncFileWriter) oldLogFiles() ([]logInfo, error) { + files, err := os.ReadDir(w.dir()) + if err != nil { + return nil, fmt.Errorf("can't read log file directory: %s", err) + } + logFiles := []logInfo{} + + prefix := filepath.Base(w.filePath) + + for _, f := range files { + if f.IsDir() { + continue + } + k := f.Name() + if t, err := w.timeFromName(k, prefix); err == nil { + logFiles = append(logFiles, logInfo{t, f}) + } + } + + sort.Sort(byFormatTime(logFiles)) + + return logFiles, nil +} + +// logInfo is a convenience struct to return the filename and its embedded +// timestamp. +type logInfo struct { + timestamp time.Time + fs.DirEntry +} + +// byFormatTime sorts by newest time formatted in the name. +type byFormatTime []logInfo + +func (b byFormatTime) Less(i, j int) bool { + return b[i].timestamp.After(b[j].timestamp) +} + +func (b byFormatTime) Swap(i, j int) { + b[i], b[j] = b[j], b[i] +} + +func (b byFormatTime) Len() int { + return len(b) +} + +func (w *AsyncFileWriter) timeFromName(filename, prefix string) (time.Time, error) { + if !strings.HasPrefix(filename, prefix) || len(filename) == len(prefix) { + return time.Time{}, errors.New("mismatched prefix") + } + ts := filename[len(prefix)+1:] + a, _ := time.Parse(backupTimeFormat, ts) + return a, nil +} + +func (w *AsyncFileWriter) clearBackups() error { + if w.maxBackups == 0 { + return nil + } + files, err := w.oldLogFiles() + if err != nil { + return err + } + var remove []logInfo + if w.maxBackups > 0 && w.maxBackups < len(files) { + preserved := make(map[string]bool) + for _, f := range files { + fn := f.Name() + preserved[fn] = true + + if len(preserved) > w.maxBackups { + remove = append(remove, f) + } + } + } + for _, f := range remove { + errRemove := os.Remove(filepath.Join(w.dir(), f.Name())) + if err == nil && errRemove != nil { + err = errRemove + } + } + return err } diff --git a/log/async_file_writer_test.go b/log/async_file_writer_test.go index ab12808856..078e3c5df3 100644 --- a/log/async_file_writer_test.go +++ b/log/async_file_writer_test.go @@ -2,14 +2,17 @@ package log import ( "os" + "path/filepath" "strconv" "strings" "testing" "time" + + "github.com/stretchr/testify/assert" ) func TestWriterHourly(t *testing.T) { - w := NewAsyncFileWriter("./hello.log", 100, 1) + w := NewAsyncFileWriter("./hello.log", 100, 1, 1) w.Start() w.Write([]byte("hello\n")) w.Write([]byte("world\n")) @@ -67,3 +70,24 @@ func TestGetNextRotationHour(t *testing.T) { t.Run("TestGetNextRotationHour_"+strconv.Itoa(i), test(tc.now, tc.delta, tc.expectedHour)) } } + +func TestClearBackups(t *testing.T) { + dir := "./test" + os.Mkdir(dir, 0700) + w := NewAsyncFileWriter("./test/bsc.log", 100, 1, 1) + defer os.RemoveAll(dir) + fakeCurrentTime := time.Now() + name := "" + data := []byte("data") + for i := 0; i < 5; i++ { + name = w.filePath + "." + fakeCurrentTime.Format(backupTimeFormat) + _ = os.WriteFile(name, data, 0700) + fakeCurrentTime = fakeCurrentTime.Add(time.Hour * 1) + } + oldFiles, _ := w.oldLogFiles() + assert.True(t, len(oldFiles) == 5) + w.clearBackups() + remainFiles, _ := w.oldLogFiles() + assert.True(t, len(remainFiles) == 1) + assert.Equal(t, remainFiles[0].Name(), filepath.Base(name)) +} diff --git a/log/handler.go b/log/handler.go index bc407857f6..c4b0e9bdf9 100644 --- a/log/handler.go +++ b/log/handler.go @@ -75,14 +75,14 @@ func FileHandler(path string, fmtr Format) (Handler, error) { // RotatingFileHandler returns a handler which writes log records to file chunks // at the given path. When a file's size reaches the limit, the handler creates // a new file named after the timestamp of the first log record it will contain. -func RotatingFileHandler(filePath string, limit uint, formatter Format, rotateHours uint) (Handler, error) { +func RotatingFileHandler(filePath string, limit uint, maxBackups uint, formatter Format, rotateHours uint) (Handler, error) { if _, err := os.Stat(path.Dir(filePath)); os.IsNotExist(err) { err := os.MkdirAll(path.Dir(filePath), 0755) if err != nil { return nil, fmt.Errorf("could not create directory %s, %v", path.Dir(filePath), err) } } - fileWriter := NewAsyncFileWriter(filePath, int64(limit), rotateHours) + fileWriter := NewAsyncFileWriter(filePath, int64(limit), int(maxBackups), rotateHours) fileWriter.Start() return StreamHandler(fileWriter, formatter), nil } diff --git a/log/logger.go b/log/logger.go index 5b89e699ec..3223742ea7 100644 --- a/log/logger.go +++ b/log/logger.go @@ -290,8 +290,8 @@ func (c Ctx) toArray() []interface{} { return arr } -func NewFileLvlHandler(logPath string, maxBytesSize uint, level string, rotateHours uint) Handler { - rfh, err := RotatingFileHandler(logPath, maxBytesSize, LogfmtFormat(), rotateHours) +func NewFileLvlHandler(logPath string, maxBytesSize uint, maxBackups uint, level string, rotateHours uint) Handler { + rfh, err := RotatingFileHandler(logPath, maxBytesSize, maxBackups, LogfmtFormat(), rotateHours) if err != nil { panic(err) } diff --git a/node/config.go b/node/config.go index dc27d48a58..bc30a0ab0a 100644 --- a/node/config.go +++ b/node/config.go @@ -513,6 +513,7 @@ type LogConfig struct { MaxBytesSize *uint `toml:",omitempty"` Level *string `toml:",omitempty"` RotateHours *uint `toml:",omitempty"` + MaxBackups *uint `toml:",omitempty"` // TermTimeFormat is the time format used for console logging. TermTimeFormat *string `toml:",omitempty"` diff --git a/node/node.go b/node/node.go index 37be56b0ad..7f8872f9cf 100644 --- a/node/node.go +++ b/node/node.go @@ -118,7 +118,12 @@ func New(conf *Config) (*Node, error) { rotateHours = *conf.LogConfig.RotateHours } - log.Root().SetHandler(log.NewFileLvlHandler(logFilePath, *conf.LogConfig.MaxBytesSize, *conf.LogConfig.Level, rotateHours)) + maxBackups := uint(0) + if conf.LogConfig.MaxBackups != nil { + maxBackups = *conf.LogConfig.MaxBackups + } + + log.Root().SetHandler(log.NewFileLvlHandler(logFilePath, *conf.LogConfig.MaxBytesSize, maxBackups, *conf.LogConfig.Level, rotateHours)) } } if conf.Logger == nil {