From 55a3fa1992f67f0cf0ebf95f9d3708ec974eb384 Mon Sep 17 00:00:00 2001 From: Manfred Touron <94029+moul@users.noreply.github.com> Date: Wed, 9 Jun 2021 09:22:14 +0200 Subject: [PATCH] feat: v1 Signed-off-by: Manfred Touron <94029+moul@users.noreply.github.com> --- Makefile | 9 ++- README.md | 48 ++++++++++--- doc.go | 2 +- example_test.go | 9 +++ go.mod | 7 +- go.sum | 24 +++++++ logman.go | 151 +++++++++++++++++++++++++++++++++++++++++ logman_test.go | 174 ++++++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 405 insertions(+), 19 deletions(-) create mode 100644 example_test.go create mode 100644 logman.go create mode 100644 logman_test.go diff --git a/Makefile b/Makefile index 793aefa..525d89f 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,9 @@ GOPKG ?= moul.io/logman include rules.mk -lint: - cd tool/lint; make -.PHONY: lint +generate: + GO111MODULE=off go get github.com/campoy/embedmd + mkdir -p .tmp + go doc -all > .tmp/godoc.txt + embedmd -w README.md + rm -rf .tmp diff --git a/README.md b/README.md index df4c9ea..dd9e53c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # logman -:smile: logman +:smile: golang library to organize log files (create by date, GC, etc) [![go.dev reference](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white)](https://pkg.go.dev/moul.io/logman) [![License](https://img.shields.io/badge/license-Apache--2.0%20%2F%20MIT-%2397ca00.svg)](https://github.com/moul/logman/blob/main/COPYRIGHT) @@ -18,18 +18,44 @@ [![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/moul/logman) +## Example + +[embedmd]:# (example_test.go /import\ / $) +```go +import "moul.io/logman" + +func Example() { + writer, _ := logman.NewWriteCloser("./path/to/logdir/", "my-app") + defer writer.Close() + writer.Write([]byte("hello world!")) +} +``` + ## Usage -[embedmd]:# (.tmp/usage.txt console) -```console -foo@bar:~$ logman hello world - _ _ _ _ - __ _ ___ | | __ _ _ _ __ _ ___ _ _ ___ _ __ ___ ___ | |_ ___ _ __ _ __ | | __ _ | |_ ___ -/ _` |/ _ \| |/ _` || ' \ / _` ||___|| '_|/ -_)| '_ \/ _ \|___|| _|/ -_)| ' \ | '_ \| |/ _` || _|/ -_) -\__, |\___/|_|\__,_||_||_|\__, | |_| \___|| .__/\___/ \__|\___||_|_|_|| .__/|_|\__,_| \__|\___| -|___/ |___/ |_| |_| -12 CPUs, /home/moul/.gvm/pkgsets/go1.16/global/bin/logman, fwrz, go1.16 -args ["logman","hello","world"] +[embedmd]:# (.tmp/godoc.txt txt /FUNCTIONS/ $) +```txt +FUNCTIONS + +func LogfileGC(logDir string, max int) error +func NewWriteCloser(target, kind string) (io.WriteCloser, error) + +TYPES + +type Logfile struct { + Dir string + Name string + Size int64 + Kind string + Time time.Time + Latest bool + Errs error `json:"Errs,omitempty"` +} + +func LogfileList(logDir string) ([]*Logfile, error) + +func (l Logfile) Path() string + ``` ## Install diff --git a/doc.go b/doc.go index 509d53f..13871db 100644 --- a/doc.go +++ b/doc.go @@ -26,4 +26,4 @@ // |/ | | \ \ / / ' \/ _ \/ // / / | // || | | | | | /_/_/_/\___/\_,_/_/ | // +--------------------------------------------------------------+ -package main // import "moul.io/logman" +package logman diff --git a/example_test.go b/example_test.go new file mode 100644 index 0000000..550977a --- /dev/null +++ b/example_test.go @@ -0,0 +1,9 @@ +package logman_test + +import "moul.io/logman" + +func Example() { + writer, _ := logman.NewWriteCloser("./path/to/logdir/", "my-app") + defer writer.Close() + writer.Write([]byte("hello world!")) +} diff --git a/go.mod b/go.mod index f1bc263..2f32cf7 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,8 @@ module moul.io/logman go 1.13 require ( - github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e // indirect + github.com/stretchr/testify v1.7.0 github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027 - golang.org/x/mod v0.4.2 // indirect - golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44 // indirect - golang.org/x/tools v0.1.0 // indirect + go.uber.org/multierr v1.7.0 + moul.io/u v1.24.0 ) diff --git a/go.sum b/go.sum index 8ffa6a9..61f3aff 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,31 @@ +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027 h1:lK99QQdH3yBWY6aGilF+IRlQIdmhzLrsEmF6JgN+Ryw= github.com/tailscale/depaware v0.0.0-20201214215404-77d1e9757027/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -39,4 +56,11 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +moul.io/u v1.24.0 h1:USCFlvwr26ej2WMYTbS3XMsjq7UfziUpKQjH5kApIPY= +moul.io/u v1.24.0/go.mod h1:ytlQ/zt+Sdk+PFGEx+fpTivoa0ieA5yMo6itRswIWNQ= diff --git a/logman.go b/logman.go new file mode 100644 index 0000000..ccdb3fb --- /dev/null +++ b/logman.go @@ -0,0 +1,151 @@ +package logman + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "time" + + "go.uber.org/multierr" + "moul.io/u" +) + +func NewWriteCloser(target, kind string) (io.WriteCloser, error) { + var filename string + switch { + case strings.HasSuffix(target, ".log"): // use the indicated 'path' as filename + filename = target + default: // automatically create a new file in the 'path' directory following a pattern + startTime := time.Now().Format(filePatternDateLayout) + filename = filepath.Join( + target, + fmt.Sprintf("%s-%s.log", kind, startTime), + ) + // run gc + { + err := LogfileGC(target, 20) + if err != nil { + return nil, err + } + } + } + + if dir := filepath.Dir(filename); !u.DirExists(dir) { + err := os.MkdirAll(dir, 0o711) + if err != nil { + return nil, err + } + } + + var writer io.WriteCloser + if u.FileExists(filename) { + var err error + writer, err = os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, os.ModeAppend) + if err != nil { + return nil, err + } + } else { + var err error + writer, err = os.Create(filename) + if err != nil { + return nil, err + } + } + + return writer, nil +} + +type Logfile struct { + Dir string + Name string + Size int64 + Kind string + Time time.Time + Latest bool + Errs error `json:"Errs,omitempty"` +} + +func (l Logfile) Path() string { + return filepath.Join(l.Dir, l.Name) +} + +const filePatternDateLayout = "2006-01-02T15-04-05.000" + +var filePatternRegex = regexp.MustCompile(`(?m)^(.*)-(\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}.\d{3}).log$`) + +func LogfileList(logDir string) ([]*Logfile, error) { + files, err := ioutil.ReadDir(logDir) + if err != nil { + return nil, err + } + + logfiles := []*Logfile{} + for _, file := range files { + sub := filePatternRegex.FindStringSubmatch(file.Name()) + if sub == nil { + continue + } + t, err := time.Parse(filePatternDateLayout, sub[2]) + var errs error + if err != nil { + errs = multierr.Append(errs, err) + } + + logfiles = append(logfiles, &Logfile{ + Dir: logDir, + Name: file.Name(), + Size: file.Size(), + Kind: sub[1], + Time: t, + Errs: errs, + }) + } + + // compute latest + if len(logfiles) > 0 { + var maxTime time.Time + for _, file := range logfiles { + if file.Time.After(maxTime) { + maxTime = file.Time + } + } + for _, file := range logfiles { + if file.Time == maxTime { + file.Latest = true + } + } + } + + return logfiles, nil +} + +func LogfileGC(logDir string, max int) error { + if !u.DirExists(logDir) { + return nil + } + files, err := LogfileList(logDir) + if err != nil { + return err + } + if len(files) < max { + return nil + } + + sort.Slice(files, func(i, j int) bool { + return files[i].Time.Before(files[j].Time) + }) + + var errs error + for i := 0; i < len(files)-max; i++ { + err := os.Remove(files[i].Path()) + if err != nil { + errs = multierr.Append(errs, err) + } + } + return errs +} diff --git a/logman_test.go b/logman_test.go new file mode 100644 index 0000000..89b2009 --- /dev/null +++ b/logman_test.go @@ -0,0 +1,174 @@ +package logman_test + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + "moul.io/logman" + "moul.io/u" +) + +func TestLogfile(t *testing.T) { + // setup volatile directory for the test + tempdir, err := ioutil.TempDir("", "logutil-file") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + // check loading log files from an invalid directory + { + files, err := logman.LogfileList(filepath.Join(tempdir, "doesnotexist")) + require.Error(t, err) + require.Nil(t, files) + } + + // check loading files from empty valid directory + { + files, err := logman.LogfileList(tempdir) + require.NoError(t, err) + require.Empty(t, files) + } + + // create dummy files + { + dummyNames := []string{ + "2021-05-25T21-12-02.650.log", + "cli.info-2021-05-25T21-12-02.aaa.log", + "blah.log", + } + for _, name := range dummyNames { + f, err := os.Create(filepath.Join(tempdir, name)) + require.NoError(t, err) + err = f.Close() + require.NoError(t, err) + } + } + + // check loading files from valid directory with only dummy files + { + files, err := logman.LogfileList(tempdir) + require.NoError(t, err) + require.Empty(t, files) + } + + // create a first logger of kind-1 + { + writer, err := logman.NewWriteCloser(tempdir, "kind-1") + require.NoError(t, err) + require.NotNil(t, writer) + _, err = writer.Write([]byte("blah\n")) + require.NoError(t, err) + err = writer.Close() + require.NoError(t, err) + } + + // check loading files from the directory, should have one now + { + files, err := logman.LogfileList(tempdir) + require.NoError(t, err) + require.Len(t, files, 1) + require.Equal(t, files[0].Dir, tempdir) + require.NotEmpty(t, files[0].Name) + require.Equal(t, files[0].Path(), filepath.Join(tempdir, files[0].Name)) + require.True(t, u.FileExists(files[0].Path())) + require.True(t, files[0].Latest) + require.Equal(t, files[0].Kind, "kind-1") + } + + // create a second logger of kind-1 + { + time.Sleep(time.Second) + writer, err := logman.NewWriteCloser(tempdir, "kind-1") + require.NoError(t, err) + require.NotNil(t, writer) + _, err = writer.Write([]byte("blah blah\n")) + require.NoError(t, err) + err = writer.Close() + require.NoError(t, err) + } + + // check loading files from the directory, should have two now + { + files, err := logman.LogfileList(tempdir) + require.NoError(t, err) + require.Len(t, files, 2) + for _, file := range files { + require.Equal(t, file.Dir, tempdir) + require.NotEmpty(t, file.Name) + require.Equal(t, file.Path(), filepath.Join(tempdir, file.Name)) + require.True(t, u.FileExists(file.Path())) + } + } + + // try to gc with fewer files than the limit + { + err := logman.LogfileGC(tempdir, 10) + require.NoError(t, err) + } + + // create 10 new files + { + for i := 0; i < 10; i++ { + writer, err := logman.NewWriteCloser(tempdir, fmt.Sprintf("hello-%d", i)) + require.NoError(t, err) + err = writer.Close() + require.NoError(t, err) + } + } + + // check loading files from the directory, should have twelve now + { + files, err := logman.LogfileList(tempdir) + require.NoError(t, err) + require.Len(t, files, 12) + for _, file := range files { + require.Equal(t, file.Dir, tempdir) + require.NotEmpty(t, file.Name) + require.Equal(t, file.Path(), filepath.Join(tempdir, file.Name)) + require.True(t, u.FileExists(file.Path())) + } + } + + // try to gc with fewer files than the limit + { + err := logman.LogfileGC(tempdir, 10) + require.NoError(t, err) + } + + // check loading files from the directory, should have ten now + { + files, err := logman.LogfileList(tempdir) + require.NoError(t, err) + require.Len(t, files, 10) + } + + // try to gc with the current amount of files + { + err := logman.LogfileGC(tempdir, 10) + require.NoError(t, err) + } + + // check loading files from the directory, should still have ten + { + files, err := logman.LogfileList(tempdir) + require.NoError(t, err) + require.Len(t, files, 10) + } + + // try to gc with only one + { + err := logman.LogfileGC(tempdir, 1) + require.NoError(t, err) + } + + // check loading files from the directory, should now have only one + { + files, err := logman.LogfileList(tempdir) + require.NoError(t, err) + require.Len(t, files, 1) + } +}