Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: access log remove new log not the old and skip rotate empty log #38309

Merged
merged 1 commit into from
Dec 17, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion internal/proxy/accesslog/global_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
"google.golang.org/grpc/peer"
Expand Down Expand Up @@ -94,14 +95,15 @@ func TestAccessLogger_DynamicEnable(t *testing.T) {
ok := _globalL.Write(accessInfo)
assert.False(t, ok)

etcdCli, _ := etcd.GetEtcdClient(
etcdCli, err := etcd.GetEtcdClient(
Params.EtcdCfg.UseEmbedEtcd.GetAsBool(),
Params.EtcdCfg.EtcdUseSSL.GetAsBool(),
Params.EtcdCfg.Endpoints.GetAsStrings(),
Params.EtcdCfg.EtcdTLSCert.GetValue(),
Params.EtcdCfg.EtcdTLSKey.GetValue(),
Params.EtcdCfg.EtcdTLSCACert.GetValue(),
Params.EtcdCfg.EtcdTLSMinVersion.GetValue())
require.NoError(t, err)

// enable access log
ctx, cancel := context.WithCancel(context.Background())
Expand Down
7 changes: 5 additions & 2 deletions internal/proxy/accesslog/writer.go
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,10 @@
}

func (l *RotateWriter) rotate() error {
if l.size == 0 {
return nil
}

if err := l.closeFile(); err != nil {
return err
}
Expand Down Expand Up @@ -330,7 +334,7 @@
}

if l.maxBackups >= 0 && l.maxBackups < len(files) {
for _, f := range files[l.maxBackups:] {
for _, f := range files[:len(files)-l.maxBackups] {

Check warning on line 337 in internal/proxy/accesslog/writer.go

View check run for this annotation

Codecov / codecov/patch

internal/proxy/accesslog/writer.go#L337

Added line #L337 was not covered by tests
errRemove := os.Remove(path.Join(l.dir(), f.fileName))
if err == nil && errRemove != nil {
err = errRemove
Expand Down Expand Up @@ -436,7 +440,6 @@
}
if t, err := timeFromName(f.Name(), prefix, ext); err == nil {
logFiles = append(logFiles, logInfo{t, f.Name()})
continue
}
}

Expand Down
26 changes: 22 additions & 4 deletions internal/proxy/accesslog/writer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/milvus-io/milvus/pkg/util/paramtable"
)
Expand Down Expand Up @@ -112,28 +113,41 @@ func TestRotateWriter_SizeRotate(t *testing.T) {
Params.Save(Params.ProxyCfg.AccessLog.RemotePath.Key, "access_log/")
Params.Save(Params.ProxyCfg.AccessLog.MaxSize.Key, "1")
defer os.RemoveAll(testPath)
fileNum := 5

logger, err := NewRotateWriter(&Params.ProxyCfg.AccessLog, &Params.MinioCfg)
assert.NoError(t, err)
require.NoError(t, err)
defer logger.handler.Clean()
defer logger.Close()

// return error when write text large than file maxsize
num := 1024 * 1024
text := getText(num + 1)
_, err = logger.Write(text)
assert.Error(t, err)

for i := 1; i <= 2; i++ {
for i := 1; i <= fileNum+1; i++ {
text = getText(num)
n, err := logger.Write(text)
assert.Equal(t, num, n)
assert.NoError(t, err)
}

// assert minio files
time.Sleep(time.Duration(1) * time.Second)
logfiles, err := logger.handler.listAll()
remoteFiles, err := logger.handler.listAll()
assert.NoError(t, err)
assert.Equal(t, 1, len(logfiles))
assert.Equal(t, fileNum, len(remoteFiles))

// assert local sealed files num
localFields, err := logger.oldLogFiles()
assert.NoError(t, err)
assert.Equal(t, fileNum, len(localFields))

// old files should in order
for i := 0; i < fileNum-1; i++ {
assert.True(t, localFields[i].timestamp.Before(localFields[i+1].timestamp))
}
}

func TestRotateWriter_LocalRetention(t *testing.T) {
Expand All @@ -150,9 +164,13 @@ func TestRotateWriter_LocalRetention(t *testing.T) {
assert.NoError(t, err)
defer logger.Close()

// write two sealed files
logger.Write([]byte("Test"))
logger.Rotate()
logger.Write([]byte("Test"))
logger.Rotate()
time.Sleep(time.Duration(1) * time.Second)

logFiles, err := logger.oldLogFiles()
assert.NoError(t, err)
assert.Equal(t, 1, len(logFiles))
Expand Down
Loading