Merge branch 'feature/preps-for-rc9' into 'master'

Preps for RC9

See merge request lightmeter/controlcenter!989

Author: leandrosansilva

VERSION.txt

@@ -1 +1 @@
-2.0.0-RC8
+2.0.0-RC9

logeater/dirwatcher/dir_importer.go

@@ -9,6 +9,13 @@ import (
 	"container/heap"
 	"errors"
 	"fmt"
+	"io"
+	"path"
+	"regexp"
+	"sort"
+	"strconv"
+	"time"
+
 	"github.com/rs/zerolog/log"
 	"gitlab.com/lightmeter/controlcenter/logeater/announcer"
 	"gitlab.com/lightmeter/controlcenter/pkg/closers"
@@ -17,12 +24,6 @@ import (
 	parsertimeutil "gitlab.com/lightmeter/controlcenter/pkg/postfix/logparser/timeutil"
 	"gitlab.com/lightmeter/controlcenter/util/errorutil"
 	"gitlab.com/lightmeter/controlcenter/util/timeutil"
-	"io"
-	"path"
-	"regexp"
-	"sort"
-	"strconv"
-	"time"
 )
 
 type fileEntry struct {
@@ -35,11 +36,10 @@ type fileEntryList []fileEntry
 type fileQueues map[string]fileEntryList
 
 func sortedEntriesFilteredByPatternAndMoreRecentThanTime(list fileEntryList, r filenameRecognizer, initialTime time.Time) fileEntryList {
-	entries := make(fileEntryList, 0, len(list))
-
 	type rec struct {
-		entry fileEntry
-		index int
+		entry      fileEntry
+		index      int
+		compressed bool
 	}
 
 	recs := []rec{}
@@ -70,7 +70,17 @@ func sortedEntriesFilteredByPatternAndMoreRecentThanTime(list fileEntryList, r f
 			return index
 		}()
 
-		recs = append(recs, rec{entry: entry, index: index})
+		recs = append(recs, rec{entry: entry, index: index, compressed: len(matches[5]) != 0})
+	}
+
+	// no need to do the sorting with no entries at all.
+	if len(recs) == 0 {
+		return fileEntryList{}
+	}
+
+	// No need to do the sorting for a single entry.
+	if len(recs) == 1 {
+		return fileEntryList{recs[0].entry}
 	}
 
 	sort.Slice(recs, func(i, j int) bool {
@@ -83,11 +93,27 @@ func sortedEntriesFilteredByPatternAndMoreRecentThanTime(list fileEntryList, r f
 			return false
 		}
 
+		// same filename, but one of them is compressed.
+		// the compressed goes first
+		if recs[i].index == recs[j].index {
+			return recs[i].compressed
+		}
+
 		return recs[i].index*int(r.order) < recs[j].index*int(r.order)
 	})
 
-	for _, r := range recs {
-		entries = append(entries, r.entry)
+	entries := make(fileEntryList, 0, len(list))
+
+	// NOTE: here we know we have at least one entry
+	entries = append(entries, recs[0].entry)
+
+	for i, lastAddedRec := 1, recs[0]; i < len(recs); i++ {
+		r := recs[i]
+
+		if r.index != lastAddedRec.index {
+			entries = append(entries, r.entry)
+			lastAddedRec = r
+		}
 	}
 
 	return entries

logeater/dirwatcher/regression_test.go

@@ -5,12 +5,14 @@
 package dirwatcher
 
 import (
+	"testing"
+	"time"
+
 	. "github.com/smartystreets/goconvey/convey"
 	"gitlab.com/lightmeter/controlcenter/pkg/postfix"
 	parsertimeutil "gitlab.com/lightmeter/controlcenter/pkg/postfix/logparser/timeutil"
 	"gitlab.com/lightmeter/controlcenter/util/testutil"
 	"gitlab.com/lightmeter/controlcenter/util/timeutil"
-	"testing"
 )
 
 func TestRegressionIssue368(t *testing.T) {
@@ -199,3 +201,34 @@ Mar  8 10:11:12 cloud2 postfix/qmgr[1428]: 5EEC73E8C6: removed`, ``),
 		})
 	})
 }
+
+func TestDuplicatedFiles(t *testing.T) {
+	Convey("If the same file seems to be present in both compressed and non compressed form, choose the compressed ont", t, func() {
+		Convey("Some files are duplicated, due to rsync'ng generated artifacts", func() {
+			f := fileEntryList{
+				fileEntry{filename: "mail.log", modificationTime: testutil.MustParseTime(`2023-03-02 19:38:51 +0100`)},
+				fileEntry{filename: "mail.log-20220722", modificationTime: testutil.MustParseTime(`2022-07-25 15:07:12 +0200`)},
+				fileEntry{filename: "mail.log-20220722.bz2", modificationTime: testutil.MustParseTime(`2022-07-25 15:07:12 +0200`)},
+				fileEntry{filename: "mail.log-20220728", modificationTime: testutil.MustParseTime(`2022-07-29 16:14:44 +0200`)},
+				fileEntry{filename: "mail.log-20220728.bz2", modificationTime: testutil.MustParseTime(`2022-07-29 16:14:44 +0200`)},
+				fileEntry{filename: "mail.log-20220731", modificationTime: testutil.MustParseTime(`2022-08-02 14:20:14 +0200`)},
+				fileEntry{filename: "mail.log-20220731.bz2", modificationTime: testutil.MustParseTime(`2022-08-02 14:20:14 +0200`)},
+				fileEntry{filename: "mail.log-20220804", modificationTime: testutil.MustParseTime(`2022-08-22 11:20:08 +0200`)},
+			}
+
+			// We ignore all files that have a correspondent compressed version of it
+			So(buildFilesToImport(f, BuildLogPatterns([]string{"mail.log", "mail.err", "mail.warn"}), time.Time{}), ShouldResemble,
+				fileQueues{
+					"mail.log": fileEntryList{
+						fileEntry{filename: "mail.log-20220722.bz2", modificationTime: testutil.MustParseTime(`2022-07-25 15:07:12 +0200`)},
+						fileEntry{filename: "mail.log-20220728.bz2", modificationTime: testutil.MustParseTime(`2022-07-29 16:14:44 +0200`)},
+						fileEntry{filename: "mail.log-20220731.bz2", modificationTime: testutil.MustParseTime(`2022-08-02 14:20:14 +0200`)},
+						fileEntry{filename: "mail.log-20220804", modificationTime: testutil.MustParseTime(`2022-08-22 11:20:08 +0200`)},
+						fileEntry{filename: "mail.log", modificationTime: testutil.MustParseTime(`2023-03-02 19:38:51 +0100`)},
+					},
+					"mail.err":  {},
+					"mail.warn": {},
+				})
+		})
+	})
+}

release_notes/2.0.0-RC9

@@ -0,0 +1,17 @@
+# ControlCenter 2.0.0-RC9
+
+It turns out I was wrong when said we'd not release another RC. Oops, I just did.
+
+It fixes an issue that makes the log parser get confused when parsing log files
+in the presence of compressed log files in the same directory (which is usually done)
+by logrotate.
+
+There's no Gitlab issue in this repo, but this is the relevant MR:
+
+- https://gitlab.com/lightmeter/controlcenter/-/merge_requests/988
+
+To reiterate, ControlCenter is not actively developed anymore, as our focus as
+organization shifted from the Postfix monitoring space.
+
+Being realist, there might eventually be some maintenance releases like the current one,
+but please do not expect new features.