2.0-RC8

Author: leandrosansilva

.gitlab-ci.yml

@@ -39,7 +39,7 @@ reuse-lint:
 
 lint:
   extends: .on-non-tag
-  image: golangci/golangci-lint:v1.42-alpine
+  image: golangci/golangci-lint:v1.45-alpine
   stage: test
   allow_failure: true
   needs: []

.golangci.yml

@@ -13,6 +13,10 @@ run:
 linters:
   enable-all: true
   disable:
+    - forcetypeassert # TODO: this feels like a duplicated linter, not sure if we should disable it globally!
+    - nilnil
+    - ireturn
+    - varnamelen
     - gomoddirectives # TODO: remove this as soon as as https://github.com/imdario/mergo/pull/198 and https://github.com/imdario/mergo/pull/197 are merged
     - promlinter
     - nolintlint

README.md

@@ -95,7 +95,7 @@ The following dependencies are needed during development:
 
 - Bash
 - [Git](https://git-scm.com) on any recent (as in 2020) version.
-- [Go compiler](https://golang.org/) version 1.15 or newer.
+- [Go compiler](https://golang.org/) version 1.18 or newer.
 - [GCC](https://gcc.gnu.org/) version 9.3 or newer.
 - Libc development files. Both [glibc](https://www.gnu.org/software/libc/) and [musl](https://www.musl-libc.org/) have been successfully tested.
 - [GNU Make](https://www.gnu.org/software/make/manual/make.html) or compatible.
@@ -278,6 +278,7 @@ Here are all parameters you can set through environment variables, and their res
 - `LIGHTMETER_LOG_FORMAT=prepend-rfc3339` (`-log_format`)
 - `LIGHTMETER_LOG_FILE_PATTERNS=mail.log:mail.err:mail.warn:zimbra.log:maillog` (`-log_file_patterns`)
 - `LIGHTMETER_I_KNOW_WHAT_I_AM_DOING_NOT_USING_A_REVERSE_PROXY=true` (`-i_know_what_am_doing_not_using_a_reverse_proxy`)
+- `LIGHTMETER_NODE_TYPE="single"` (`-node_type`)
 
 ### Rotated files
 
@@ -321,6 +322,16 @@ In case you are having an error similar to:
 
 This means you should have a file `mail.log`, which means you should check your Postfix installation and ensure it's emitting logs properly.
 
+### Support for multiple servers
+
+There's some preliminary support for reading logs from multiple servers, when they are part of the `$mynetwork` postfix configuration.
+
+More work is needed, but it works well enough for our internal use cases.
+
+To enable it, set the environment variable `LIGHTMETER_NODE_TYPE=multi`, or use the command line argument `-node_type=multi`.
+
+**TODO**: document how it works, caveats and so on. This somehow references #4.
+
 ### Reading from Logstash
 
 **NOTE**: this is a very experimental feature, not well tested or supported. It can eat your logs!

VERSION.txt

@@ -1 +1 @@
-2.0.0-RC7
+2.0.0-RC8

api/dashboard.go

@@ -6,6 +6,10 @@ package api
 
 import (
 	"context"
+	"net/http"
+	"strconv"
+	"time"
+
 	"gitlab.com/lightmeter/controlcenter/dashboard"
 	"gitlab.com/lightmeter/controlcenter/httpauth/auth"
 	"gitlab.com/lightmeter/controlcenter/httpmiddleware"
@@ -14,8 +18,6 @@ import (
 	"gitlab.com/lightmeter/controlcenter/util/httputil"
 	"gitlab.com/lightmeter/controlcenter/util/timeutil"
 	"gitlab.com/lightmeter/controlcenter/version"
-	"net/http"
-	"time"
 )
 
 type handler struct {
@@ -128,6 +130,80 @@ func (h deliveryStatusHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)
 	return servePairsFromTimeInterval(w, r, h.dashboard.DeliveryStatus, interval)
 }
 
+type trafficBySenderOverTimeHandler struct {
+	f func(context.Context, timeutil.TimeInterval, int) (dashboard.MailTrafficPerSenderOverTimeResult, error)
+}
+
+// @Summary Messages sent by mailbox over time
+// @Param from query string true "Initial date in the format 1999-12-23"
+// @Param to   query string true "Final date in the format 1999-12-23"
+// @Param granularity query integer 12 "Time granularity in hours"
+// @Produce json
+// @Success 200 {object} dashboard.MailTrafficPerSenderOverTimeResult
+// @Failure 422 {string} string "desc"
+// @Router /api/v0/sentMailsByMailbox  [get]
+
+// @Summary Messages bounced by mailbox over time
+// @Param from query string true "Initial date in the format 1999-12-23"
+// @Param to   query string true "Final date in the format 1999-12-23"
+// @Param granularity query integer 12 "Time granularity in hours"
+// @Produce json
+// @Success 200 {object} dashboard.MailTrafficPerSenderOverTimeResult
+// @Failure 422 {string} string "desc"
+// @Router /api/v0/bouncedMailsByMailbox  [get]
+
+// @Summary Messages deferred by mailbox over time
+// @Param from query string true "Initial date in the format 1999-12-23"
+// @Param to   query string true "Final date in the format 1999-12-23"
+// @Param granularity query integer 12 "Time granularity in hours"
+// @Produce json
+// @Success 200 {object} dashboard.MailTrafficPerSenderOverTimeResult
+// @Failure 422 {string} string "desc"
+// @Router /api/v0/deferredMailsByMailbox  [get]
+
+// @Summary Messages expired by mailbox over time
+// @Param from query string true "Initial date in the format 1999-12-23"
+// @Param to   query string true "Final date in the format 1999-12-23"
+// @Param granularity query integer 12 "Time granularity in hours"
+// @Produce json
+// @Success 200 {object} dashboard.MailTrafficPerSenderOverTimeResult
+// @Failure 422 {string} string "desc"
+// @Router /api/v0/expiredMailsByMailbox  [get]
+
+// @Summary Messages received by mailbox over time
+// @Param from query string true "Initial date in the format 1999-12-23"
+// @Param to   query string true "Final date in the format 1999-12-23"
+// @Param granularity query integer 12 "Time granularity in hours"
+// @Produce json
+// @Success 200 {object} dashboard.MailTrafficPerSenderOverTimeResult
+// @Failure 422 {string} string "desc"
+// @Router /api/v0/receivedMailsByMailbox  [get]
+
+// @Summary Number of inbound replies
+// @Param from query string true "Initial date in the format 1999-12-23"
+// @Param to   query string true "Final date in the format 1999-12-23"
+// @Param granularity query integer 12 "Time granularity in hours"
+// @Produce json
+// @Success 200 {object} dashboard.MailTrafficPerSenderOverTimeResult
+// @Failure 422 {string} string "desc"
+// @Router /api/v0/inboundRepliesByMailbox [get]
+
+func (h trafficBySenderOverTimeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) error {
+	interval := httpmiddleware.GetIntervalFromContext(r)
+
+	granularity, err := strconv.Atoi(r.Form.Get("granularity"))
+	if err != nil {
+		return httperror.NewHTTPStatusCodeError(http.StatusUnprocessableEntity, err)
+	}
+
+	result, err := h.f(r.Context(), interval, granularity)
+	if err != nil {
+		return err
+	}
+
+	return httputil.WriteJson(w, result, http.StatusOK)
+}
+
 type appVersionHandler struct{}
 
 type appVersion struct {
@@ -144,14 +220,25 @@ func (appVersionHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) error
 	return httputil.WriteJson(w, appVersion{Version: version.Version, Commit: version.Commit, TagOrBranch: version.TagOrBranch}, http.StatusOK)
 }
 
-func HttpDashboard(auth *auth.Authenticator, mux *http.ServeMux, timezone *time.Location, dashboard dashboard.Dashboard) {
+func HttpDashboard(auth *auth.Authenticator, mux *http.ServeMux, timezone *time.Location, d dashboard.Dashboard) {
 	authenticated := httpmiddleware.WithDefaultStack(auth, httpmiddleware.RequestWithInterval(timezone))
 	unauthenticated := httpmiddleware.WithDefaultStackWithoutAuth()
 
-	mux.Handle("/api/v0/countByStatus", authenticated.WithEndpoint(countByStatusHandler{dashboard}))
-	mux.Handle("/api/v0/topBusiestDomains", authenticated.WithEndpoint(topBusiestDomainsHandler{dashboard}))
-	mux.Handle("/api/v0/topBouncedDomains", authenticated.WithEndpoint(topBouncedDomainsHandler{dashboard}))
-	mux.Handle("/api/v0/topDeferredDomains", authenticated.WithEndpoint(topDeferredDomainsHandler{dashboard}))
-	mux.Handle("/api/v0/deliveryStatus", authenticated.WithEndpoint(deliveryStatusHandler{dashboard}))
+	for k, v := range map[string]func(context.Context, timeutil.TimeInterval, int) (dashboard.MailTrafficPerSenderOverTimeResult, error){
+		"/api/v0/sentMailsByMailbox":      d.SentMailsByMailbox,
+		"/api/v0/bouncedMailsByMailbox":   d.BouncedMailsByMailbox,
+		"/api/v0/deferredMailsByMailbox":  d.DeferredMailsByMailbox,
+		"/api/v0/expiredMailsByMailbox":   d.ExpiredMailsByMailbox,
+		"/api/v0/receivedMailsByMailbox":  d.ReceivedMailsByMailbox,
+		"/api/v0/inboundRepliesByMailbox": d.InboundRepliesByMailbox,
+	} {
+		mux.Handle(k, authenticated.WithEndpoint(trafficBySenderOverTimeHandler{v}))
+	}
+
+	mux.Handle("/api/v0/countByStatus", authenticated.WithEndpoint(countByStatusHandler{d}))
+	mux.Handle("/api/v0/topBusiestDomains", authenticated.WithEndpoint(topBusiestDomainsHandler{d}))
+	mux.Handle("/api/v0/topBouncedDomains", authenticated.WithEndpoint(topBouncedDomainsHandler{d}))
+	mux.Handle("/api/v0/topDeferredDomains", authenticated.WithEndpoint(topDeferredDomainsHandler{d}))
+	mux.Handle("/api/v0/deliveryStatus", authenticated.WithEndpoint(deliveryStatusHandler{d}))
 	mux.Handle("/api/v0/appVersion", unauthenticated.WithEndpoint(appVersionHandler{}))
 }

api/detective.go

@@ -91,7 +91,7 @@ func requireDetectiveAuth(auth *httpauth.Authenticator, settingsReader metadata.
 			isAuthenticated := isUserAuthenticated(auth, r)
 
 			settings := detectivesettings.Settings{}
-			err := settingsReader.RetrieveJson(r.Context(), detectivesettings.SettingKey, &settings)
+			err := settingsReader.RetrieveJson(r.Context(), detectivesettings.SettingsKey, &settings)
 			if err != nil && !errors.Is(err, metadata.ErrNoSuchKey) {
 				return httperror.NewHTTPStatusCodeError(http.StatusInternalServerError, errorutil.Wrap(err))
 			}

api/detective_auth_test.go

@@ -6,6 +6,13 @@ package api
 
 import (
 	"context"
+	"net/http"
+	"net/http/cookiejar"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+	"time"
+
 	"github.com/golang/mock/gomock"
 	. "github.com/smartystreets/goconvey/convey"
 	"gitlab.com/lightmeter/controlcenter/detective"
@@ -17,12 +24,6 @@ import (
 	"gitlab.com/lightmeter/controlcenter/pkg/runner"
 	detectivesettings "gitlab.com/lightmeter/controlcenter/settings/detective"
 	"gitlab.com/lightmeter/controlcenter/util/testutil"
-	"net/http"
-	"net/http/cookiejar"
-	"net/http/httptest"
-	"net/url"
-	"testing"
-	"time"
 )
 
 func init() {
@@ -49,7 +50,7 @@ func buildTestEnv(t *testing.T) (*httptest.Server, *mock_detective.MockDetective
 
 	detective := mock_detective.NewMockDetective(ctrl)
 
-	auth := auth.NewAuthenticator(registrar, dir)
+	auth := auth.NewAuthenticator(registrar, dir, nil)
 	mux := http.NewServeMux()
 
 	settingdDB, removeDB := testutil.TempDBConnectionMigrated(t, "master")

api/gen.go

@@ -4,4 +4,4 @@
 
 package api
 
-//go:generate go run -mod vendor github.com/swaggo/swag/cmd/swag init --generalInfo api.go
+//go:generate go run -mod vendor github.com/swaggo/swag/cmd/swag init --parseDependency --parseInternal --generalInfo api.go

api/rawlogs.go

@@ -5,19 +5,21 @@
 package api
 
 import (
+	"bufio"
 	"compress/gzip"
 	"errors"
 	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"time"
+
 	"gitlab.com/lightmeter/controlcenter/httpauth/auth"
 	"gitlab.com/lightmeter/controlcenter/httpmiddleware"
 	"gitlab.com/lightmeter/controlcenter/rawlogsdb"
 	"gitlab.com/lightmeter/controlcenter/util/errorutil"
 	"gitlab.com/lightmeter/controlcenter/util/httputil"
 	"gitlab.com/lightmeter/controlcenter/util/timeutil"
-	"io"
-	"net/http"
-	"strconv"
-	"time"
 )
 
 //nolint:structcheck,unused
@@ -123,7 +125,12 @@ func (h fetchRawLogLinesToWriterHandler) ServeHTTP(w http.ResponseWriter, r *htt
 
 	defer errorutil.UpdateErrorFromCall(releaseWriter, &err)
 
-	if err := h.accessor.FetchLogsInIntervalToWriter(r.Context(), interval, writer); err != nil {
+	// use a 4KB buffer to improve write throughput, reducing the number of write() calls in the final socket
+	bufferedWriter := bufio.NewWriterSize(writer, 4096)
+
+	defer errorutil.UpdateErrorFromCall(bufferedWriter.Flush, &err)
+
+	if err := h.accessor.FetchLogsInIntervalToWriter(r.Context(), interval, bufferedWriter); err != nil {
 		return errorutil.Wrap(err)
 	}
 

app_test.go

@@ -6,6 +6,11 @@ package main
 
 import (
 	"context"
+	"os"
+	"path"
+	"testing"
+	"time"
+
 	. "github.com/smartystreets/goconvey/convey"
 	"gitlab.com/lightmeter/controlcenter/config"
 	"gitlab.com/lightmeter/controlcenter/detective/escalator"
@@ -16,10 +21,6 @@ import (
 	"gitlab.com/lightmeter/controlcenter/util/testutil"
 	"gitlab.com/lightmeter/controlcenter/util/timeutil"
 	"gitlab.com/lightmeter/controlcenter/workspace"
-	"os"
-	"path"
-	"testing"
-	"time"
 )
 
 func init() {
@@ -55,9 +56,10 @@ func TestMain(t *testing.T) {
 		// Only import the logs in a workspace
 		config := config.Config{
 			WorkspaceDirectory: wsDir,
-			DirToWatch:         path.Join(logsDir, "logs_sample"),
+			DirsToWatch:        []string{path.Join(logsDir, "logs_sample")},
 			ImportOnly:         true,
 			LogFormat:          "default",
+			MultiNodeType:      "single",
 		}
 
 		// first execution