Merge branch 'feature/release-2.0-RC6' into 'master'

Preparations for 2.0-RC6

See merge request lightmeter/controlcenter!956

Author: leandrosansilva

.gitlab-ci.yml

@@ -91,7 +91,7 @@ test:
   needs: []
   script:
     - make mocks
-    - go test ./... -race -v -tags="sqlite_json" | go run github.com/jstemmer/go-junit-report > report.xml
+    - go test ./... -race -v | go run github.com/jstemmer/go-junit-report > report.xml
   artifacts:
     reports:
       junit: report.xml
@@ -103,7 +103,7 @@ cover:
   needs: []
   script:
     - go get -u github.com/t-yuki/gocover-cobertura
-    - ./tools/go_test.sh -tags sqlite_json -coverprofile=coverinfo_with_gen.txt
+    - ./tools/go_test.sh -coverprofile=coverinfo_with_gen.txt
     - grep -v '.gen.go' < coverinfo_with_gen.txt > coverinfo.txt
     - go run github.com/t-yuki/gocover-cobertura < coverinfo.txt > cobertura.xml
     - go tool cover -html=coverinfo.txt -o coverage.html

Makefile

@@ -22,31 +22,33 @@ all:
 	$(error Use make (dev|release|static_release) instead)
 
 race:
-	./tools/go_test.sh -race -tags="sqlite_json"
+	./tools/go_test.sh -race
 
 BUILD_DEPENDENCIES = go gcc ragel npm vue
 $(foreach exec,$(BUILD_DEPENDENCIES),\
     $(if $(shell command -v $(exec) 2> /dev/null),$(info Found executable `$(exec)`),$(error "Build dependency program $(exec) could not be found in PATH. Check README.md for more info")))
 
 dev_headless_pre_build: mocks translations swag static_www postfix_parser domain_mapping_list po2go www
 
+pre_build: export SKIP_VUE_SOURCE_MAPS := false
 dev_pre_build: npminstall mocks translations frontend_root swag static_www postfix_parser domain_mapping_list po2go
 
+pre_build: export SKIP_VUE_SOURCE_MAPS := true
 pre_build: npminstall translations frontend_root static_www postfix_parser domain_mapping_list po2go email_notification_template
 
 pre_release: pre_build recommendation_release
 
 dev_bin: dev_pre_build recommendation_dev
-	go build -tags="dev include no_postgres no_mysql no_clickhouse no_mssql sqlite_json" -o "lightmeter" -ldflags "${BUILD_INFO_FLAGS}"
+	go build -tags="dev include no_postgres no_mysql no_clickhouse no_mssql" -o "lightmeter" -ldflags "${BUILD_INFO_FLAGS}"
 
 dev_headless_bin: dev_headless_pre_build recommendation_dev
-	go build -tags="dev include no_postgres no_mysql no_clickhouse no_mssql sqlite_json" -o "lightmeter" -ldflags "${BUILD_INFO_FLAGS}"
+	go build -tags="dev include no_postgres no_mysql no_clickhouse no_mssql" -o "lightmeter" -ldflags "${BUILD_INFO_FLAGS}"
 
 release_bin: pre_release
-	go build -tags="release include no_postgres no_mysql no_clickhouse no_mssql sqlite_json" -o "lightmeter" -ldflags "${BUILD_INFO_FLAGS}"
+	go build -tags="release include no_postgres no_mysql no_clickhouse no_mssql" -o "lightmeter" -ldflags "${BUILD_INFO_FLAGS}"
 
 static_release_bin: pre_release
-	go build -tags="release include no_postgres no_mysql no_clickhouse no_mssql sqlite_json" -o "lightmeter" -ldflags \
+	go build -tags="release include no_postgres no_mysql no_clickhouse no_mssql" -o "lightmeter" -ldflags \
 		"${BUILD_INFO_FLAGS} -linkmode external -extldflags '-static' -s -w" -a -v
 
 static_www:
@@ -81,7 +83,7 @@ timeutil_mock:
 	go generate -tags="dev" gitlab.com/lightmeter/controlcenter/util/timeutil
 
 detective_mock:
-	go generate -tags="dev sqlite_json" gitlab.com/lightmeter/controlcenter/detective
+	go generate -tags="dev" gitlab.com/lightmeter/controlcenter/detective
 
 po2go:
 	go generate -tags="dev" gitlab.com/lightmeter/controlcenter/po

VERSION.txt

@@ -1 +1 @@
-2.0.0-RC5
+2.0.0-RC6

acceptance_tests/specs/6_message_detective.spec

@@ -11,7 +11,7 @@ Search for messages
 * Click on "Search"
 
 * Expect "1" detective results
-* Expect to see "Sent"
+* Expect to see "Received"
 * Expect to see "sender@sender.example.com"
 * Expect to see "user@recipient.example.com"
 * Expect to see "1F5A2194AA9"

api/detective_test.go

@@ -96,9 +96,11 @@ func TestDetectiveCheckMessageDeliveryHandler(t *testing.T) {
 							testutil.MustParseTime(`2009-02-14 00:31:30 +0000`),
 							detective.Status(parser.SentStatus),
 							"2.0.0",
+							[]string{"google.com"},
 							nil,
 							"user1@example.org",
 							[]string{"user2@example.org"},
+							nil,
 						},
 						},
 					},

api/rawlogs_test.go

@@ -13,6 +13,7 @@ import (
 	"gitlab.com/lightmeter/controlcenter/httpauth"
 	"gitlab.com/lightmeter/controlcenter/httpauth/auth"
 	"gitlab.com/lightmeter/controlcenter/metadata"
+	"gitlab.com/lightmeter/controlcenter/pkg/postfix"
 	"gitlab.com/lightmeter/controlcenter/rawlogsdb"
 	"gitlab.com/lightmeter/controlcenter/util/testutil"
 	"gitlab.com/lightmeter/controlcenter/util/timeutil"
@@ -46,6 +47,10 @@ func (f *fakeRawLogsFetcher) CountLogLinesInInterval(context.Context, timeutil.T
 	return 42, nil
 }
 
+func (f *fakeRawLogsFetcher) FetchLogLine(context.Context, time.Time, postfix.Sum) (string, error) {
+	return "", nil
+}
+
 func TestFetchingRawLogs(t *testing.T) {
 	Convey("Fetch Raw Logs", t, func() {
 		registrar := &auth.FakeRegistrar{

cli_usage.md

@@ -1,5 +1,5 @@
 ```
-Lightmeter ControlCenter 2.0.0-RC4
+Lightmeter ControlCenter 2.0.0-RC6
 
  Example call: 
 

config/config.go

@@ -9,12 +9,12 @@ import (
 	"flag"
 	"fmt"
 	"os"
-	"strconv"
 	"strings"
 	"time"
 
 	"github.com/rs/zerolog"
 	"gitlab.com/lightmeter/controlcenter/metadata"
+	"gitlab.com/lightmeter/controlcenter/util/envutil"
 	"gitlab.com/lightmeter/controlcenter/util/errorutil"
 	"gitlab.com/lightmeter/controlcenter/version"
 )
@@ -67,20 +67,20 @@ func ParseWithErrorHandling(cmdlineArgs []string, lookupenv func(string) (string
 	fs.BoolVar(&conf.ShouldWatchFromStdin, "stdin", false, "Read log lines from stdin")
 
 	fs.StringVar(&conf.WorkspaceDirectory, "workspace",
-		lookupEnvOrString("LIGHTMETER_WORKSPACE", "/var/lib/lightmeter_workspace", lookupenv),
+		envutil.LookupEnvOrString("LIGHTMETER_WORKSPACE", "/var/lib/lightmeter_workspace", lookupenv),
 		"Path to the directory to store all working data")
 
 	fs.BoolVar(&conf.ImportOnly, "importonly", false,
 		"Only import existing logs, exiting immediately, without running the full application.")
 
-	b, err := lookupEnvOrBool("LIGHTMETER_LOGS_USE_RSYNC", false, lookupenv)
+	b, err := envutil.LookupEnvOrBool("LIGHTMETER_LOGS_USE_RSYNC", false, lookupenv)
 	if err != nil {
 		return conf, err
 	}
 
 	fs.BoolVar(&conf.RsyncedDir, "logs_use_rsync", b, "Log directory is updated by rsync")
 
-	logYear, err := lookupEnvOrInt("LIGHTMETER_LOGS_STARTING_YEAR", 0, lookupenv)
+	logYear, err := envutil.LookupEnvOrInt("LIGHTMETER_LOGS_STARTING_YEAR", 0, lookupenv)
 	if err != nil {
 		return conf, err
 	}
@@ -91,17 +91,17 @@ func ParseWithErrorHandling(cmdlineArgs []string, lookupenv func(string) (string
 	fs.BoolVar(&conf.ShowVersion, "version", false, "Show Version Information")
 
 	fs.StringVar(&conf.DirToWatch, "watch_dir",
-		lookupEnvOrString("LIGHTMETER_WATCH_DIR", "", lookupenv),
+		envutil.LookupEnvOrString("LIGHTMETER_WATCH_DIR", "", lookupenv),
 		"Path to the directory where postfix stores its log files, to be watched")
 
 	fs.StringVar(&conf.Address, "listen",
-		lookupEnvOrString("LIGHTMETER_LISTEN", ":8080", lookupenv),
+		envutil.LookupEnvOrString("LIGHTMETER_LISTEN", ":8080", lookupenv),
 		"Network Address to listen to")
 
 	var stringLogLevel = "INFO"
 
 	fs.StringVar(&stringLogLevel, "log_level",
-		lookupEnvOrString("LIGHTMETER_LOG_LEVEL", "INFO", lookupenv),
+		envutil.LookupEnvOrString("LIGHTMETER_LOG_LEVEL", "INFO", lookupenv),
 		"Log level (DEBUG, INFO, WARN, or ERROR. Default: INFO)")
 
 	fs.StringVar(&conf.EmailToChange, "email_reset", "", "Change user info (email, name or password; depends on -workspace)")
@@ -111,23 +111,23 @@ func ParseWithErrorHandling(cmdlineArgs []string, lookupenv func(string) (string
 	fs.StringVar(&conf.ChangeUserInfoNewName, "new_user_name", "", "Update user name (depends on -email_reset)")
 
 	fs.StringVar(&conf.Socket, "logs_socket",
-		lookupEnvOrString("LIGHTMETER_LOGS_SOCKET", "", lookupenv),
+		envutil.LookupEnvOrString("LIGHTMETER_LOGS_SOCKET", "", lookupenv),
 		"Receive logs via a Socket. E.g. unix=/tmp/lightemter.sock or tcp=localhost:9999")
 
 	fs.StringVar(&conf.LogFormat, "log_format",
-		lookupEnvOrString("LIGHTMETER_LOG_FORMAT", "default", lookupenv),
+		envutil.LookupEnvOrString("LIGHTMETER_LOG_FORMAT", "default", lookupenv),
 		"Expected log format from external sources (like logstash, etc.)")
 
 	var unparsedDefaultSettings string
 
-	fs.StringVar(&unparsedDefaultSettings, "default_settings", lookupEnvOrString("LIGHTMETER_DEFAULT_SETTINGS", `{}`, lookupenv), "JSON string for default settings")
+	fs.StringVar(&unparsedDefaultSettings, "default_settings", envutil.LookupEnvOrString("LIGHTMETER_DEFAULT_SETTINGS", `{}`, lookupenv), "JSON string for default settings")
 
 	var unparsedLogPatterns string
 
-	fs.StringVar(&unparsedLogPatterns, "log_file_patterns", lookupEnvOrString("LIGHTMETER_LOG_FILE_PATTERNS", "", lookupenv),
+	fs.StringVar(&unparsedLogPatterns, "log_file_patterns", envutil.LookupEnvOrString("LIGHTMETER_LOG_FILE_PATTERNS", "", lookupenv),
 		`An optional colon separated list of the base filenames for the Postfix log files. Example: "mail.log:mail.err:mail.log" or "maillog"`)
 
-	proxyConf, err := lookupEnvOrBool("LIGHTMETER_I_KNOW_WHAT_I_AM_DOING_NOT_USING_A_REVERSE_PROXY", false, lookupenv)
+	proxyConf, err := envutil.LookupEnvOrBool("LIGHTMETER_I_KNOW_WHAT_I_AM_DOING_NOT_USING_A_REVERSE_PROXY", false, lookupenv)
 	if err != nil {
 		return conf, err
 	}
@@ -136,9 +136,9 @@ func ParseWithErrorHandling(cmdlineArgs []string, lookupenv func(string) (string
 		proxyConf, "Used when you are accessing the application without a reverse proxy (e.g. apache2, nginx or traefik), "+
 			"which is unsupported by us at the moment and might lead to security issues")
 
-	fs.StringVar(&conf.RegisteredUserEmail, "registered_user_email", lookupEnvOrString("LIGHTMETER_REGISTERED_USER_EMAIL", "", lookupenv), "Experimental: static user e-mail")
-	fs.StringVar(&conf.RegisteredUserName, "registered_user_name", lookupEnvOrString("LIGHTMETER_REGISTERED_USER_NAME", "", lookupenv), "Experimental: static user name")
-	fs.StringVar(&conf.RegisteredUserPassword, "registered_user_password", lookupEnvOrString("LIGHTMETER_REGISTERED_USER_PASSWORD", "", lookupenv), "Experimental: static user password")
+	fs.StringVar(&conf.RegisteredUserEmail, "registered_user_email", envutil.LookupEnvOrString("LIGHTMETER_REGISTERED_USER_EMAIL", "", lookupenv), "Experimental: static user e-mail")
+	fs.StringVar(&conf.RegisteredUserName, "registered_user_name", envutil.LookupEnvOrString("LIGHTMETER_REGISTERED_USER_NAME", "", lookupenv), "Experimental: static user name")
+	fs.StringVar(&conf.RegisteredUserPassword, "registered_user_password", envutil.LookupEnvOrString("LIGHTMETER_REGISTERED_USER_PASSWORD", "", lookupenv), "Experimental: static user password")
 
 	fs.BoolVar(&conf.GenerateDovecotConfig, "dovecot_conf_gen", false, "Generate Dovecot Configuration")
 	fs.BoolVar(&conf.DovecotConfigIsOld, "dovecot_conf_is_old", false, "Requires -dovecot_conf_gen. Use if if you're using a Dovecot older than 2.3.1")
@@ -179,37 +179,3 @@ func ParseWithErrorHandling(cmdlineArgs []string, lookupenv func(string) (string
 
 	return conf, nil
 }
-
-func lookupEnvOrString(key string, defaultVal string, loopkupenv func(string) (string, bool)) string {
-	if val, ok := loopkupenv(key); ok {
-		return val
-	}
-
-	return defaultVal
-}
-
-func lookupEnvOrBool(key string, defaultVal bool, loopkupenv func(string) (string, bool)) (bool, error) {
-	if val, ok := loopkupenv(key); ok {
-		v, err := strconv.ParseBool(val)
-		if err != nil {
-			return v, fmt.Errorf("Boolean env var %v boolean value could not be parsed: %w", key, err)
-		}
-
-		return v, nil
-	}
-
-	return defaultVal, nil
-}
-
-func lookupEnvOrInt(key string, defaultVal int64, loopkupenv func(string) (string, bool)) (int64, error) {
-	if val, ok := loopkupenv(key); ok {
-		v, err := strconv.ParseInt(val, 10, 32)
-		if err != nil {
-			return v, fmt.Errorf("Integer env var %v integer value could not be parsed: %w", key, err)
-		}
-
-		return v, nil
-	}
-
-	return defaultVal, nil
-}

deliverydb/cleaning.go

@@ -108,6 +108,15 @@ func tryToDeleteDeliveryQueue(tx *sql.Tx, deliveryId int64, stmts dbconn.TxPrepa
 	return nil
 }
 
+func tryToDeleteLogLinesRefs(tx *sql.Tx, deliveryId int64, stmts dbconn.TxPreparedStmts) error {
+	//nolint:sqlclosecheck
+	if _, err := stmts.Get(deleteLogLinesRefsByDeliveryId).Exec(deliveryId); err != nil {
+		return errorutil.Wrap(err)
+	}
+
+	return nil
+}
+
 func makeCleanAction(maxAge time.Duration, batchSize int) dbrunner.Action {
 	return func(tx *sql.Tx, stmts dbconn.TxPreparedStmts) (err error) {
 		// NOTE: the time in the database is in Seconds
@@ -144,6 +153,10 @@ func makeCleanAction(maxAge time.Duration, batchSize int) dbrunner.Action {
 				return errorutil.Wrap(err)
 			}
 
+			if err := tryToDeleteLogLinesRefs(tx, deliveryId, stmts); err != nil {
+				return errorutil.Wrap(err)
+			}
+
 			// TODO:
 			// maybe delete sender and recipient and orig_recipient domain parts,
 			// although they are very unlikely to grow too much over time

deliverydb/delivery.go

@@ -58,6 +58,9 @@ const (
 	countDeliveriesWithMessageId
 	deleteMessageIdById
 
+	insertLogLineRef
+	deleteLogLinesRefsByDeliveryId
+
 	lastStmtKey
 )
 
@@ -129,12 +132,14 @@ from
 	deliveries join delivery_queue on delivery_queue.delivery_id = deliveries.id
 where
 	delivery_queue.queue_id = ?`,
-	deleteDeliveryQueueById:       `delete from delivery_queue where id = ?`,
-	deleteExpiredQueuesByQueueId:  `delete from expired_queues where queue_id = ?`,
-	deleteQueueParentingByQueueId: `delete from queue_parenting where parent_queue_id = ? or child_queue_id = ?`,
-	deleteQueueById:               `delete from queues where id = ?`,
-	countDeliveriesWithMessageId:  `select count(*) from deliveries where message_id = ? and delivery_ts < ?`,
-	deleteMessageIdById:           `delete from messageids where id = ?`,
+	deleteDeliveryQueueById:        `delete from delivery_queue where id = ?`,
+	deleteExpiredQueuesByQueueId:   `delete from expired_queues where queue_id = ?`,
+	deleteQueueParentingByQueueId:  `delete from queue_parenting where parent_queue_id = ? or child_queue_id = ?`,
+	deleteQueueById:                `delete from queues where id = ?`,
+	countDeliveriesWithMessageId:   `select count(*) from deliveries where message_id = ? and delivery_ts < ?`,
+	deleteMessageIdById:            `delete from messageids where id = ?`,
+	insertLogLineRef:               `insert into log_lines_ref(delivery_id, ref_type, time, checksum) values(?, ?, ?, ?)`,
+	deleteLogLinesRefsByDeliveryId: `delete from log_lines_ref where delivery_id = ?`,
 }
 
 func setupDomainMapping(conn dbconn.RwConn, m *domainmapping.Mapper) error {
@@ -339,6 +344,16 @@ func insertMandatoryResultFields(tx *sql.Tx, stmts dbconn.TxPreparedStmts, tr tr
 		return nil, errorutil.Wrap(err)
 	}
 
+	deliveryId, err := result.LastInsertId()
+	if err != nil {
+		return nil, errorutil.Wrap(err)
+	}
+
+	//nolint:sqlclosecheck
+	if _, err := stmts.Get(insertLogLineRef).Exec(deliveryId, tracking.ResultDeliveryLineChecksum, tr[tracking.ResultDeliveryTimeKey].Int64(), tr[tracking.ResultDeliveryLineChecksum].Int64()); err != nil {
+		return nil, errorutil.Wrap(err)
+	}
+
 	return result, nil
 }