add visitor id to custom events and pageviews

cmd/addevents/app.go

@@ -57,7 +57,8 @@ func (a App) AddPageviewsEvents() {
 					randomBrowser(),
 					"benchbot",
 					randomReferrerDomain(),
-					"XX",
+					randomCountryCode(),
+					fmt.Sprintf("prisme_%X", rand.Uint64()%uint64(a.cfg.BatchSize*a.cfg.BatchCount/3)),
 				)
 				if err != nil {
 					panic(err)
@@ -109,7 +110,8 @@ func (a App) AddCustomEvents() {
 					randomBrowser(),
 					"benchbot",
 					randomReferrerDomain(),
-					"XX",
+					randomCountryCode(),
+					fmt.Sprintf("prisme_%X", rand.Uint64()%uint64(a.cfg.BatchSize*a.cfg.BatchCount/3)),
 					name,
 					keys,
 					values,
@@ -160,3 +162,30 @@ func randomCustomEvent() (string, []string, []string) {
 		panic("not implemented")
 	}
 }
+
+var countryCodes = []string{
+	"AF", "AX", "AL", "DZ", "AS", "AD", "AO", "AI", "AQ", "AG", "AR", "AM", "AW",
+	"AU", "AT", "AZ", "BS", "BH", "BD", "BB", "BY", "BE", "BZ", "BJ", "BM", "BT",
+	"BO", "BQ", "BA", "BW", "BV", "BR", "IO", "BN", "BG", "BF", "BI", "CV",
+	"KH", "CM", "CA", "KY", "CF", "TD", "CL", "CN", "CX", "CC", "CO", "KM", "CD",
+	"CG", "CK", "CR", "CI", "HR", "CU", "CW", "CY", "CZ", "DK", "DJ", "DM", "DO",
+	"EC", "EG", "SV", "GQ", "ER", "EE", "SZ", "ET", "FK", "FO", "FJ", "FI", "FR",
+	"GF", "PF", "TF", "GA", "GM", "GE", "DE", "GH", "GI", "GR", "GL", "GD", "GP",
+	"GU", "GT", "GG", "GN", "GW", "GY", "HT", "HM", "VA", "HN", "HK", "HU", "IS",
+	"IN", "ID", "IR", "IQ", "IE", "IM", "IL", "IT", "JM", "JP", "JE", "JO", "KZ",
+	"KE", "KI", "KP", "KR", "KW", "KG", "LA", "LV", "LB", "LS", "LR", "LY", "LI",
+	"LT", "LU", "MO", "MG", "MW", "MY", "MV", "ML", "MT", "MH", "MQ", "MR", "MU",
+	"YT", "MX", "FM", "MD", "MC", "MN", "ME", "MS", "MA", "MZ", "MM", "NA", "NR",
+	"NP", "NL", "NC", "NZ", "NI", "NE", "NG", "NU", "NF", "MK", "MP", "NO", "OM",
+	"PK", "PW", "PS", "PA", "PG", "PY", "PE", "PH", "PN", "PL", "PT", "PR", "QA",
+	"RE", "RO", "RU", "RW", "BL", "SH", "KN", "LC", "MF", "PM", "VC", "WS", "SM",
+	"ST", "SA", "SN", "RS", "SC", "SL", "SG", "SX", "SK", "SI", "SB", "SO", "ZA",
+	"GS", "SS", "ES", "LK", "SD", "SR", "SJ", "SE", "CH", "SY", "TW", "TJ", "TZ",
+	"TH", "TL", "TG", "TK", "TO", "TT", "TN", "TR", "TM", "TC", "TV", "UG", "UA",
+	"AE", "GB", "UM", "US", "UY", "UZ", "VU", "VE", "VN", "VG", "VI", "WF", "EH",
+	"YE", "ZM", "ZW", "XX",
+}
+
+func randomCountryCode() string {
+	return randomItem(countryCodes)
+}

cmd/server/full/wire.go

@@ -13,6 +13,7 @@ import (
 	"github.com/prismelabs/analytics/pkg/services/grafana"
 	"github.com/prismelabs/analytics/pkg/services/ipgeolocator"
 	"github.com/prismelabs/analytics/pkg/services/originregistry"
+	"github.com/prismelabs/analytics/pkg/services/saltmanager"
 	"github.com/prismelabs/analytics/pkg/services/teardown"
 	"github.com/prismelabs/analytics/pkg/services/uaparser"
 	"github.com/prismelabs/analytics/pkg/wired"
@@ -39,6 +40,7 @@ func Initialize(logger wired.BootstrapLogger) wired.App {
 		middlewares.ProvideRequestId,
 		middlewares.ProvideStatic,
 		originregistry.ProvideEnvVarService,
+		saltmanager.ProvideService,
 		teardown.ProvideService,
 		uaparser.ProvideService,
 		wired.ProvideApp,

cmd/server/ingestion/wire.go

@@ -11,6 +11,7 @@ import (
 	"github.com/prismelabs/analytics/pkg/services/eventstore"
 	"github.com/prismelabs/analytics/pkg/services/ipgeolocator"
 	"github.com/prismelabs/analytics/pkg/services/originregistry"
+	"github.com/prismelabs/analytics/pkg/services/saltmanager"
 	"github.com/prismelabs/analytics/pkg/services/teardown"
 	"github.com/prismelabs/analytics/pkg/services/uaparser"
 	"github.com/prismelabs/analytics/pkg/wired"
@@ -34,6 +35,7 @@ func Initialize(logger wired.BootstrapLogger) wired.App {
 		middlewares.ProvideRequestId,
 		middlewares.ProvideStatic,
 		originregistry.ProvideEnvVarService,
+		saltmanager.ProvideService,
 		teardown.ProvideService,
 		uaparser.ProvideService,
 		wired.ProvideApp,

flake.nix

@@ -31,7 +31,7 @@
                 ];
               };
             };
-            packages = rec {
+            packages = {
               default = pkgs.buildGoModule {
                 pname = "prisme";
                 version = "0.14.0";

pkg/embedded/ch_migrations/20240417104420_add_pageviews_visitor_id.up.sql

@@ -0,0 +1 @@
+ALTER TABLE prisme.events_pageviews ADD COLUMN visitor_id String DEFAULT 'prisme_00';

pkg/embedded/ch_migrations/20240417104421_BREAK_recreate_events_pageviews_table.up.sql

@@ -0,0 +1,28 @@
+RENAME TABLE events_pageviews TO events_pageviews_old;
+
+-- Add visitor_id to primary key so it can be used for sampling.
+
+-- It is beneficial for queries to order the primary key columns by cardinality in ascending order.
+-- https://clickhouse.com/docs/en/optimize/sparse-primary-indexes#summary
+CREATE TABLE events_pageviews (
+	timestamp DateTime('UTC'),
+	domain String,
+	path String,
+	operating_system LowCardinality(String),
+	browser_family LowCardinality(String),
+	device LowCardinality(String),
+	referrer_domain String,
+	country_code LowCardinality(String),
+	visitor_id String
+)
+ENGINE = MergeTree
+PRIMARY KEY (domain, path, toDate(timestamp), xxh3(visitor_id))
+ORDER BY (domain, path, toDate(timestamp), xxh3(visitor_id), operating_system, browser_family, device, referrer_domain, country_code)
+SAMPLE BY xxh3(visitor_id)
+PARTITION BY toYYYYMM(timestamp);
+
+-- Move rows to new table.
+INSERT INTO events_pageviews SELECT * FROM events_pageviews_old;
+
+-- Delete old table.
+DROP TABLE events_pageviews_old;