test: Add ProxySQL to our Docker development stack (#6754)

Add an upstream ProxySQL container to our docker-compose. Configure
ProxySQL to manage database connections for our unit and integration
tests.

Fixes #5873

.gitignore

@@ -39,3 +39,6 @@ tags
 .vscode/*
 .hierarchy/
 .softhsm-tokens/
+
+# ProxySQL log files
+test/proxysql/*.log*

docker-compose.yml

@@ -39,6 +39,7 @@ services:
       - 4003:4003 # OCSP
     depends_on:
       - bmysql
+      - bproxysql
       - bredis_1
       - bredis_2
       - bconsul
@@ -61,6 +62,22 @@ services:
     command: mysqld --bind-address=0.0.0.0 --slow-query-log --log-output=TABLE --log-queries-not-using-indexes=ON
     logging:
       driver: none
+
+  bproxysql:
+    image: proxysql/proxysql:2.4.4
+    # The --initial flag force resets the ProxySQL database on startup. By
+    # default, ProxySQL ignores new configuration if the database already
+    # exists. Without this flag, new configuration wouldn't be applied until you
+    # ran `docker compose down`.
+    entrypoint: proxysql -f --idle-threads -c /test/proxysql/proxysql.cnf --initial
+    volumes:
+      - ./test/:/test/:cached
+    depends_on:
+      - bmysql
+    networks:
+      bluenet:
+        aliases:
+          - boulder-proxysql
 
   bredis_1:
     image: redis:6.2.7

sa/database_test.go

@@ -59,7 +59,7 @@ func TestDbSettings(t *testing.T) {
 		ConnMaxLifetime: 100,
 		ConnMaxIdleTime: 100,
 	}
-	_, err := NewDbMap("sa@tcp(boulder-mysql:3306)/boulder_sa_integration", dbSettings)
+	_, err := NewDbMap("sa@tcp(boulder-proxysql:6033)/boulder_sa_integration", dbSettings)
 	if err != nil {
 		t.Errorf("connecting to DB: %s", err)
 	}
@@ -78,8 +78,8 @@ func TestDbSettings(t *testing.T) {
 }
 
 func TestNewDbMap(t *testing.T) {
-	const mysqlConnectURL = "policy:password@tcp(boulder-mysql:3306)/boulder_policy_integration?readTimeout=800ms&writeTimeout=800ms"
-	const expected = "policy:password@tcp(boulder-mysql:3306)/boulder_policy_integration?clientFoundRows=true&parseTime=true&readTimeout=800ms&writeTimeout=800ms&long_query_time=0.6400000000000001&max_statement_time=0.76&sql_mode=%27STRICT_ALL_TABLES%27"
+	const mysqlConnectURL = "policy:password@tcp(boulder-proxysql:6033)/boulder_policy_integration?readTimeout=800ms&writeTimeout=800ms"
+	const expected = "policy:password@tcp(boulder-proxysql:6033)/boulder_policy_integration?clientFoundRows=true&parseTime=true&readTimeout=800ms&writeTimeout=800ms&long_query_time=0.6400000000000001&max_statement_time=0.76&sql_mode=%27STRICT_ALL_TABLES%27"
 	oldSQLOpen := sqlOpen
 	defer func() {
 		sqlOpen = oldSQLOpen

sa/db-users/boulder_sa.sql

@@ -14,6 +14,7 @@ CREATE USER IF NOT EXISTS 'ocsp_update'@'localhost';
 CREATE USER IF NOT EXISTS 'ocsp_update_ro'@'localhost';
 CREATE USER IF NOT EXISTS 'test_setup'@'localhost';
 CREATE USER IF NOT EXISTS 'badkeyrevoker'@'localhost';
+CREATE USER IF NOT EXISTS 'proxysql'@'localhost';
 
 -- Storage Authority
 GRANT SELECT,INSERT ON certificates TO 'sa'@'localhost';
@@ -88,5 +89,8 @@ GRANT SELECT ON certificateStatus TO 'badkeyrevoker'@'localhost';
 GRANT SELECT ON precertificates TO 'badkeyrevoker'@'localhost';
 GRANT SELECT ON registrations TO 'badkeyrevoker'@'localhost';
 
+-- ProxySQL --
+GRANT ALL PRIVILEGES ON monitor TO 'proxysql'@'localhost';
+
 -- Test setup and teardown
 GRANT ALL PRIVILEGES ON * to 'test_setup'@'localhost';

sa/db/dbconfig.yml

@@ -1,20 +1,20 @@
 # https://github.com/rubenv/sql-migrate#readme
 boulder_sa_test:
   dialect: mysql
-  datasource: root@tcp(boulder-mysql:3306)/boulder_sa_test?parseTime=true
+  datasource: root@tcp(boulder-proxysql:6033)/boulder_sa_test?parseTime=true
   dir: boulder_sa
 
 boulder_sa_integration:
   dialect: mysql
-  datasource: root@tcp(boulder-mysql:3306)/boulder_sa_integration?parseTime=true
+  datasource: root@tcp(boulder-proxysql:6033)/boulder_sa_integration?parseTime=true
   dir: boulder_sa
 
 incidents_sa_test:
   dialect: mysql
-  datasource: root@tcp(boulder-mysql:3306)/incidents_sa_test?parseTime=true
+  datasource: root@tcp(boulder-proxysql:6033)/incidents_sa_test?parseTime=true
   dir: incidents_sa
 
 incidents_sa_integration:
   dialect: mysql
-  datasource: root@tcp(boulder-mysql:3306)/incidents_sa_integration?parseTime=true
+  datasource: root@tcp(boulder-proxysql:6033)/incidents_sa_integration?parseTime=true
   dir: incidents_sa

test/db.go

@@ -40,7 +40,7 @@ func ResetIncidentsTestDatabase(t testing.TB) func() {
 }
 
 func resetTestDatabase(t testing.TB, dbPrefix string) func() {
-	db, err := sql.Open("mysql", fmt.Sprintf("test_setup@tcp(boulder-mysql:3306)/%s_sa_test", dbPrefix))
+	db, err := sql.Open("mysql", fmt.Sprintf("test_setup@tcp(boulder-proxysql:6033)/%s_sa_test", dbPrefix))
 	if err != nil {
 		t.Fatalf("Couldn't create db: %s", err)
 	}

test/integration-test.py

@@ -165,7 +165,7 @@ def check_slow_queries():
         \G
     """
     output = subprocess.check_output(
-      ["mysql", "-h", "boulder-mysql", "-e", query],
+      ["mysql", "-h", "boulder-proxysql", "-e", query],
       stderr=subprocess.STDOUT).decode()
     if len(output) > 0:
         print(output)

test/proxysql/README.md

@@ -0,0 +1,77 @@
+# ProxySQL in Boulder
+
+In an effort to keep Boulder's development environment reasonably close to
+production we use ProxySQL in our Docker stack to proxy connections to our
+MariaDB database.
+
+## Ports
+
+ProxySQL listens on the following ports:
+  - `6033` Proxy MySQL Interface
+  - `6032` Admin MySQL Interface
+  - `6080` Admin Web Interface
+
+## Accessing the Admin MySQL Interface
+
+```bash
+mysql -uradmin -pradmin -h 127.0.0.1 --port 6032
+```
+
+### MacOS
+
+You will need to bind the port in `docker-compose.yml`, like so:
+
+```yaml
+  bproxysql:
+    ports:
+      - 6032:6032
+```
+
+## Accessing the Admin Web Interface
+
+You can access the ProxySQL web UI at https://127.0.0.1:6080. The default
+username/ password are `stats`/ `stats`.
+
+### MacOS
+
+You will need to bind the port in `docker-compose.yml`, like so:
+
+```yaml
+  bproxysql:
+    ports:
+      - 6080:6080
+```
+
+## Sending queries to a file
+
+To log all queries routed through the ProxySQL query parser, uncomment the
+following line in the `mysql_variables` section of `test/proxysql/proxysql.cnf`,
+like so:
+
+```ini
+# If mysql_query_rules are marked log=1, they will be logged here. If unset,
+# no queries are logged.
+eventslog_filename="/test/proxysql/events.log"
+```
+
+Then set `log = 1;` for `rule_id = 1;` in the `mysql_query_rules` section, like so:
+
+```
+{
+    rule_id = 1;
+    active = 1;
+    # Log all queries.
+    match_digest = ".";
+    # Set log=1 to log all queries to the eventslog_filename under
+    # mysql_variables.
+    log = 1;
+    apply = 0;
+},
+```
+
+## Sending ProxySQL logs to a file
+
+Replace the `entrypoint:` under `bproxysql` in `docker-compose.yml` with
+`/test/proxysql/entrypoint.sh`. This is necessary because if you attempt to run
+ProxySQL in the background (by removing the `-f` flag) Docker will simply kill
+the container.

test/proxysql/entrypoint.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+exec proxysql -f --idle-threads -c /test/proxysql/proxysql.cnf --initial 2>&1 | tee -a /test/proxysql/proxysql.log

test/proxysql/proxysql.cnf

@@ -0,0 +1,159 @@
+datadir = "/var/lib/proxysql";
+errorlog = "/test/proxysql/proxysql.log";
+admin_variables =
+{
+	# https://proxysql.com/documentation/global-variables/admin-variables Note
+	# that while admin variables are documented with an 'admin-' prefix, they
+	# are specified in the configuration with the prefix stripped.
+	mysql_ifaces = "0.0.0.0:6032";
+	# admin:admin is only used for local connections. For remote connections,
+	# use radmin:radmin
+	admin_credentials = "admin:admin;radmin:radmin";
+	web_enabled = "true";
+	# Web UI is disabled by default.
+	web_port = 6080;
+	# These are the credentials used for the web interface.
+	stats_credentials = "stats:stats";
+	debug = True;
+};
+mysql_variables =
+{
+	threads = 4;
+	max_connections = 10240;
+	have_compress = True;
+	poll_timeout = 2000;
+	interfaces = "0.0.0.0:6033";
+	stacksize = 1048576;
+	max_allowed_packet = 16777216;
+	# Allow up to 20 seconds to find a server, to limit how many failures
+	# Boulder sees when we do a primary swap
+	connect_timeout_server = 20000;
+	connect_timeout_server_max = 20000;
+	monitor_username = "proxysql";
+	monitor_password = "";
+	monitor_history = 600000;
+	monitor_connect_interval = 60000;
+	monitor_ping_interval = 10000;
+	monitor_read_only_interval = 1000;
+	monitor_read_only_timeout = 500;
+	monitor_writer_is_also_reader = False;
+	commands_stats = True;
+	sessions_sort = True;
+	connect_retries_on_failure = 10;
+	# Keep 90% of configured connections open.
+	free_connections_pct = 90;
+	connection_warming = True;
+	# If mysql_query_rules are marked log=1, they will be logged here. If unset,
+	# no queries are logged.
+	# eventslog_filename="/test/proxysql/events.log"
+	eventslog_filesize = 104857600;
+	eventslog_default_log = 1;
+	# The audit logs, if unset, are not logged. If set, every connection gets
+	# logged. Given Boulder's connection strategy, this can be noisy.
+	# auditlog_filename="/test/proxysql/audit.log"
+	auditlog_filesize = 104857600;
+};
+mysql_servers =
+(
+	{
+		address = "boulder-mysql";
+		port = 3306;
+		hostgroup = 0;
+		max_connections = 100;
+		max_latency_ms = 200;
+	}
+);
+mysql_users =
+(
+	{
+		username = "root";
+	},
+	{
+		username = "policy";
+	},
+	{
+		username = "sa";
+	},
+	{
+		username = "sa_ro";
+	},
+	{
+		username = "ocsp_resp";
+	},
+	{
+		username = "revoker";
+	},
+	{
+		username = "importer";
+	},
+	{
+		username = "mailer";
+	},
+	{
+		username = "cert_checker";
+	},
+	{
+		username = "ocsp_update";
+	},
+	{
+		username = "ocsp_update_ro";
+	},
+	{
+		username = "test_setup";
+	},
+	{
+		username = "badkeyrevoker";
+	},
+	{
+		username = "incidents_sa";
+	}
+);
+mysql_query_rules =
+(
+	{
+		rule_id = 1;
+		active = 1;
+		match_digest = ".";
+		log = 0;
+		apply = 0;
+	},
+	{
+		rule_id = 10;
+		username = "sa";
+		timeout = 4900;
+	},
+	{
+		rule_id = 11;
+		username = "sa_ro";
+		timeout = 4900;
+	},
+	{
+		rule_id = 14;
+		username = "ocsp_update";
+		timeout = 600000;
+	},
+	{
+		rule_id = 15;
+		username = "ocsp_update_ro";
+		timeout = 60000;
+	},
+	{
+		rule_id = 16;
+		username = "badkeyrevoker";
+		timeout = 3600000;
+	},
+	{
+		rule_id = 17;
+		username = "mailer";
+		timeout = 1800000;
+	},
+	{
+		rule_id = 18;
+		username = "ocsp_resp";
+		timeout = 4900;
+	}
+);
+scheduler =
+(
+
+);

test/secrets/backfiller_dburl

@@ -1 +1 @@
-sa@tcp(boulder-mysql:3306)/boulder_sa_integration
+sa@tcp(boulder-proxysql:6033)/boulder_sa_integration

test/secrets/badkeyrevoker_dburl

@@ -1 +1 @@
-badkeyrevoker@tcp(boulder-mysql:3306)/boulder_sa_integration
+badkeyrevoker@tcp(boulder-proxysql:6033)/boulder_sa_integration

test/secrets/cert_checker_dburl

@@ -1 +1 @@
-cert_checker@tcp(boulder-mysql:3306)/boulder_sa_integration
+cert_checker@tcp(boulder-proxysql:6033)/boulder_sa_integration

test/secrets/expiration_mailer_dburl

@@ -1 +1 @@
-mailer@tcp(boulder-mysql:3306)/boulder_sa_integration
+mailer@tcp(boulder-proxysql:6033)/boulder_sa_integration

test/secrets/incidents_dburl

@@ -1 +1 @@
-incidents_sa@tcp(boulder-mysql:3306)/incidents_sa_integration?readTimeout=14s&timeout=1s
+incidents_sa@tcp(boulder-proxysql:6033)/incidents_sa_integration?readTimeout=14s&timeout=1s

test/secrets/mailer_dburl

@@ -1 +1 @@
-mailer@tcp(boulder-mysql:3306)/boulder_sa_integration
+mailer@tcp(boulder-proxysql:6033)/boulder_sa_integration

test/secrets/ocsp_responder_dburl

@@ -1 +1 @@
-ocsp_resp@tcp(boulder-mysql:3306)/boulder_sa_integration?readTimeout=800ms&writeTimeout=800ms&timeout=100ms
+ocsp_resp@tcp(boulder-proxysql:6033)/boulder_sa_integration?readTimeout=800ms&writeTimeout=800ms&timeout=100ms

test/secrets/ocsp_updater_dburl

@@ -1 +1 @@
-ocsp_update@tcp(boulder-mysql:3306)/boulder_sa_integration?readTimeout=800ms&writeTimeout=800ms&timeout=100ms
+ocsp_update@tcp(boulder-proxysql:6033)/boulder_sa_integration?readTimeout=800ms&writeTimeout=800ms&timeout=100ms

test/secrets/ocsp_updater_ro_dburl

@@ -1 +1 @@
-ocsp_update_ro@tcp(boulder-mysql:3306)/boulder_sa_integration?readTimeout=800ms&writeTimeout=800ms&timeout=100ms
+ocsp_update_ro@tcp(boulder-proxysql:6033)/boulder_sa_integration?readTimeout=800ms&writeTimeout=800ms&timeout=100ms

test/secrets/purger_dburl

@@ -1 +1 @@
-purger@tcp(boulder-mysql:3306)/boulder_sa_integration
+purger@tcp(boulder-proxysql:6033)/boulder_sa_integration

test/secrets/revoker_dburl

@@ -1 +1 @@
-revoker@tcp(boulder-mysql:3306)/boulder_sa_integration
+revoker@tcp(boulder-proxysql:6033)/boulder_sa_integration

test/secrets/sa_dburl

@@ -1 +1 @@
-sa@tcp(boulder-mysql:3306)/boulder_sa_integration?readTimeout=14s&writeTimeout=14s&timeout=1s
+sa@tcp(boulder-proxysql:6033)/boulder_sa_integration?readTimeout=14s&writeTimeout=14s&timeout=1s

test/secrets/sa_ro_dburl

@@ -1 +1 @@
-sa_ro@tcp(boulder-mysql:3306)/boulder_sa_integration?readTimeout=14s&writeTimeout=14s&timeout=1s
+sa_ro@tcp(boulder-proxysql:6033)/boulder_sa_integration?readTimeout=14s&writeTimeout=14s&timeout=1s