Merge pull request MaterializeInc#24047 from philip-stoev/platform-ch…

…ecks-persist-txn

persist-txn tests in Platform Checks

ci/nightly/pipeline.template.yml

@@ -668,6 +668,28 @@ steps:
               composition: platform-checks
               args: [--scenario=DropCreateDefaultReplica, "--seed=$BUILDKITE_JOB_ID"]
 
+      - id: checks-persist-txn-toggle
+        label: "Checks + Toggle persist_txn"
+        timeout_in_minutes: 45
+        artifact_paths: junit_*.xml
+        agents:
+          queue: builder-linux-x86_64
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: platform-checks
+              args: [--scenario=PersistTxnToggle, "--seed=$BUILDKITE_JOB_ID"]
+
+      - id: checks-persist-txn-fending
+        label: "Checks + persist_txn fencing"
+        timeout_in_minutes: 45
+        artifact_paths: junit_*.xml
+        agents:
+          queue: builder-linux-x86_64
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: platform-checks
+              args: [--scenario=PersistTxnFencing, "--seed=$BUILDKITE_JOB_ID"]
+
       - id: checks-upgrade-matrix
         label: "Random upgrades over the entire matrix"
         timeout_in_minutes: 600

misc/images/materialized/entrypoint.sh

@@ -28,19 +28,21 @@ hosted offering we run these services scaled across many machines.
 ********************************* WARNING ********************************
 EOF
 
-COCKROACH_SKIP_ENABLING_DIAGNOSTIC_REPORTING=true cockroach start-single-node \
-    --insecure \
-    --background \
-    --store=/mzdata/cockroach
+if [ -z "${MZ_NO_BUILTIN_COCKROACH:-}" ]; then
+  COCKROACH_SKIP_ENABLING_DIAGNOSTIC_REPORTING=true cockroach start-single-node \
+      --insecure \
+      --background \
+      --store=/mzdata/cockroach
 
-# See: https://github.com/cockroachdb/cockroach/issues/93892
-# See: https://github.com/MaterializeInc/materialize/issues/16726
-cockroach sql --insecure -e "SET CLUSTER SETTING sql.stats.forecasts.enabled = false"
+  # See: https://github.com/cockroachdb/cockroach/issues/93892
+  # See: https://github.com/MaterializeInc/materialize/issues/16726
+  cockroach sql --insecure -e "SET CLUSTER SETTING sql.stats.forecasts.enabled = false"
 
-cockroach sql --insecure -e "CREATE SCHEMA IF NOT EXISTS consensus"
-cockroach sql --insecure -e "CREATE SCHEMA IF NOT EXISTS storage"
-cockroach sql --insecure -e "CREATE SCHEMA IF NOT EXISTS adapter"
-cockroach sql --insecure -e "CREATE SCHEMA IF NOT EXISTS tsoracle"
+  cockroach sql --insecure -e "CREATE SCHEMA IF NOT EXISTS consensus"
+  cockroach sql --insecure -e "CREATE SCHEMA IF NOT EXISTS storage"
+  cockroach sql --insecure -e "CREATE SCHEMA IF NOT EXISTS adapter"
+  cockroach sql --insecure -e "CREATE SCHEMA IF NOT EXISTS tsoracle"
+fi
 
 if [[ ! -f /mzdata/environment-id ]]; then
   echo "docker-container-$(cat /proc/sys/kernel/random/uuid)-0" > /mzdata/environment-id

misc/python/materialize/checks/actions.py

@@ -19,6 +19,10 @@
 
 
 class Action:
+    def __init__(self) -> None:
+        self.mz_service = None
+        self.phase = None
+
     def execute(self, e: Executor) -> None:
         assert False
 
@@ -36,9 +40,9 @@ def __init__(self, input: str, dedent: bool = True) -> None:
         self.handle: Any | None = None
         self.caller = getframeinfo(stack()[1][0])
 
-    def execute(self, e: Executor) -> None:
+    def execute(self, e: Executor, mz_service: str | None = None) -> None:
         """Pass testdrive actions to be run by an Executor-specific implementation."""
-        self.handle = e.testdrive(self.input, self.caller)
+        self.handle = e.testdrive(self.input, self.caller, mz_service)
 
     def join(self, e: Executor) -> None:
         e.join(self.handle)
@@ -57,13 +61,14 @@ def join(self, e: Executor) -> None:
 
 
 class Initialize(Action):
-    def __init__(self, scenario: "Scenario") -> None:
+    def __init__(self, scenario: "Scenario", mz_service: str | None = None) -> None:
         self.checks = scenario.check_objects
+        self.mz_service = mz_service
 
     def execute(self, e: Executor) -> None:
         for check in self.checks:
             print(f"Running initialize() from {check}")
-            check.start_initialize(e)
+            check.start_initialize(e, self)
 
     def join(self, e: Executor) -> None:
         for check in self.checks:
@@ -75,32 +80,35 @@ def __init__(
         self,
         scenario: "Scenario",
         phase: int | None = None,
+        mz_service: str | None = None,
     ) -> None:
         assert phase is not None
         self.phase = phase - 1
+        self.mz_service = mz_service
 
         self.checks = scenario.check_objects
 
     def execute(self, e: Executor) -> None:
         assert self.phase is not None
         for check in self.checks:
             print(f"Running manipulate() from {check}")
-            check.start_manipulate(e, self.phase)
+            check.start_manipulate(e, self)
 
     def join(self, e: Executor) -> None:
         assert self.phase is not None
         for check in self.checks:
-            check.join_manipulate(e, self.phase)
+            check.join_manipulate(e, self)
 
 
 class Validate(Action):
-    def __init__(self, scenario: "Scenario") -> None:
+    def __init__(self, scenario: "Scenario", mz_service: str | None = None) -> None:
         self.checks = scenario.check_objects
+        self.mz_service = mz_service
 
     def execute(self, e: Executor) -> None:
         for check in self.checks:
             print(f"Running validate() from {check}")
-            check.start_validate(e)
+            check.start_validate(e, self)
 
     def join(self, e: Executor) -> None:
         for check in self.checks:

misc/python/materialize/checks/all_checks/alter_index.py

@@ -67,11 +67,11 @@ def manipulate(self) -> list[Testdrive]:
                 # When upgrading from old version without roles the indexes are
                 # owned by default_role, thus we have to change the owner
                 # before altering them:
-                $[version>=4700] postgres-execute connection=postgres://mz_system:materialize@materialized:6877
+                $[version>=4700] postgres-execute connection=postgres://mz_system:materialize@${testdrive.materialize-internal-sql-addr}
                 ALTER INDEX alter_index_table_primary_idx OWNER TO materialize;
                 ALTER INDEX alter_index_source_primary_idx OWNER TO materialize;
 
-                $[version>=5500] postgres-execute connection=postgres://mz_system:materialize@materialized:6877
+                $[version>=5500] postgres-execute connection=postgres://mz_system:materialize@${testdrive.materialize-internal-sql-addr}
                 ALTER SYSTEM SET enable_index_options = true
                 ALTER SYSTEM SET enable_logical_compaction_window = true
 

misc/python/materialize/checks/all_checks/cluster.py

@@ -19,19 +19,19 @@ def manipulate(self) -> list[Testdrive]:
             Testdrive(dedent(s))
             for s in [
                 """
-                $[version>=5900] postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+                $[version>=5900] postgres-execute connection=postgres://mz_system@${testdrive.materialize-internal-sql-addr}
                 GRANT CREATECLUSTER ON SYSTEM TO materialize
 
-                $[version<5900] postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+                $[version<5900] postgres-execute connection=postgres://mz_system@${testdrive.materialize-internal-sql-addr}
                 ALTER ROLE materialize CREATECLUSTER
 
                 > CREATE CLUSTER create_cluster1 REPLICAS (replica1 (SIZE '2-2'));
                 """,
                 """
-                $[version>=5900] postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+                $[version>=5900] postgres-execute connection=postgres://mz_system@${testdrive.materialize-internal-sql-addr}
                 GRANT CREATECLUSTER ON SYSTEM TO materialize
 
-                $[version<5900] postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+                $[version<5900] postgres-execute connection=postgres://mz_system@${testdrive.materialize-internal-sql-addr}
                 ALTER ROLE materialize CREATECLUSTER
 
                 > CREATE CLUSTER create_cluster2 REPLICAS (replica1 (SIZE '2-2'));

misc/python/materialize/checks/all_checks/databases.py

@@ -18,10 +18,10 @@ def manipulate(self) -> list[Testdrive]:
             Testdrive(dedent(s))
             for s in [
                 """
-                $[version>=5900] postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+                $[version>=5900] postgres-execute connection=postgres://mz_system@${testdrive.materialize-internal-sql-addr}
                 GRANT CREATEDB ON SYSTEM TO materialize
 
-                $[version<5900] postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+                $[version<5900] postgres-execute connection=postgres://mz_system@${testdrive.materialize-internal-sql-addr}
                 ALTER ROLE materialize CREATEDB
 
                 > CREATE DATABASE to_be_created1;
@@ -30,10 +30,10 @@ def manipulate(self) -> list[Testdrive]:
                 > INSERT INTO t1 VALUES (1);
                 """,
                 """
-                $[version>=5900] postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+                $[version>=5900] postgres-execute connection=postgres://mz_system@${testdrive.materialize-internal-sql-addr}
                 GRANT CREATEDB ON SYSTEM TO materialize
 
-                $[version<5900] postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+                $[version<5900] postgres-execute connection=postgres://mz_system@${testdrive.materialize-internal-sql-addr}
                 ALTER ROLE materialize CREATEDB
 
                 > CREATE DATABASE to_be_created2;
@@ -90,7 +90,7 @@ def manipulate(self) -> list[Testdrive]:
                 # When upgrading from old version without roles the database is
                 # owned by default_role, thus we have to change the owner
                 # before dropping it:
-                $[version>=4700] postgres-execute connection=postgres://mz_system:materialize@materialized:6877
+                $[version>=4700] postgres-execute connection=postgres://mz_system:materialize@${testdrive.materialize-internal-sql-addr}
                 ALTER DATABASE to_be_dropped OWNER TO materialize;
 
                 > DROP DATABASE to_be_dropped CASCADE;

misc/python/materialize/checks/all_checks/default_privileges.py

@@ -29,7 +29,7 @@ def initialize(self) -> Testdrive:
             > CREATE ROLE defpriv_role1
             >[version<5900] ALTER ROLE defpriv_role1 CREATEDB CREATECLUSTER
 
-            $[version>=5900] postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+            $[version>=5900] postgres-execute connection=postgres://mz_system@${testdrive.materialize-internal-sql-addr}
             GRANT CREATEDB, CREATECLUSTER ON SYSTEM TO defpriv_role1
 
             > CREATE TABLE defpriv_table1 (c int)
@@ -48,7 +48,7 @@ def manipulate(self) -> list[Testdrive]:
                 > CREATE ROLE defpriv_role2
                 >[version<5900] ALTER ROLE defpriv_role2 CREATEDB CREATECLUSTER
 
-                $[version>=5900] postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+                $[version>=5900] postgres-execute connection=postgres://mz_system@${testdrive.materialize-internal-sql-addr}
                 GRANT CREATEDB, CREATECLUSTER ON SYSTEM TO defpriv_role2
 
                 > CREATE TABLE defpriv_table2 (c int)
@@ -60,7 +60,7 @@ def manipulate(self) -> list[Testdrive]:
                 > CREATE ROLE defpriv_role3
                 >[version<5900] ALTER ROLE defpriv_role3 CREATEDB CREATECLUSTER
 
-                $[version>=5900] postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+                $[version>=5900] postgres-execute connection=postgres://mz_system@${testdrive.materialize-internal-sql-addr}
                 GRANT CREATEDB, CREATECLUSTER ON SYSTEM TO defpriv_role3
 
                 > CREATE TABLE defpriv_table3 (c int)

misc/python/materialize/checks/all_checks/drop_index.py

@@ -43,7 +43,7 @@ def manipulate(self) -> list[Testdrive]:
                 # When upgrading from old version without roles the indexes are
                 # owned by default_role, thus we have to change the owner
                 # before dropping them:
-                $[version>=4700] postgres-execute connection=postgres://mz_system:materialize@materialized:6877
+                $[version>=4700] postgres-execute connection=postgres://mz_system:materialize@${testdrive.materialize-internal-sql-addr}
                 ALTER INDEX drop_index_table_primary_idx OWNER TO materialize;
                 ALTER INDEX drop_index_index2 OWNER TO materialize;
 

misc/python/materialize/checks/all_checks/drop_table.py

@@ -37,7 +37,7 @@ def manipulate(self) -> list[Testdrive]:
                 # When upgrading from old version without roles the table is
                 # owned by default_role, thus we have to change the owner
                 # before dropping it:
-                $[version>=4700] postgres-execute connection=postgres://mz_system:materialize@materialized:6877
+                $[version>=4700] postgres-execute connection=postgres://mz_system:materialize@${testdrive.materialize-internal-sql-addr}
                 ALTER TABLE drop_table2 OWNER TO materialize;
 
                 > DROP TABLE drop_table2;

misc/python/materialize/checks/all_checks/owners.py

@@ -18,15 +18,15 @@ class Owners(Check):
     def _create_objects(self, role: str, i: int, expensive: bool = False) -> str:
         s = dedent(
             f"""
-            $[version>=5200] postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+            $[version>=5200] postgres-execute connection=postgres://mz_system@${{testdrive.materialize-internal-sql-addr}}
             GRANT CREATE ON DATABASE materialize TO {role}
             GRANT CREATE ON SCHEMA materialize.public TO {role}
             GRANT CREATE ON CLUSTER default TO {role}
-            $[version>=5900] postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+            $[version>=5900] postgres-execute connection=postgres://mz_system@${{testdrive.materialize-internal-sql-addr}}
             GRANT CREATEDB ON SYSTEM TO {role}
-            $[version<5900] postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+            $[version<5900] postgres-execute connection=postgres://mz_system@${{testdrive.materialize-internal-sql-addr}}
             ALTER ROLE {role} CREATEDB
-            $ postgres-execute connection=postgres://{role}@materialized:6875/materialize
+            $ postgres-execute connection=postgres://{role}@${{testdrive.materialize-sql-addr}}
             CREATE DATABASE owner_db{i}
             CREATE SCHEMA owner_schema{i}
             CREATE CONNECTION owner_kafka_conn{i} FOR KAFKA {self._kafka_broker()}
@@ -53,7 +53,7 @@ def _create_objects(self, role: str, i: int, expensive: bool = False) -> str:
     def _alter_object_owners(self, i: int, expensive: bool = False) -> str:
         s = dedent(
             f"""
-            $ postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+            $ postgres-execute connection=postgres://mz_system@${{testdrive.materialize-internal-sql-addr}}
             ALTER DATABASE owner_db{i} OWNER TO other_owner
             ALTER SCHEMA owner_schema{i} OWNER TO other_owner
             ALTER CONNECTION owner_kafka_conn{i} OWNER TO other_owner
@@ -102,7 +102,7 @@ def _drop_objects(
         ]
         if success:
             return (
-                f"$ postgres-execute connection=postgres://{role}@materialized:6875/materialize\n"
+                f"$ postgres-execute connection=postgres://{role}@${{testdrive.materialize-sql-addr}}\n"
                 + "\n".join(cmds)
                 + "\n"
             )
@@ -122,16 +122,16 @@ def initialize(self) -> Testdrive:
         return Testdrive(
             dedent(
                 """
-                $[version>=5900] postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+                $[version>=5900] postgres-execute connection=postgres://mz_system@${testdrive.materialize-internal-sql-addr}
                 GRANT CREATEROLE ON SYSTEM TO materialize
 
-                $[version<5900] postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+                $[version<5900] postgres-execute connection=postgres://mz_system@${testdrive.materialize-internal-sql-addr}
                 ALTER ROLE materialize CREATEROLE
 
                 > CREATE ROLE owner_role_01
                 >[version<5900] ALTER ROLE owner_role_01 CREATEDB CREATECLUSTER
 
-                $[version>=5900] postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+                $[version>=5900] postgres-execute connection=postgres://mz_system@${testdrive.materialize-internal-sql-addr}
                 GRANT CREATEDB, CREATECLUSTER ON SYSTEM TO owner_role_01
 
                 > CREATE ROLE other_owner
@@ -151,16 +151,16 @@ def manipulate(self) -> list[Testdrive]:
                 + self._alter_object_owners(4)
                 + dedent(
                     """
-                    $[version>=5900] postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+                    $[version>=5900] postgres-execute connection=postgres://mz_system@${testdrive.materialize-internal-sql-addr}
                     GRANT CREATEROLE ON SYSTEM TO materialize
 
-                    $[version<5900] postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+                    $[version<5900] postgres-execute connection=postgres://mz_system@${testdrive.materialize-internal-sql-addr}
                     ALTER ROLE materialize CREATEROLE
 
                     > CREATE ROLE owner_role_02
                     >[version<5900] ALTER ROLE owner_role_02 CREATEDB CREATECLUSTER
 
-                    $[version>=5900] postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+                    $[version>=5900] postgres-execute connection=postgres://mz_system@${testdrive.materialize-internal-sql-addr}
                     GRANT CREATEDB, CREATECLUSTER ON SYSTEM TO owner_role_02
                     """
                 ),
@@ -172,16 +172,16 @@ def manipulate(self) -> list[Testdrive]:
                 + self._alter_object_owners(8)
                 + dedent(
                     """
-                    $[version>=5900] postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+                    $[version>=5900] postgres-execute connection=postgres://mz_system@${testdrive.materialize-internal-sql-addr}
                     GRANT CREATEROLE ON SYSTEM TO materialize
 
-                    $[version<5900] postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+                    $[version<5900] postgres-execute connection=postgres://mz_system@${testdrive.materialize-internal-sql-addr}
                     ALTER ROLE materialize CREATEROLE
 
                     > CREATE ROLE owner_role_03
                     >[version<5900] ALTER ROLE owner_role_03 CREATEDB CREATECLUSTER
 
-                    $[version>=5900] postgres-execute connection=postgres://mz_system@materialized:6877/materialize
+                    $[version>=5900] postgres-execute connection=postgres://mz_system@${testdrive.materialize-internal-sql-addr}
                     GRANT CREATEDB, CREATECLUSTER ON SYSTEM TO owner_role_03
                     """
                 ),