await quorum+1 improvements

1. If khepri_db is enabled, rabbitmq_metadata is a critical component
2. When waiting for quorum+1, periodically log what doesn't have the
   quorum+1
   - for components: just list them
   - for queues: list how many we are waiting for and how to display
     them (because there could be a large number, logging that
     could be impractical or even dangerous)
3. make the tests signficantly faster by using a single group

(cherry picked from commit 6ca2022)

Author: mkuratczyk

deps/rabbit/src/rabbit_upgrade_preparation.erl

@@ -15,6 +15,7 @@
 %%
 
 -define(SAMPLING_INTERVAL, 200).
+-define(LOGGING_FREQUENCY, ?SAMPLING_INTERVAL * 100).
 
 await_online_quorum_plus_one(Timeout) ->
     Iterations = ceil(Timeout / ?SAMPLING_INTERVAL),
@@ -30,7 +31,11 @@ online_members(Component) ->
                                                   erlang, whereis, [Component])).
 
 endangered_critical_components() ->
-    CriticalComponents = [rabbit_stream_coordinator],
+    CriticalComponents = [rabbit_stream_coordinator] ++
+                            case rabbit_feature_flags:is_enabled(khepri_db) of
+                                true -> [rabbitmq_metadata];
+                                false -> []
+                            end,
     Nodes = rabbit_nodes:list_members(),
     lists:filter(fun (Component) ->
                          NumAlive = length(online_members(Component)),
@@ -57,6 +62,21 @@ do_await_safe_online_quorum(IterationsLeft) ->
     case EndangeredQueues =:= [] andalso endangered_critical_components() =:= [] of
         true -> true;
         false ->
+            case IterationsLeft rem ?LOGGING_FREQUENCY of
+                0 ->
+                    case length(EndangeredQueues) of
+                        0 -> ok;
+                        N -> rabbit_log:info("Waiting for ~p queues to have quorum+1 members online."
+                                             "You can list them with `rabbitmq-diagnostics check_if_node_is_quorum_critical`", [N])
+                    end,
+                    case endangered_critical_components() of
+                        [] -> ok;
+                        _ -> rabbit_log:info("Waiting for the following critical components to have quorum+1 members online: ~p.",
+                                             [endangered_critical_components()])
+                    end;
+                _ ->
+                    ok
+            end,
             timer:sleep(?SAMPLING_INTERVAL),
             do_await_safe_online_quorum(IterationsLeft - 1)
     end.

deps/rabbit/test/upgrade_preparation_SUITE.erl

@@ -14,20 +14,16 @@
 
 all() ->
     [
-      {group, quorum_queue},
-      {group, stream}
+      {group, clustered}
     ].
 
 groups() ->
     [
-     {quorum_queue, [], [
-         await_quorum_plus_one_qq
-     ]},
-     {stream, [], [
-         await_quorum_plus_one_stream
-     ]},
-     {stream_coordinator, [], [
-         await_quorum_plus_one_stream_coordinator
+     {clustered, [], [
+         await_quorum_plus_one_qq,
+         await_quorum_plus_one_stream,
+         await_quorum_plus_one_stream_coordinator,
+         await_quorum_plus_one_rabbitmq_metadata
      ]}
     ].
 
@@ -44,21 +40,14 @@ end_per_suite(Config) ->
     rabbit_ct_helpers:run_teardown_steps(Config).
 
 init_per_group(Group, Config) ->
-    case rabbit_ct_helpers:is_mixed_versions() of
-        true ->
-            %% in a 3.8/3.9 mixed cluster, ra will not cluster across versions,
-            %% so quorum plus one will not be achieved
-            {skip, "not mixed versions compatible"};
-        _ ->
-            Config1 = rabbit_ct_helpers:set_config(Config,
-                                                   [
-                                                    {rmq_nodes_count, 3},
-                                                    {rmq_nodename_suffix, Group}
-                                                   ]),
-            rabbit_ct_helpers:run_steps(Config1,
-                                        rabbit_ct_broker_helpers:setup_steps() ++
-                                            rabbit_ct_client_helpers:setup_steps())
-    end.
+    Config1 = rabbit_ct_helpers:set_config(Config,
+                                           [
+                                            {rmq_nodes_count, 3},
+                                            {rmq_nodename_suffix, Group}
+                                           ]),
+    rabbit_ct_helpers:run_steps(Config1,
+                                rabbit_ct_broker_helpers:setup_steps() ++
+                                rabbit_ct_client_helpers:setup_steps()).
 
 end_per_group(_Group, Config) ->
     rabbit_ct_helpers:run_steps(Config,
@@ -120,12 +109,24 @@ await_quorum_plus_one_stream_coordinator(Config) ->
     %% no queues/streams beyond this point
 
     ok = rabbit_ct_broker_helpers:stop_node(Config, B),
-    %% this should fail because the corrdinator has only 2 running nodes
+    %% this should fail because the coordinator has only 2 running nodes
     ?assertNot(await_quorum_plus_one(Config, 0)),
 
     ok = rabbit_ct_broker_helpers:start_node(Config, B),
     ?assert(await_quorum_plus_one(Config, 0)).
 
+await_quorum_plus_one_rabbitmq_metadata(Config) ->
+    Nodes = [A, B, _C] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
+    ok = rabbit_ct_broker_helpers:enable_feature_flag(Config, Nodes, khepri_db),
+    ?assert(await_quorum_plus_one(Config, A)),
+
+    ok = rabbit_ct_broker_helpers:stop_node(Config, B),
+    %% this should fail because rabbitmq_metadata has only 2 running nodes
+    ?assertNot(await_quorum_plus_one(Config, A)),
+
+    ok = rabbit_ct_broker_helpers:start_node(Config, B),
+    ?assert(await_quorum_plus_one(Config, A)).
+
 %%
 %% Implementation
 %%