Backport test fixes from main to v4.1.x

.github/workflows/test-make-target.yaml

@@ -90,6 +90,7 @@ jobs:
     - name: RUN TESTS
       if: inputs.plugin != 'rabbitmq_cli'
       run: |
+        sudo netstat -ntp
         make -C deps/${{ inputs.plugin }} ${{ inputs.make_target }} RABBITMQ_METADATA_STORE=${{ inputs.metadata_store }}
 
     # rabbitmq_cli needs a correct broker version for two of its tests.

deps/rabbit/Makefile

@@ -241,10 +241,10 @@ define ct_master.erl
 	peer:call(Pid2, net_kernel, set_net_ticktime, [5]),
 	peer:call(Pid3, net_kernel, set_net_ticktime, [5]),
 	peer:call(Pid4, net_kernel, set_net_ticktime, [5]),
-	peer:call(Pid1, persistent_term, put, [rabbit_ct_tcp_port_base, 23000]),
-	peer:call(Pid2, persistent_term, put, [rabbit_ct_tcp_port_base, 25000]),
-	peer:call(Pid3, persistent_term, put, [rabbit_ct_tcp_port_base, 27000]),
-	peer:call(Pid4, persistent_term, put, [rabbit_ct_tcp_port_base, 29000]),
+	peer:call(Pid1, persistent_term, put, [rabbit_ct_tcp_port_base, 16000]),
+	peer:call(Pid2, persistent_term, put, [rabbit_ct_tcp_port_base, 20000]),
+	peer:call(Pid3, persistent_term, put, [rabbit_ct_tcp_port_base, 24000]),
+	peer:call(Pid4, persistent_term, put, [rabbit_ct_tcp_port_base, 28000]),
 	[{[_], {ok, Results}}] = ct_master_fork:run("$1"),
 	peer:stop(Pid4),
 	peer:stop(Pid3),

deps/rabbit/test/amqp_auth_SUITE.erl

@@ -120,12 +120,17 @@ init_per_group(Group, Config0) ->
                Config1,
                rabbit_ct_broker_helpers:setup_steps() ++
                rabbit_ct_client_helpers:setup_steps()),
-    Vhost = <<"test vhost">>,
-    User = <<"test user">>,
-    ok = rabbit_ct_broker_helpers:add_vhost(Config, Vhost),
-    ok = rabbit_ct_broker_helpers:add_user(Config, User),
-    [{test_vhost, Vhost},
-     {test_user, User}] ++ Config.
+    case Config of
+        _ when is_list(Config) ->
+            Vhost = <<"test vhost">>,
+            User = <<"test user">>,
+            ok = rabbit_ct_broker_helpers:add_vhost(Config, Vhost),
+            ok = rabbit_ct_broker_helpers:add_user(Config, User),
+            [{test_vhost, Vhost},
+             {test_user, User}] ++ Config;
+        {skip, _} = Skip ->
+            Skip
+    end.
 
 end_per_group(_Group, Config) ->
     ok = rabbit_ct_broker_helpers:delete_user(Config, ?config(test_user, Config)),

deps/rabbit/test/clustering_management_SUITE.erl

@@ -76,7 +76,6 @@ groups() ->
                                                  status_with_alarm,
                                                  pid_file_and_await_node_startup_in_khepri,
                                                  await_running_count_in_khepri,
-                                                 start_with_invalid_schema_in_path,
                                                  persistent_cluster_id,
                                                  stop_start_cluster_node,
                                                  restart_cluster_node,
@@ -337,7 +336,7 @@ restart_cluster_node(Config) ->
     assert_clustered([Rabbit, Hare]).
 
 join_and_part_cluster_in_khepri(Config) ->
-    [Rabbit, Hare, Bunny] = cluster_members(Config),
+    [Rabbit, Bunny, Hare] = cluster_members(Config),
     assert_not_clustered(Rabbit),
     assert_not_clustered(Hare),
     assert_not_clustered(Bunny),
@@ -447,38 +446,38 @@ join_to_start_interval(Config) ->
     assert_clustered([Rabbit, Hare]).
 
 join_cluster_in_minority(Config) ->
-    [Rabbit, Hare, Bunny] = cluster_members(Config),
+    [Rabbit, Bunny, Hare] = cluster_members(Config),
     assert_not_clustered(Rabbit),
     assert_not_clustered(Hare),
     assert_not_clustered(Bunny),
 
-    stop_join_start(Config, Bunny, Rabbit),
+    stop_join_start(Config, Rabbit, Bunny),
     assert_clustered([Rabbit, Bunny]),
-    ok = rabbit_ct_broker_helpers:stop_node(Config, Bunny),
+    ok = rabbit_ct_broker_helpers:stop_node(Config, Rabbit),
 
     ok = stop_app(Config, Hare),
-    ?assertEqual(ok, join_cluster(Config, Hare, Rabbit, false)),
+    ?assertEqual(ok, join_cluster(Config, Hare, Bunny, false)),
 
-    ok = rabbit_ct_broker_helpers:start_node(Config, Bunny),
+    ok = rabbit_ct_broker_helpers:start_node(Config, Rabbit),
     ?assertEqual(ok, join_cluster(Config, Hare, Rabbit, false)),
     ?assertEqual(ok, start_app(Config, Hare)),
 
     assert_clustered([Rabbit, Bunny, Hare]).
 
 join_cluster_with_rabbit_stopped(Config) ->
-    [Rabbit, Hare, Bunny] = cluster_members(Config),
+    [Rabbit, Bunny, Hare] = cluster_members(Config),
     assert_not_clustered(Rabbit),
     assert_not_clustered(Hare),
     assert_not_clustered(Bunny),
 
-    stop_join_start(Config, Bunny, Rabbit),
+    stop_join_start(Config, Rabbit, Bunny),
     assert_clustered([Rabbit, Bunny]),
-    ok = stop_app(Config, Bunny),
+    ok = stop_app(Config, Rabbit),
 
     ok = stop_app(Config, Hare),
-    ?assertEqual(ok, join_cluster(Config, Hare, Rabbit, false)),
+    ?assertEqual(ok, join_cluster(Config, Hare, Bunny, false)),
 
-    ok = start_app(Config, Bunny),
+    ok = start_app(Config, Rabbit),
     ?assertEqual(ok, join_cluster(Config, Hare, Rabbit, false)),
     ?assertEqual(ok, start_app(Config, Hare)),
 
@@ -1119,7 +1118,7 @@ await_running_count_in_khepri(Config) ->
                                               await_running_count, [5, 1000])).
 
 start_nodes_in_reverse_order(Config) ->
-    [Rabbit, Hare, Bunny] = cluster_members(Config),
+    [Rabbit, Bunny, Hare] = cluster_members(Config),
     assert_not_clustered(Rabbit),
     assert_not_clustered(Hare),
     assert_not_clustered(Bunny),
@@ -1142,7 +1141,7 @@ start_nodes_in_reverse_order(Config) ->
 
 %% Test booting nodes in the wrong order for Mnesia. Interesting...
 start_nodes_in_stop_order(Config) ->
-    [Rabbit, Hare, Bunny] = cluster_members(Config),
+    [Rabbit, Bunny, Hare] = cluster_members(Config),
     assert_not_clustered(Rabbit),
     assert_not_clustered(Hare),
     assert_not_clustered(Bunny),
@@ -1167,7 +1166,7 @@ start_nodes_in_stop_order(Config) ->
     end.
 
 start_nodes_in_stop_order_in_khepri(Config) ->
-    [Rabbit, Hare, Bunny] = cluster_members(Config),
+    [Rabbit, Bunny, Hare] = cluster_members(Config),
     assert_not_clustered(Rabbit),
     assert_not_clustered(Hare),
     assert_not_clustered(Bunny),
@@ -1190,7 +1189,7 @@ start_nodes_in_stop_order_in_khepri(Config) ->
 
 %% TODO test force_boot with Khepri involved
 start_nodes_in_stop_order_with_force_boot(Config) ->
-    [Rabbit, Hare, Bunny] = cluster_members(Config),
+    [Rabbit, Bunny, Hare] = cluster_members(Config),
     assert_not_clustered(Rabbit),
     assert_not_clustered(Hare),
     assert_not_clustered(Bunny),

deps/rabbit/test/rabbit_stream_queue_SUITE.erl

@@ -1565,13 +1565,13 @@ format(Config) ->
     case length(Nodes) of
         3 ->
             [_, Server2, Server3] = Nodes,
-            ok = rabbit_control_helper:command(stop_app, Server2),
             ok = rabbit_control_helper:command(stop_app, Server3),
+            ok = rabbit_control_helper:command(stop_app, Server2),
 
             Fmt2 = rabbit_ct_broker_helpers:rpc(Config, Server, rabbit_stream_queue,
                                                ?FUNCTION_NAME, [QRecord, #{}]),
-            ok = rabbit_control_helper:command(start_app, Server2),
             ok = rabbit_control_helper:command(start_app, Server3),
+            ok = rabbit_control_helper:command(start_app, Server2),
             ?assertEqual(stream, proplists:get_value(type, Fmt2)),
             ?assertEqual(minority, proplists:get_value(state, Fmt2)),
             ?assertEqual(Server, proplists:get_value(leader, Fmt2)),
@@ -2743,7 +2743,7 @@ retry_if_coordinator_unavailable(Config, Server, Cmd, Retry) ->
             case re:run(Msg, ".*coordinator_unavailable.*", [{capture, none}]) of
                 match ->
                     ct:pal("Attempt to execute command ~p failed, coordinator unavailable", [Cmd]),
-                    retry_if_coordinator_unavailable(Config, Ch, Cmd, Retry - 1);
+                    retry_if_coordinator_unavailable(Config, Server, Cmd, Retry - 1);
                 _ ->
                     exit(Error)
             end

deps/rabbit/test/unit_credit_flow_SUITE.erl

@@ -11,6 +11,9 @@
 
 -compile(export_all).
 
+suite() ->
+    [{timetrap, {minutes, 3}}].
+
 all() ->
     [
       {group, sequential_tests}

deps/rabbitmq_ct_helpers/src/rabbit_ct_broker_helpers.erl

@@ -173,7 +173,8 @@
     user/1,
 
     configured_metadata_store/1,
-    await_metadata_store_consistent/2
+    await_metadata_store_consistent/2,
+    do_nodes_run_same_ra_machine_version/2
   ]).
 
 %% Internal functions exported to be used by rpc:call/4.
@@ -1174,6 +1175,12 @@ ra_last_applied(ServerId) ->
     #{last_applied := LastApplied} = ra:key_metrics(ServerId),
     LastApplied.
 
+do_nodes_run_same_ra_machine_version(Config, RaMachineMod) ->
+    [MacVer1 | MacVerN] = MacVers = rpc_all(Config, RaMachineMod, version, []),
+    ct:pal("Ra machine versions of ~s: ~0p", [RaMachineMod, MacVers]),
+    is_integer(MacVer1) andalso
+    lists:all(fun(MacVer) -> MacVer =:= MacVer1 end, MacVerN).
+
 rewrite_node_config_file(Config, Node) ->
     NodeConfig = get_node_config(Config, Node),
     I = if

deps/rabbitmq_peer_discovery_consul/test/system_SUITE.erl

@@ -83,9 +83,27 @@ init_per_testcase(Testcase, Config)
     case Config3 of
         _ when is_list(Config3) ->
             try
-                _ = rabbit_ct_broker_helpers:rpc_all(
-                      Config3, rabbit_peer_discovery_backend, api_version, []),
-                Config3
+                SameMacVer = (
+                  rabbit_ct_broker_helpers:
+                  do_nodes_run_same_ra_machine_version(
+                    Config3, khepri_machine)),
+                case SameMacVer of
+                    true ->
+                        _ = rabbit_ct_broker_helpers:rpc_all(
+                              Config3,
+                              rabbit_peer_discovery_backend, api_version, []),
+                        Config3;
+                    false ->
+                        Config5 = rabbit_ct_helpers:run_steps(
+                                    Config3,
+                                    rabbit_ct_client_helpers:teardown_steps()
+                                    ++
+                                    rabbit_ct_broker_helpers:teardown_steps()),
+                        rabbit_ct_helpers:testcase_finished(Config5, Testcase),
+                        {skip,
+                         "Nodes are using different Khepri Ra machine "
+                         "versions; clustering will likely fail"}
+                end
             catch
                 error:{exception, undef,
                        [{rabbit_peer_discovery_backend, api_version, _, _}

deps/rabbitmq_peer_discovery_etcd/test/system_SUITE.erl

@@ -92,9 +92,27 @@ init_per_testcase(Testcase, Config)
     case Config3 of
         _ when is_list(Config3) ->
             try
-                _ = rabbit_ct_broker_helpers:rpc_all(
-                      Config3, rabbit_peer_discovery_backend, api_version, []),
-                Config3
+                SameMacVer = (
+                  rabbit_ct_broker_helpers:
+                  do_nodes_run_same_ra_machine_version(
+                    Config3, khepri_machine)),
+                case SameMacVer of
+                    true ->
+                        _ = rabbit_ct_broker_helpers:rpc_all(
+                              Config3,
+                              rabbit_peer_discovery_backend, api_version, []),
+                        Config3;
+                    false ->
+                        Config5 = rabbit_ct_helpers:run_steps(
+                                    Config3,
+                                    rabbit_ct_client_helpers:teardown_steps()
+                                    ++
+                                    rabbit_ct_broker_helpers:teardown_steps()),
+                        rabbit_ct_helpers:testcase_finished(Config5, Testcase),
+                        {skip,
+                         "Nodes are using different Khepri Ra machine "
+                         "versions; clustering will likely fail"}
+                end
             catch
                 error:{exception, undef,
                        [{rabbit_peer_discovery_backend, api_version, _, _}
@@ -239,7 +257,9 @@ wait_for_etcd(EtcdEndpoints) ->
     Timeout = 60000,
     rabbit_ct_helpers:await_condition(
       fun() ->
-              case eetcd:open(test, EtcdEndpoints) of
+              Ret = eetcd:open(test, EtcdEndpoints),
+              ct:pal("Ret = ~p", [Ret]),
+              case Ret of
                   {ok, _Pid} -> true;
                   _          -> false
               end