Add specific stream protocol counters to track protocol errors

deps/rabbit/include/rabbit_global_counters.hrl

@@ -0,0 +1,2 @@
+-define(NUM_PROTOCOL_COUNTERS, 8).
+-define(NUM_PROTOCOL_QUEUE_TYPE, 8).

deps/rabbit/src/rabbit_global_counters.erl

@@ -10,8 +10,10 @@
 -export([
          boot_step/0,
          init/1,
+         init/2,
          overview/0,
          prometheus_format/0,
+         increase_protocol_counter/3,
          messages_received/2,
          messages_received_confirm/2,
          messages_routed/2,
@@ -41,6 +43,7 @@
 -define(MESSAGES_CONFIRMED, 6).
 -define(PUBLISHERS, 7).
 -define(CONSUMERS, 8).
+%% Note: ?NUM_PROTOCOL_COUNTERS needs to be up-to-date. See include/rabbit_global_counters.hrl
 -define(PROTOCOL_COUNTERS,
             [
                 {
@@ -86,6 +89,7 @@
 -define(MESSAGES_GET_EMPTY, 6).
 -define(MESSAGES_REDELIVERED, 7).
 -define(MESSAGES_ACKNOWLEDGED, 8).
+%% Note: ?NUM_PROTOCOL_QUEUE_TYPE_COUNTERS needs to be up-to-date. See include/rabbit_global_counters.hrl
 -define(PROTOCOL_QUEUE_TYPE_COUNTERS,
             [
                 {
@@ -128,14 +132,17 @@ boot_step() ->
     init([{protocol, amqp091}, {queue_type, rabbit_quorum_queue}]),
     init([{protocol, amqp091}, {queue_type, rabbit_stream_queue}]).
 
-init(Labels = [{protocol, Protocol}, {queue_type, QueueType}]) ->
+init(Labels) ->
+    init(Labels, []).
+
+init(Labels = [{protocol, Protocol}, {queue_type, QueueType}], Extra) ->
     _ = seshat_counters:new_group(?MODULE),
-    Counters = seshat_counters:new(?MODULE, Labels, ?PROTOCOL_QUEUE_TYPE_COUNTERS),
+    Counters = seshat_counters:new(?MODULE, Labels, ?PROTOCOL_QUEUE_TYPE_COUNTERS ++ Extra),
     persistent_term:put({?MODULE, Protocol, QueueType}, Counters),
     ok;
-init(Labels = [{protocol, Protocol}]) ->
+init(Labels = [{protocol, Protocol}], Extra) ->
     _ = seshat_counters:new_group(?MODULE),
-    Counters = seshat_counters:new(?MODULE, Labels, ?PROTOCOL_COUNTERS),
+    Counters = seshat_counters:new(?MODULE, Labels, ?PROTOCOL_COUNTERS ++ Extra),
     persistent_term:put({?MODULE, Protocol}, Counters),
     ok.
 
@@ -145,6 +152,9 @@ overview() ->
 prometheus_format() ->
     seshat_counters:prometheus_format(?MODULE).
 
+increase_protocol_counter(Protocol, Counter, Num) ->
+    counters:add(fetch(Protocol), Counter, Num).
+
 messages_received(Protocol, Num) ->
     counters:add(fetch(Protocol), ?MESSAGES_RECEIVED, Num).
 

deps/rabbitmq_prometheus/metrics.md

@@ -67,6 +67,31 @@ To generate these:
 | rabbitmq_global_publishers                                  | Publishers currently connected |
 | rabbitmq_global_consumers                                   | Consumers currently connected |
 
+#### Stream global counters
+
+These metrics are specific to the stream protocol.
+
+| Metric                                                      | Description                                                                                                  |
+| ---                                                         | ---                                                                                                          |
+| stream_error_stream_does_not_exist_total                    | Total number of commands rejected with stream does not exist error |
+| stream_error_subscription_id_already_exists_total           | Total number of commands failed with subscription id already exists |
+| stream_error_subscription_id_does_not_exist_total           | Total number of commands failed with subscription id does not exist |
+| stream_error_stream_already_exists_total                    | Total number of commands failed with stream already exists |
+| stream_error_stream_not_available_total                     | Total number of commands failed with stream not available |
+| stream_error_sasl_mechanism_not_supported_total             | Total number of commands failed with sasl mechanism not supported |
+| stream_error_authentication_failure_total                   | Total number of commands failed with authentication failure |
+| stream_error_sasl_error_total                               | Total number of commands failed with sasl error |
+| stream_error_sasl_challenge_total                           | Total number of commands failed with sasl challenge |
+| stream_error_sasl_authentication_failure_loopback_total     | Total number of commands failed with sasl authentication failure loopback |
+| stream_error_vhost_access_failure_total                     | Total number of commands failed with vhost access failure |
+| stream_error_unknown_frame_total                            | Total number of commands failed with unknown frame |
+| stream_error_frame_too_large_total                          | Total number of commands failed with frame too large |
+| stream_error_internal_error_total                           | Total number of commands failed with internal error |
+| stream_error_access_refused_total                           | Total number of commands failed with access refused |
+| stream_error_precondition_failed_total                      | Total number of commands failed with precondition failed |
+| stream_error_publisher_does_not_exist_total                 | Total number of commands failed with publisher does not exist |
+
+
 ### Generic
 
 | Metric                                    | Description                                        |

deps/rabbitmq_stream/BUILD.bazel

@@ -41,9 +41,6 @@ BUILD_DEPS = [
 
 DEPS = [
     "//deps/rabbitmq_stream_common:bazel_erlang_lib",
-]
-
-RUNTIME_DEPS = [
     "//deps/rabbit:bazel_erlang_lib",
 ]
 
@@ -53,7 +50,6 @@ rabbitmq_lib(
     app_module = APP_MODULE,
     app_name = APP_NAME,
     build_deps = BUILD_DEPS,
-    runtime_deps = RUNTIME_DEPS,
     deps = DEPS,
 )
 
@@ -105,6 +101,7 @@ rabbitmq_integration_suite(
         },
     },
     deps = [
+        "//deps/rabbit:bazel_erlang_lib",
         "//deps/rabbitmq_stream_common:bazel_erlang_lib",
     ],
 )

deps/rabbitmq_stream/include/rabbit_stream_metrics.hrl

@@ -1,2 +1,94 @@
+-include_lib("rabbit/include/rabbit_global_counters.hrl").
+
 -define(TABLE_CONSUMER, rabbit_stream_consumer_created).
 -define(TABLE_PUBLISHER, rabbit_stream_publisher_created).
+
+-define(STREAM_DOES_NOT_EXIST, ?NUM_PROTOCOL_COUNTERS + 1).
+-define(SUBSCRIPTION_ID_ALREADY_EXISTS, ?NUM_PROTOCOL_COUNTERS + 2).
+-define(SUBSCRIPTION_ID_DOES_NOT_EXIST, ?NUM_PROTOCOL_COUNTERS + 3).
+-define(STREAM_ALREADY_EXISTS, ?NUM_PROTOCOL_COUNTERS + 4).
+-define(STREAM_NOT_AVAILABLE, ?NUM_PROTOCOL_COUNTERS + 5).
+-define(SASL_MECHANISM_NOT_SUPPORTED, ?NUM_PROTOCOL_COUNTERS + 6).
+-define(AUTHENTICATION_FAILURE, ?NUM_PROTOCOL_COUNTERS + 7).
+-define(SASL_ERROR, ?NUM_PROTOCOL_COUNTERS + 8).
+-define(SASL_CHALLENGE, ?NUM_PROTOCOL_COUNTERS + 9).
+-define(SASL_AUTHENTICATION_FAILURE_LOOPBACK, ?NUM_PROTOCOL_COUNTERS + 10).
+-define(VHOST_ACCESS_FAILURE, ?NUM_PROTOCOL_COUNTERS + 11).
+-define(UNKNOWN_FRAME, ?NUM_PROTOCOL_COUNTERS + 12).
+-define(FRAME_TOO_LARGE, ?NUM_PROTOCOL_COUNTERS + 13).
+-define(INTERNAL_ERROR, ?NUM_PROTOCOL_COUNTERS + 14).
+-define(ACCESS_REFUSED, ?NUM_PROTOCOL_COUNTERS + 15).
+-define(PRECONDITION_FAILED, ?NUM_PROTOCOL_COUNTERS + 16).
+-define(PUBLISHER_DOES_NOT_EXIST, ?NUM_PROTOCOL_COUNTERS + 17).
+
+-define(PROTOCOL_COUNTERS,
+        [
+         {
+          stream_error_stream_does_not_exist_total, ?STREAM_DOES_NOT_EXIST, counter,
+          "Total number of commands rejected with stream does not exist error"
+         },
+         {
+          stream_error_subscription_id_already_exists_total, ?SUBSCRIPTION_ID_ALREADY_EXISTS, counter,
+          "Total number of commands failed with subscription id already exists"
+         },
+         {
+          stream_error_subscription_id_does_not_exist_total, ?SUBSCRIPTION_ID_DOES_NOT_EXIST, counter,
+          "Total number of commands failed with subscription id does not exist"
+         },
+         {
+          stream_error_stream_already_exists_total, ?STREAM_ALREADY_EXISTS, counter,
+          "Total number of commands failed with stream already exists"
+         },
+         {
+          stream_error_stream_not_available_total, ?STREAM_NOT_AVAILABLE, counter,
+          "Total number of commands failed with stream not available"
+         },
+         {
+          stream_error_sasl_mechanism_not_supported_total, ?SASL_MECHANISM_NOT_SUPPORTED, counter,
+          "Total number of commands failed with sasl mechanism not supported"
+         },
+         {
+          stream_error_authentication_failure_total, ?AUTHENTICATION_FAILURE, counter,
+          "Total number of commands failed with authentication failure"
+         },
+         {
+          stream_error_sasl_error_total, ?SASL_ERROR, counter,
+          "Total number of commands failed with sasl error"
+         },
+         {
+          stream_error_sasl_challenge_total, ?SASL_CHALLENGE, counter,
+          "Total number of commands failed with sasl challenge"
+         },
+         {
+          stream_error_sasl_authentication_failure_loopback_total, ?SASL_AUTHENTICATION_FAILURE_LOOPBACK, counter,
+          "Total number of commands failed with sasl authentication failure loopback"
+         },
+         {
+          stream_error_vhost_access_failure_total, ?VHOST_ACCESS_FAILURE, counter,
+          "Total number of commands failed with vhost access failure"
+         },
+         {
+          stream_error_unknown_frame_total, ?UNKNOWN_FRAME, counter,
+          "Total number of commands failed with unknown frame"
+         },
+         {
+          stream_error_frame_too_large_total, ?FRAME_TOO_LARGE, counter,
+          "Total number of commands failed with frame too large"
+         },
+         {
+          stream_error_internal_error_total, ?INTERNAL_ERROR, counter,
+          "Total number of commands failed with internal error"
+         },
+         {
+          stream_error_access_refused_total, ?ACCESS_REFUSED, counter,
+          "Total number of commands failed with access refused"
+         },
+         {
+          stream_error_precondition_failed_total, ?PRECONDITION_FAILED, counter,
+          "Total number of commands failed with precondition failed"
+         },
+         {
+          stream_error_publisher_does_not_exist_total, ?PUBLISHER_DOES_NOT_EXIST, counter,
+          "Total number of commands failed with publisher does not exist"
+         }
+        ]).

deps/rabbitmq_stream/src/rabbit_stream.erl

@@ -34,10 +34,11 @@
 
 -include_lib("rabbit_common/include/rabbit.hrl").
 -include_lib("rabbitmq_stream_common/include/rabbit_stream.hrl").
+-include("rabbit_stream_metrics.hrl").
 
 start(_Type, _Args) ->
     rabbit_stream_metrics:init(),
-    rabbit_global_counters:init([{protocol, stream}]),
+    rabbit_global_counters:init([{protocol, stream}], ?PROTOCOL_COUNTERS),
     rabbit_global_counters:init([{protocol, stream}, {queue_type, ?STREAM_QUEUE_TYPE}]),
     rabbit_stream_sup:start_link().