add exp backoff to grpc client reconnects to vals

config/sys.config

@@ -124,6 +124,10 @@
    {default_routers, ["/p2p/11w77YQLhgUt8HUJrMtntGGr97RyXmot1ofs5Ct2ELTmbFoYsQa","/p2p/11afuQSrmk52mgxLu91AdtDXbJ9wmqWBUxC3hvjejoXkxEZfPvY"]},
    {mark_mods, [miner_hbbft_handler]},
    {stabilization_period, 50000},
+   %% max exponetial multipler applied to default 5 sec backoff period
+   %% for grpc reconnects.  With the default value of 128 below that
+   %% gives a max backoff of 640 seconds ( 5 * 128 ), or just over 10 minutes
+   {max_backoff, 128},
    {seed_validators, [
        {"11tk4zzbyfMPYYHYda255ACoqfYFVdrUSoCWrCYfn8BoyuYrERK", "52.49.199.40"},     %% ireland
        {"115PmCR6fpFihdjw626JXYdUEdzwjh66yoWzWkMvB9CRGEx1U6G", "3.132.190.192"},    %% ohio

config/val.config.src

@@ -82,6 +82,11 @@
                       header_table_size => 4096,
                       enable_push => 1,
                       max_concurrent_streams => unlimited,
+                      max_connections => 4000,                   %% the max connections permitted, after which connections will be refused
+                      accept_rate => 100,                        %% the rate limit for for all incoming connections
+                      accept_rate_interval => per_second,        %% the rate limit period for all incoming connections
+                      accept_rate_by_ip => 40,                   %% the rate limit for any one source ip
+                      accept_rate_by_ip_interval => per_minute,  %% the rate limit period for any one source ip
                       initial_window_size => 65535,
                       max_frame_size => 16384,
                       max_header_list_size => unlimited

rebar.lock

@@ -1,7 +1,7 @@
 {"1.2.0",
 [{<<"acceptor_pool">>,
   {git,"https://github.com/novalabsxyz/acceptor_pool",
-       {ref,"56d676e00c11fd071a6bcc4059e3454960900af7"}},
+       {ref,"7e53c5cec2c867838576237c6d016e1eb5501f03"}},
   2},
  {<<"backoff">>,{pkg,<<"backoff">>,<<"1.1.6">>},2},
  {<<"base32">>,{pkg,<<"base32">>,<<"0.1.0">>},4},
@@ -101,7 +101,7 @@
   0},
  {<<"grpcbox">>,
   {git,"https://github.com/novalabsxyz/grpcbox.git",
-       {ref,"6180be0f4f4f199f824e60ae5c64ddd2daa5acff"}},
+       {ref,"9479156cb439ac2fc7bc0da34387f48678b846c7"}},
   1},
  {<<"h3">>,
   {git,"https://github.com/helium/erlang-h3.git",
@@ -180,7 +180,7 @@
   1},
  {<<"sibyl">>,
   {git,"https://github.com/helium/sibyl.git",
-       {ref,"06a32a2c9f03a63bd8d51e3bc7b3bb9759ffd2f9"}},
+       {ref,"c5d23935d8e32784ca27667116703f2f135003e2"}},
   0},
  {<<"sidejob">>,{pkg,<<"sidejob">>,<<"2.1.0">>},2},
  {<<"small_ints">>,{pkg,<<"small_ints">>,<<"0.1.0">>},4},

src/poc/miner_poc_grpc_client_statem.erl

@@ -54,7 +54,8 @@
     check_target_req_timer,
     down_events_in_period = 0,
     stability_check_timer,
-    block_age_timer
+    block_age_timer,
+    last_backoff = 1
 }).
 
 %% these are config vars the miner is interested in, if they change we
@@ -204,17 +205,28 @@ setup(enter, _OldState, Data) ->
         connection_pid = undefined,
         conn_monitor_ref = undefined
     }};
-setup(info, active_check, Data) ->
+setup(info, active_check, #data{ last_backoff = LastBackoff} = Data) ->
     %% env var `enable_grpc_client` will have a value of true
     %% if validator challenges are enabled
     %% set from miner lora light
     case application:get_env(miner, enable_grpc_client, false) of
         true ->
-            erlang:send_after(?VALIDATOR_RECONNECT_DELAY, self(), find_validator);
+            %% increment backoff counter with a lil bit of jitter
+            %% we will do this each time we hit this state
+            %% which we will keep on doing if the client
+            %% cannot connect to a selected val
+            %% the backoff value is used as a multiplier on the
+            %% default reconnect delay applied before entering the 'find validator' state
+            %% if we successfully connect to a val, backoff will be reset to 1
+            %% the backoff is defaulted to a multiple of 128, which gives us 640 secs or approx 10 mins
+            erlang:send_after(?VALIDATOR_RECONNECT_DELAY * LastBackoff, self(), find_validator),
+            MaxBackOff = application:get_env(miner, max_backoff, 128),
+            {keep_state, Data#data{last_backoff = backoff:rand_increment(LastBackoff, MaxBackOff)}};
         false ->
-            erlang:send_after(?ACTIVE_CHECK_DELAY, self(), active_check)
-    end,
-    {keep_state, Data};
+            erlang:send_after(?ACTIVE_CHECK_DELAY, self(), active_check),
+            {keep_state, Data}
+    end;
+
 setup(info, find_validator, Data) ->
     %% ask a random seed validator for the address of a 'proper' validator
     %% we will then use this as our default durable validator
@@ -260,7 +272,8 @@ setup(
                     conn_monitor_ref = M,
                     stability_check_timer = SCTRef,
                     block_age_timer = BACTRef,
-                    down_events_in_period = 0
+                    down_events_in_period = 0,
+                    last_backoff = 1
                 },
                 [{next_event, info, fetch_config}]};
         {error, _} ->