http/client: Add "total new connections" metrics

The metrics is the monotonic counter that gets ++ed every time the
factory is kicked for new connection.

The recommended usage is to get the time-derivative from the counter and
check _this_ value. If it's large, it means that the existing
connections are not getting back to pool due to connection errors or
unexpected server status codes. This, in turn, can be used to explain
large delays in requests, as each time a request is run over a new
connection it doesn't benefit from connections keep-alive and gets extra
latency.

Signed-off-by: Pavel Emelyanov <xemul@scylladb.com>

Closes scylladb#1718

include/seastar/http/client.hh

@@ -168,6 +168,7 @@ class client {
     std::unique_ptr<connection_factory> _new_connections;
     unsigned _nr_connections = 0;
     unsigned _max_connections;
+    unsigned long _total_new_connections = 0;
     condition_variable _wait_con;
     connections_list_t _pool;
 
@@ -266,6 +267,17 @@ public:
     unsigned idle_connections_nr() const noexcept {
         return _pool.size();
     }
+
+    /**
+     * \brief Returns the total number of connection factory invocations made so far
+     *
+     * This is the monotonically-increasing counter describing how "frequently" the
+     * client kicks its factory for new connections.
+     */
+
+    unsigned long total_new_connections_nr() const noexcept {
+        return _total_new_connections;
+    }
 };
 
 } // experimental namespace

src/http/client.cc

@@ -237,6 +237,7 @@ future<client::connection_ptr> client::get_connection() {
         });
     }
 
+    _total_new_connections++;
     return _new_connections->make().then([cr = internal::client_ref(this)] (connected_socket cs) mutable {
         http_log.trace("created new http connection {}", cs.local_address());
         auto con = seastar::make_shared<connection>(std::move(cs), std::move(cr));