apache · bneradt · Dec 16, 2020 · Oct 24, 2021 · Aug 3, 2021 · Oct 25, 2021
diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst
@@ -3887,6 +3887,13 @@ Client-Related Configuration
 
    Enables (``1``) or disables (``0``) TLSv1_3 in the ATS client context. If not specified, enabled by default
 
+.. ts:cv:: CONFIG proxy.config.ssl.client.alpn_protocols STRING ""
+
+   Set the alpn string that ATS will send to origin during new connections.  By default no ALPN string will be set.
+   To enable HTTP/2 communication to the origin, set this to "h2,http1.1".
+
+   :overridable:
+
 .. ts:cv:: CONFIG proxy.config.ssl.async.handshake.enabled INT 0
 
    Enables the use of OpenSSL async job during the TLS handshake.  Traffic
@@ -4025,7 +4032,17 @@ HTTP/2 Configuration
 .. ts:cv:: CONFIG proxy.config.http2.initial_window_size_in INT 65535
    :reloadable:
 
-   The initial window size for inbound connections.
+   The initial window size for inbound connection streams.
+
+.. ts:cv:: CONFIG proxy.config.http2.session_initial_window_size_in INT 0
+   :reloadable:
+
+   The initial window size for inbound connection session.  HTTP/2 provides both
+   a per stream window and a session wide window.  Each data byte exchanged decrements
+   the window of the associated stream and the session window.  To allow for multiple
+   active streams, the session window should be larger than the stream window.
+   |TS| verifies that the session initial window is always at least as large as the
+   stream initial window.
 
 .. ts:cv:: CONFIG proxy.config.http2.max_frame_size INT 16384
    :reloadable:

diff --git a/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst b/doc/admin-guide/monitoring/statistics/core/http-connection.en.rst
@@ -168,6 +168,21 @@ HTTP/2
 
    Represents the current number of HTTP/2 active connections from client to the |TS|.
 
+.. ts:stat:: global proxy.process.http2.total_server_connections integer
+   :type: counter
+
+   Represents the total number of HTTP/2 connections from |TS| to the origin.
+
+.. ts:stat:: global proxy.process.http2.current_server_connections integer
+   :type: gauge
+
+   Represents the current number of HTTP/2 connections from |TS| to the origin.
+
+.. ts:stat:: global proxy.process.http2.current_active_server_connections integer
+   :type: gauge
+
+   Represents the current number of HTTP/2 active connections from |TS| to the origin.
+
 .. ts:stat:: global proxy.process.http2.connection_errors integer
    :type: counter
 

diff --git a/doc/admin-guide/monitoring/statistics/core/http-transaction.en.rst b/doc/admin-guide/monitoring/statistics/core/http-transaction.en.rst
@@ -165,6 +165,16 @@ HTTP/2
 
    Represents the current number of HTTP/2 streams from client to the |TS|.
 
+.. ts:stat:: global proxy.process.http2.total_server_streams integer
+   :type: counter
+
+   Represents the total number of HTTP/2 streams from |TS| to the origin.
+
+.. ts:stat:: global proxy.process.http2.current_server_streams integer
+   :type: gauge
+
+   Represents the current number of HTTP/2 streams from |TS| to the origin.
+
 .. ts:stat:: global proxy.process.http2.total_transactions_time integer
    :type: counter
    :units: seconds

diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in
@@ -885,6 +885,7 @@ typedef enum {
   TS_CONFIG_PLUGIN_VC_DEFAULT_BUFFER_INDEX,
   TS_CONFIG_PLUGIN_VC_DEFAULT_BUFFER_WATER_MARK,
   TS_CONFIG_NET_SOCK_NOTSENT_LOWAT,
+  TS_CONFIG_SSL_CLIENT_ALPN_PROTOCOLS,
   TS_CONFIG_LAST_ENTRY
 } TSOverridableConfigKey;
 

diff --git a/include/tscore/ink_defs.h b/include/tscore/ink_defs.h
@@ -110,6 +110,8 @@ countof(const T (&)[N])
 #define MAP_SHARED_MAP_NORESERVE (MAP_SHARED)
 #endif
 
+#define MAX_ALPN_STRING 30
+
 /* Variables
  */
 extern int debug_level;

diff --git a/iocore/eventsystem/I_EThread.h b/iocore/eventsystem/I_EThread.h
@@ -47,6 +47,7 @@ class PreWarmQueue;
 
 class Event;
 class Continuation;
+class ConnectingPool;
 
 enum ThreadType {
   REGULAR = 0,
@@ -352,6 +353,7 @@ class EThread : public Thread
 
   ServerSessionPool *server_session_pool = nullptr;
   PreWarmQueue *prewarm_queue            = nullptr;
+  ConnectingPool *connecting_pool        = nullptr;
 
   /** Default handler used until it is overridden.
 

diff --git a/iocore/eventsystem/I_Thread.h b/iocore/eventsystem/I_Thread.h
@@ -121,6 +121,7 @@ class Thread
   ProxyAllocator quicNetVCAllocator;
   ProxyAllocator http1ClientSessionAllocator;
   ProxyAllocator http2ClientSessionAllocator;
+  ProxyAllocator http2ServerSessionAllocator;
   ProxyAllocator http2StreamAllocator;
   ProxyAllocator httpSMAllocator;
   ProxyAllocator quicClientSessionAllocator;

diff --git a/iocore/net/ALPNSupport.cc b/iocore/net/ALPNSupport.cc
@@ -145,3 +145,36 @@ ALPNSupport::registerNextProtocolSet(SSLNextProtocolSet *s, const SessionProtoco
   this->npnSet       = s;
   npnSet->create_npn_advertisement(protoenabled, &npn, &npnsz);
 }
+
+bool
+ALPNSupport::process_alpn_protocols(const std::string_view protocols, unsigned char *client_alpn_protocols, int &alpn_array_len)
+{
+  // Count up the number of separators
+  int start     = 0;
+  size_t offset = protocols.find(',', start);
+  int index     = 0;
+  bool retval   = true;
+  while (offset != protocols.npos) {
+    if ((index + 1 + static_cast<int>(offset) - start) > alpn_array_len) {
+      retval = false;
+      break;
+    }
+    client_alpn_protocols[index++] = offset - start;
+    memcpy(client_alpn_protocols + index, protocols.data() + start, offset - start);
+    index += offset - start;
+    start  = offset + 1;
+    offset = protocols.find(',', start);
+  }
+  // Copy in the last string
+  offset = protocols.length();
+  if ((index + 1 + static_cast<int>(offset) - start) > alpn_array_len) {
+    retval         = false;
+    alpn_array_len = 0;
+  } else {
+    client_alpn_protocols[index++] = offset - start;
+    memcpy(client_alpn_protocols + index, protocols.data() + start, offset - start);
+    index += offset - start;
+    alpn_array_len = index;
+  }
+  return retval;
+}
diff --git a/iocore/net/I_NetVConnection.h b/iocore/net/I_NetVConnection.h
@@ -228,6 +228,9 @@ struct NetVCOptions {
 
   bool tls_upstream = false;
 
+  unsigned char alpn_protocols_array[MAX_ALPN_STRING];
+  int alpn_protocols_array_size = 0;
+
   /// Reset all values to defaults.
 
   /**

diff --git a/iocore/net/P_ALPNSupport.h b/iocore/net/P_ALPNSupport.h
@@ -45,6 +45,7 @@ class ALPNSupport
   void enableProtocol(int idx);
   void clear();
   bool setSelectedProtocol(const unsigned char *proto, unsigned int len);
+  static bool process_alpn_protocols(const std::string_view protocols, unsigned char *alpn_array, int &alpn_array_len);
 
   int advertise_next_protocol(SSL *ssl, const unsigned char **out, unsigned *outlen);
   int select_next_protocol(SSL *ssl, const unsigned char **out, unsigned char *outlen, const unsigned char *in, unsigned inlen);

diff --git a/iocore/net/P_SSLConfig.h b/iocore/net/P_SSLConfig.h
@@ -101,6 +101,9 @@ struct SSLConfigParams : public ConfigInfo {
   long ssl_ctx_options;
   long ssl_client_ctx_options;
 
+  unsigned char alpn_protocols_array[MAX_ALPN_STRING];
+  int alpn_protocols_array_size = 0;
+
   char *server_tls13_cipher_suites;
   char *client_tls13_cipher_suites;
   char *server_groups_list;

diff --git a/iocore/net/SSLConfig.cc b/iocore/net/SSLConfig.cc
@@ -258,6 +258,16 @@ SSLConfigParams::initialize()
   }
 #endif
 
+  // Read in the protocol string for ALPN to origin
+  char *clientALPNProtocols;
+  REC_ReadConfigStringAlloc(clientALPNProtocols, "proxy.config.ssl.client.alpn_protocols");
+
+  // Assume the protocols are comma delimited
+  if (clientALPNProtocols) {
+    this->alpn_protocols_array_size = MAX_ALPN_STRING;
+    ALPNSupport::process_alpn_protocols(clientALPNProtocols, this->alpn_protocols_array, this->alpn_protocols_array_size);
+  }
+
 #ifdef SSL_OP_CIPHER_SERVER_PREFERENCE
   REC_ReadConfigInteger(option, "proxy.config.ssl.server.honor_cipher_order");
   if (option) {

diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc
@@ -1155,6 +1155,16 @@ SSLNetVConnection::sslStartHandShake(int event, int &err)
         return EVENT_ERROR;
       }
 
+      // If it is negative, we are conscious not setting alpn (e.g. for private server sessions)
+      if (options.alpn_protocols_array_size >= 0) {
+        if (options.alpn_protocols_array_size > 0) {
+          SSL_set_alpn_protos(this->ssl, options.alpn_protocols_array, options.alpn_protocols_array_size);
+        } else if (params->alpn_protocols_array_size > 0) {
+          // Set the ALPN protocols we are requesting.
+          SSL_set_alpn_protos(this->ssl, params->alpn_protocols_array, params->alpn_protocols_array_size);
+        }
+      }
+
       SSL_set_verify(this->ssl, SSL_VERIFY_PEER, verify_callback);
 
       // SNI
@@ -1515,6 +1525,16 @@ SSLNetVConnection::sslClientHandShakeEvent(int &err)
         X509_free(cert);
       }
     }
+    {
+      const unsigned char *proto;
+      unsigned int len = 0;
+      // Make note of the negotiated protocol
+      SSL_get0_alpn_selected(ssl, &proto, &len);
+      if (len == 0) {
+        SSL_get0_next_proto_negotiated(ssl, &proto, &len);
+      }
+      this->set_negotiated_protocol_id({reinterpret_cast<const char *>(proto), static_cast<size_t>(len)});
+    }
 
     // if the handshake is complete and write is enabled reschedule the write
     if (closed == 0 && write.enabled) {

diff --git a/iocore/net/UnixNetVConnection.cc b/iocore/net/UnixNetVConnection.cc
@@ -361,6 +361,7 @@ write_to_net_io(NetHandler *nh, UnixNetVConnection *vc, EThread *thread)
 {
   NetState *s       = &vc->write;
   ProxyMutex *mutex = thread->mutex.get();
+  Continuation *c   = vc->write.vio.cont;
 
   MUTEX_TRY_LOCK(lock, s->vio.mutex, thread);
 
@@ -445,6 +446,9 @@ write_to_net_io(NetHandler *nh, UnixNetVConnection *vc, EThread *thread)
   if (towrite != ntodo && buf.writer()->write_avail()) {
     if (write_signal_and_update(VC_EVENT_WRITE_READY, vc) != EVENT_CONT) {
       return;
+    } else if (c != s->vio.cont) { /* The write vio was updated in the handler */
+      write_reschedule(nh, vc);
+      return;
     }
 
     ntodo = s->vio.ntodo();

diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc
@@ -1146,6 +1146,8 @@ static const RecordElement RecordsConfig[] =
   ,
   {RECT_CONFIG, "proxy.config.ssl.CA.cert.filename", RECD_STRING, nullptr, RECU_RESTART_TS, RR_NULL, RECC_STR, "^[^[:space:]]*$", RECA_NULL}
   ,
+  {RECT_CONFIG, "proxy.config.ssl.client.alpn_protocols", RECD_STRING, nullptr, RECU_RESTART_TS, RR_NULL, RECC_STR, "^[^[:space:]]*$", RECA_NULL}
+  ,
   {RECT_CONFIG, "proxy.config.ssl.CA.cert.path", RECD_STRING, TS_BUILD_SYSCONFDIR, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
   ,
   {RECT_CONFIG, "proxy.config.ssl.client.verify.server.policy", RECD_STRING, "ENFORCED", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
@@ -1338,6 +1340,8 @@ static const RecordElement RecordsConfig[] =
   ,
   {RECT_CONFIG, "proxy.config.http2.initial_window_size_in", RECD_INT, "65535", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL}
   ,
+  {RECT_CONFIG, "proxy.config.http2.session_initial_window_size_in", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL}
+  ,
   {RECT_CONFIG, "proxy.config.http2.max_frame_size", RECD_INT, "16384", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL}
   ,
   {RECT_CONFIG, "proxy.config.http2.header_table_size", RECD_INT, "4096", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL}
@@ -1348,6 +1352,8 @@ static const RecordElement RecordsConfig[] =
   ,
   {RECT_CONFIG, "proxy.config.http2.no_activity_timeout_in", RECD_INT, "120", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL}
   ,
+  {RECT_CONFIG, "proxy.config.http2.no_activity_timeout_out", RECD_INT, "120", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL}
+  ,
   {RECT_CONFIG, "proxy.config.http2.active_timeout_in", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL}
   ,
   {RECT_CONFIG, "proxy.config.http2.push_diary_size", RECD_INT, "256", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL}

diff --git a/plugins/lua/ts_lua_http_config.c b/plugins/lua/ts_lua_http_config.c
@@ -139,6 +139,7 @@ typedef enum {
   TS_LUA_CONFIG_PLUGIN_VC_DEFAULT_BUFFER_INDEX                = TS_CONFIG_PLUGIN_VC_DEFAULT_BUFFER_INDEX,
   TS_LUA_CONFIG_PLUGIN_VC_DEFAULT_BUFFER_WATER_MARK           = TS_CONFIG_PLUGIN_VC_DEFAULT_BUFFER_WATER_MARK,
   TS_LUA_CONFIG_NET_SOCK_NOTSENT_LOWAT                        = TS_CONFIG_NET_SOCK_NOTSENT_LOWAT,
+  TS_LUA_CONFIG_SSL_CLIENT_ALPN_PROTOCOLS                     = TS_CONFIG_SSL_CLIENT_ALPN_PROTOCOLS,
   TS_LUA_CONFIG_LAST_ENTRY                                    = TS_CONFIG_LAST_ENTRY,
 } TSLuaOverridableConfigKey;
 
@@ -264,6 +265,7 @@ ts_lua_var_item ts_lua_http_config_vars[] = {
   TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_PRIVATE_KEY_FILENAME),
   TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_CA_CERT_FILENAME),
   TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_HTTP_HOST_RESOLUTION_PREFERENCE),
+  TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_SSL_CLIENT_ALPN_PROTOCOLS),
   TS_LUA_MAKE_VAR_ITEM(TS_CONFIG_HTTP_SERVER_MIN_KEEP_ALIVE_CONNS),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_MAX),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_PER_SERVER_CONNECTION_MATCH),

diff --git a/proxy/PoolableSession.h b/proxy/PoolableSession.h
@@ -85,6 +85,7 @@ class PoolableSession : public ProxySession
   bool is_private() const;
 
   virtual void set_netvc(NetVConnection *newvc);
+  virtual bool is_multiplexing() const;
 
   // Used to determine whether the session is for parent proxy
   // it is session to origin server
@@ -237,3 +238,9 @@ PoolableSession::attach_hostname(const char *hostname)
     CryptoContext().hash_immediate(hostname_hash, (unsigned char *)hostname, strlen(hostname));
   }
 }
+
+inline bool
+PoolableSession::is_multiplexing() const
+{
+  return false;
+}
diff --git a/proxy/ProxySession.cc b/proxy/ProxySession.cc
@@ -26,6 +26,8 @@
 #include "ProxySession.h"
 #include "P_SSLNetVConnection.h"
 
+std::map<int, std::function<PoolableSession *()>> ProtocolSessionCreateMap;
+
 ProxySession::ProxySession() : VConnection(nullptr) {}
 
 ProxySession::ProxySession(NetVConnection *vc) : VConnection(nullptr), _vc(vc) {}
@@ -312,3 +314,21 @@ ProxySession::support_sni() const
 {
   return _vc ? _vc->support_sni() : false;
 }
+
+PoolableSession *
+ProxySession::protocol_creation(NetVConnection *netvc)
+{
+  // Figure out what protocol was negotiated
+  int proto_index             = SessionProtocolNameRegistry::INVALID;
+  SSLNetVConnection *sslnetvc = dynamic_cast<SSLNetVConnection *>(netvc);
+  if (sslnetvc) {
+    proto_index = sslnetvc->get_negotiated_protocol_id();
+  }
+  // No ALPN occurred. Assume it was HTTP/1.x and hope for the best
+  if (proto_index == SessionProtocolNameRegistry::INVALID) {
+    proto_index = TS_ALPN_PROTOCOL_INDEX_HTTP_1_1;
+  }
+  auto iter = ProtocolSessionCreateMap.find(proto_index);
+  ink_release_assert(iter != ProtocolSessionCreateMap.end());
+  return iter->second();
+}
diff --git a/proxy/ProxySession.h b/proxy/ProxySession.h
@@ -162,6 +162,8 @@ class ProxySession : public VConnection, public PluginUserArgs<TS_USER_ARGS_SSN>
     return nullptr;
   }
 
+  static PoolableSession *protocol_creation(NetVConnection *netvc);
+
   ////////////////////
   // Members
 

diff --git a/proxy/ProxyTransaction.cc b/proxy/ProxyTransaction.cc
@@ -235,6 +235,34 @@ ProxyTransaction::get_version(HTTPHdr &hdr) const
   return hdr.version_get();
 }
 
+bool
+ProxyTransaction::is_read_closed() const
+{
+  return false;
+}
+
+bool
+ProxyTransaction::expect_send_trailer() const
+{
+  return false;
+}
+
+void
+ProxyTransaction::set_expect_send_trailer()
+{
+}
+
+bool
+ProxyTransaction::expect_receive_trailer() const
+{
+  return false;
+}
+
+void
+ProxyTransaction::set_expect_receive_trailer()
+{
+}
+
 bool
 ProxyTransaction::allow_half_open() const
 {

diff --git a/proxy/ProxyTransaction.h b/proxy/ProxyTransaction.h
@@ -49,6 +49,11 @@ class ProxyTransaction : public VConnection
   virtual void set_inactivity_timeout(ink_hrtime timeout_in);
   virtual void cancel_inactivity_timeout();
   virtual void cancel_active_timeout();
+  virtual bool is_read_closed() const;
+  virtual bool expect_send_trailer() const;
+  virtual void set_expect_send_trailer();
+  virtual bool expect_receive_trailer() const;
+  virtual void set_expect_receive_trailer();
 
   // Implement VConnection interface.
   VIO *do_io_read(Continuation *c, int64_t nbytes = INT64_MAX, MIOBuffer *buf = nullptr) override;