Origin session reuse

doc/admin-guide/files/records.config.en.rst

@@ -3437,6 +3437,21 @@ SSL Termination
   a single segment after ~1 second of inactivity and the record size ramping
   mechanism is repeated again.
 
+.. ts:cv:: CONFIG proxy.config.ssl.origin_session_cache INT 0
+
+   This configuration enables the SSL session cache for the origin server
+   when set to ``1``.
+
+   Setting to ``0`` disables SSL session cache for the origin server.
+
+.. ts:cv:: CONFIG proxy.config.ssl.origin_session_cache.size INT 10240
+
+  This configuration specifies the maximum number of entries
+  the SSL session cache for the origin server may contain.
+
+  Setting a value less than or equal to ``0`` effectively disables
+  SSL session cache for the origin server.
+
 .. ts:cv:: CONFIG proxy.config.ssl.session_cache INT 2
 
    Enables the SSL session cache:

iocore/net/Makefile.am

@@ -150,7 +150,7 @@ libinknet_a_SOURCES = \
 	P_SSLNextProtocolSet.h \
 	P_SSLSNI.h \
 	P_SSLUtils.h \
-        P_SSLClientCoordinator.h \
+	P_SSLClientCoordinator.h \
 	P_SSLClientUtils.h \
 	P_OCSPStapling.h \
 	P_UDPConnection.h \
@@ -168,7 +168,7 @@ libinknet_a_SOURCES = \
 	ProxyProtocol.cc \
 	Socks.cc \
 	SSLCertLookup.cc \
-        SSLClientCoordinator.cc \
+	SSLClientCoordinator.cc \
 	SSLClientUtils.cc \
 	SSLConfig.cc \
 	SSLDiags.cc \

iocore/net/P_SSLConfig.h

@@ -78,6 +78,8 @@ struct SSLConfigParams : public ConfigInfo {
   int configExitOnLoadError;
   int clientCertLevel;
   int verify_depth;
+  int ssl_origin_session_cache;
+  int ssl_origin_session_cache_size;
   int ssl_session_cache; // SSL_SESSION_CACHE_MODE
   int ssl_session_cache_size;
   int ssl_session_cache_num_buckets;
@@ -118,6 +120,8 @@ struct SSLConfigParams : public ConfigInfo {
   static int ssl_handshake_timeout_in;
   char *ssl_ocsp_response_path_only;
 
+  static int origin_session_cache;
+  static size_t origin_session_cache_size;
   static size_t session_cache_number_buckets;
   static size_t session_cache_max_bucket_size;
   static bool session_cache_skip_on_lock_contention;
@@ -218,3 +222,4 @@ struct SSLTicketKeyConfig {
 };
 
 extern SSLSessionCache *session_cache;
+extern SSLOriginSessionCache *origin_sess_cache;

iocore/net/SSLClientUtils.cc

@@ -29,10 +29,13 @@
 #include "P_SSLClientUtils.h"
 #include "YamlSNIConfig.h"
 #include "SSLDiags.h"
+#include "SSLSessionCache.h"
 
 #include <openssl/err.h>
 #include <openssl/pem.h>
 
+SSLOriginSessionCache *origin_sess_cache;
+
 int
 verify_callback(int signature_ok, X509_STORE_CTX *ctx)
 {
@@ -152,6 +155,37 @@ ssl_client_cert_callback(SSL *ssl, void * /*arg*/)
   return 1;
 }
 
+static int
+ssl_new_session_callback(SSL *ssl, SSL_SESSION *sess)
+{
+  const char *tlsext_host_name = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name);
+  if (tlsext_host_name) {
+    std::string sni(tlsext_host_name);
+    origin_sess_cache->insert_session(sni, sess);
+  } else {
+    int sock_fd = SSL_get_fd(ssl);
+    sockaddr_storage addr;
+    socklen_t addr_len = sizeof(addr);
+    std::string addr_s;
+    if (sock_fd >= 0) {
+      getpeername(sock_fd, reinterpret_cast<sockaddr *>(&addr), &addr_len);
+      if (addr.ss_family == AF_INET || addr.ss_family == AF_INET6) {
+        addr_s.assign(reinterpret_cast<char *>(&addr), addr_len);
+        origin_sess_cache->insert_session(addr_s, sess);
+      } else {
+        if (is_debug_tag_set("ssl.origin_session_cache")) {
+          Debug("ssl.origin_session_cache", "unknown address family: %d", addr.ss_family);
+        }
+        return 0;
+      }
+    } else {
+      return 0;
+    }
+  }
+
+  return 1;
+}
+
 SSL_CTX *
 SSLInitClientContext(const SSLConfigParams *params)
 {
@@ -208,6 +242,11 @@ SSLInitClientContext(const SSLConfigParams *params)
 
   SSL_CTX_set_cert_cb(client_ctx, ssl_client_cert_callback, nullptr);
 
+  if (params->ssl_origin_session_cache == 1) {
+    SSL_CTX_set_session_cache_mode(client_ctx, SSL_SESS_CACHE_CLIENT | SSL_SESS_CACHE_NO_AUTO_CLEAR | SSL_SESS_CACHE_NO_INTERNAL);
+    SSL_CTX_sess_set_new_cb(client_ctx, ssl_new_session_callback);
+  }
+
   return client_ctx;
 
 fail:

iocore/net/SSLConfig.cc

@@ -61,6 +61,8 @@ int SSLConfigParams::ssl_ocsp_cache_timeout                 = 3600;
 int SSLConfigParams::ssl_ocsp_request_timeout               = 10;
 int SSLConfigParams::ssl_ocsp_update_period                 = 60;
 int SSLConfigParams::ssl_handshake_timeout_in               = 0;
+int SSLConfigParams::origin_session_cache                   = 0;
+size_t SSLConfigParams::origin_session_cache_size           = 10240;
 size_t SSLConfigParams::session_cache_number_buckets        = 1024;
 bool SSLConfigParams::session_cache_skip_on_lock_contention = false;
 size_t SSLConfigParams::session_cache_max_bucket_size       = 100;
@@ -317,13 +319,17 @@ SSLConfigParams::initialize()
   ats_free(CACertRelativePath);
 
   // SSL session cache configurations
+  REC_ReadConfigInteger(ssl_origin_session_cache, "proxy.config.ssl.origin_session_cache");
+  REC_ReadConfigInteger(ssl_origin_session_cache_size, "proxy.config.ssl.origin_session_cache.size");
   REC_ReadConfigInteger(ssl_session_cache, "proxy.config.ssl.session_cache");
   REC_ReadConfigInteger(ssl_session_cache_size, "proxy.config.ssl.session_cache.size");
   REC_ReadConfigInteger(ssl_session_cache_num_buckets, "proxy.config.ssl.session_cache.num_buckets");
   REC_ReadConfigInteger(ssl_session_cache_skip_on_contention, "proxy.config.ssl.session_cache.skip_cache_on_bucket_contention");
   REC_ReadConfigInteger(ssl_session_cache_timeout, "proxy.config.ssl.session_cache.timeout");
   REC_ReadConfigInteger(ssl_session_cache_auto_clear, "proxy.config.ssl.session_cache.auto_clear");
 
+  SSLConfigParams::origin_session_cache      = ssl_origin_session_cache;
+  SSLConfigParams::origin_session_cache_size = ssl_origin_session_cache_size;
   SSLConfigParams::session_cache_max_bucket_size =
     static_cast<size_t>(ceil(static_cast<double>(ssl_session_cache_size) / ssl_session_cache_num_buckets));
   SSLConfigParams::session_cache_skip_on_lock_contention = ssl_session_cache_skip_on_contention;
@@ -333,6 +339,10 @@ SSLConfigParams::initialize()
     session_cache = new SSLSessionCache();
   }
 
+  if (ssl_origin_session_cache == 1 && ssl_origin_session_cache_size > 0) {
+    origin_sess_cache = new SSLOriginSessionCache();
+  }
+
   // SSL record size
   REC_EstablishStaticConfigInt32(ssl_maxrecord, "proxy.config.ssl.max_record_size");
 

iocore/net/SSLSessionCache.cc

@@ -299,3 +299,75 @@ SSLSessionBucket::removeSession(const SSLSessionID &id)
 SSLSessionBucket::SSLSessionBucket() {}
 
 SSLSessionBucket::~SSLSessionBucket() {}
+
+SSLOriginSessionCache::SSLOriginSessionCache() {}
+
+SSLOriginSessionCache::~SSLOriginSessionCache()
+{
+  for (auto &x : origin_sessions) {
+    SSL_SESSION_free(x.second);
+  }
+}
+
+void
+SSLOriginSessionCache::insert_session(std::string lookup_key, SSL_SESSION *sess)
+{
+  if (is_debug_tag_set("ssl.origin_session_cache")) {
+    Debug("ssl.origin_session_cache", "insert session: %lx = %p", std::hash<std::string>{}(lookup_key), sess);
+  }
+
+  std::unique_lock lock(mutex);
+  auto node = origin_sessions.find(lookup_key);
+  if (node != origin_sessions.end()) {
+    SSL_SESSION_free(node->second);
+  } else if (origin_sessions.size() >= SSLConfigParams::origin_session_cache_size) {
+    remove_oldest_session(lock);
+  }
+  origin_sessions[lookup_key] = sess;
+}
+
+void
+SSLOriginSessionCache::remove_session(std::string lookup_key)
+{
+  if (is_debug_tag_set("ssl.origin_session_cache")) {
+    Debug("ssl.origin_session_cache", "remove session: %lx", std::hash<std::string>{}(lookup_key));
+  }
+
+  std::unique_lock lock(mutex);
+  auto node = origin_sessions.find(lookup_key);
+  if (node != origin_sessions.end()) {
+    SSL_SESSION_free(node->second);
+    origin_sessions.erase(node);
+  }
+}
+
+SSL_SESSION *
+SSLOriginSessionCache::get_session(std::string lookup_key)
+{
+  if (is_debug_tag_set("ssl.origin_session_cache")) {
+    Debug("ssl.origin_session_cache", "get session: %lx", std::hash<std::string>{}(lookup_key));
+  }
+
+  std::shared_lock lock(mutex);
+  auto node = origin_sessions.find(lookup_key);
+  if (node == origin_sessions.end()) {
+    return nullptr;
+  }
+  return node->second;
+}
+
+void
+SSLOriginSessionCache::remove_oldest_session(const std::unique_lock<std::shared_mutex> &lock)
+{
+  // Caller must hold the bucket shared_mutex with unique_lock.
+  ink_assert(lock.owns_lock());
+
+  auto node = origin_sessions.begin();
+
+  if (is_debug_tag_set("ssl.origin_session_cache")) {
+    Debug("ssl.origin_session_cache", "remove oldest session: %lx = %p", std::hash<std::string>{}(node->first), node->second);
+  }
+
+  SSL_SESSION_free(node->second);
+  origin_sessions.erase(node);
+}

iocore/net/SSLSessionCache.h

@@ -198,3 +198,20 @@ class SSLSessionCache
   SSLSessionBucket *session_bucket = nullptr;
   size_t nbuckets;
 };
+
+class SSLOriginSessionCache
+{
+public:
+  SSLOriginSessionCache();
+  ~SSLOriginSessionCache();
+
+  void insert_session(std::string lookup_key, SSL_SESSION *sess);
+  void remove_session(std::string lookup_key);
+  SSL_SESSION *get_session(std::string lookup_key);
+
+private:
+  mutable std::shared_mutex mutex;
+  std::map<std::string, SSL_SESSION *> origin_sessions;
+
+  void remove_oldest_session(const std::unique_lock<std::shared_mutex> &lock);
+};

iocore/net/SSLStats.cc

@@ -217,6 +217,10 @@ SSLInitializeStatistics()
   RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.early_data_received", RECD_INT, RECP_PERSISTENT,
                      (int)ssl_early_data_received_count, RecRawStatSyncCount);
 
+  // Origin Server Session Reuse stats
+  RecRegisterRawStat(ssl_rsb, RECT_PROCESS, "proxy.process.ssl.origin_session_reused", RECD_INT, RECP_PERSISTENT,
+                     (int)ssl_origin_session_reused_count, RecRawStatSyncCount);
+
   // Get and register the SSL cipher stats. Note that we are using the default SSL context to obtain
   // the cipher list. This means that the set of ciphers is fixed by the build configuration and not
   // filtered by proxy.config.ssl.server.cipher_suite. This keeps the set of cipher suites stable across

iocore/net/SSLStats.h

@@ -85,6 +85,7 @@ enum SSL_Stats {
   ssl_session_cache_lock_contention,
   ssl_session_cache_new_session,
   ssl_early_data_received_count, // how many times we received early data
+  ssl_origin_session_reused_count,
 
   /* error stats */
   ssl_error_syscall,

iocore/net/SSLUtils.cc

@@ -1888,8 +1888,45 @@ ssl_error_t
 SSLConnect(SSL *ssl)
 {
   ERR_clear_error();
+
+  SSL_SESSION *sess = nullptr;
+  std::string lookup_key;
+  if (SSLConfigParams::origin_session_cache == 1 && SSLConfigParams::origin_session_cache_size > 0) {
+    const char *tlsext_host_name = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name);
+    if (tlsext_host_name) {
+      lookup_key.assign(tlsext_host_name);
+      sess = origin_sess_cache->get_session(lookup_key);
+    } else {
+      int sock_fd = SSL_get_fd(ssl);
+      sockaddr_storage addr;
+      socklen_t addr_len = sizeof(addr);
+      if (sock_fd >= 0) {
+        getpeername(sock_fd, reinterpret_cast<sockaddr *>(&addr), &addr_len);
+        if (addr.ss_family == AF_INET || addr.ss_family == AF_INET6) {
+          lookup_key.assign(reinterpret_cast<char *>(&addr), addr_len);
+          sess = origin_sess_cache->get_session(lookup_key);
+        }
+      }
+    }
+  }
+
+  if (sess) {
+    SSL_set_session(ssl, sess);
+  }
+
   int ret = SSL_connect(ssl);
+
   if (ret > 0) {
+    if (sess && SSL_session_reused(ssl)) {
+      SSL_INCREMENT_DYN_STAT(ssl_origin_session_reused_count);
+      if (is_debug_tag_set("ssl.origin_session_cache")) {
+        Debug("ssl.origin_session_cache", "reused session to origin: %lx = %p", std::hash<std::string>{}(lookup_key), sess);
+      }
+    } else {
+      if (is_debug_tag_set("ssl.origin_session_cache")) {
+        Debug("ssl.origin_session_cache", "new session to origin: %lx = %p", std::hash<std::string>{}(lookup_key), sess);
+      }
+    }
     return SSL_ERROR_NONE;
   }
   int ssl_error = SSL_get_error(ssl, ret);

mgmt/RecordsConfig.cc

@@ -1132,6 +1132,10 @@ static const RecordElement RecordsConfig[] =
   ,
   {RECT_CONFIG, "proxy.config.ssl.client.sni_policy", RECD_STRING, "host", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
   ,
+  {RECT_CONFIG, "proxy.config.ssl.origin_session_cache", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
+  ,
+  {RECT_CONFIG, "proxy.config.ssl.origin_session_cache.size", RECD_INT, "10240", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
+  ,
   {RECT_CONFIG, "proxy.config.ssl.session_cache", RECD_INT, "2", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
   ,
   {RECT_CONFIG, "proxy.config.ssl.session_cache.size", RECD_INT, "102400", RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL}