HBASE-28911: Automatic SSL keystore reloading for HttpServer

hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java

@@ -35,6 +35,8 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+import java.util.Timer;
 import java.util.stream.Collectors;
 import javax.servlet.Filter;
 import javax.servlet.FilterChain;
@@ -62,6 +64,8 @@
 import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
 import org.apache.hadoop.security.authorize.AccessControlList;
 import org.apache.hadoop.security.authorize.ProxyUsers;
+import org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory;
+import org.apache.hadoop.security.ssl.FileMonitoringTimerTask;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.yetus.audience.InterfaceAudience;
@@ -179,6 +183,7 @@ public class HttpServer implements FilterContainer {
       .build();
 
   private final AccessControlList adminsAcl;
+  private Optional<Timer> configurationChangeMonitor = Optional.empty();
 
   protected final Server webServer;
   protected String appDir;
@@ -466,6 +471,15 @@ public HttpServer build() throws IOException {
             LOG.debug("Excluded SSL Cipher List:" + excludeCiphers);
           }
 
+          long storesReloadInterval =
+            conf.getLong(FileBasedKeyStoresFactory.SSL_STORES_RELOAD_INTERVAL_TPL_KEY,
+              FileBasedKeyStoresFactory.DEFAULT_SSL_STORES_RELOAD_INTERVAL);
+
+          if (storesReloadInterval > 0 && (keyStore != null || trustStore != null)) {
+            server.configurationChangeMonitor =
+              Optional.of(this.makeConfigurationChangeMonitor(storesReloadInterval, sslCtxFactory));
+          }
+
           listener = new ServerConnector(server.webServer,
             new SslConnectionFactory(sslCtxFactory, HttpVersion.HTTP_1_1.toString()),
             new HttpConnectionFactory(httpsConfig));
@@ -496,6 +510,25 @@ public HttpServer build() throws IOException {
 
     }
 
+    private Timer makeConfigurationChangeMonitor(long reloadInterval,
+      SslContextFactory.Server sslContextFactory) {
+      Timer timer = new Timer("SSL Certificates Store Monitor", true);
+      //
+      // The Jetty SSLContextFactory provides a 'reload' method which will reload both
+      // truststore and keystore certificates.
+      //
+      timer.schedule(new FileMonitoringTimerTask(Paths.get(keyStore), path -> {
+        LOG.info("Reloading certificates from store keystore " + keyStore);
+        try {
+          sslContextFactory.reload(factory -> {
+          });
+        } catch (Exception ex) {
+          LOG.error("Failed to reload SSL keystore certificates", ex);
+        }
+      }, null), reloadInterval, reloadInterval);
+      return timer;
+    }
+
   }
 
   /**
@@ -1291,6 +1324,15 @@ void openListeners() throws Exception {
    */
   public void stop() throws Exception {
     MultiException exception = null;
+    if (this.configurationChangeMonitor.isPresent()) {
+      try {
+        this.configurationChangeMonitor.get().cancel();
+      } catch (Exception e) {
+        LOG.error("Error while canceling configuration monitoring timer for webapp"
+          + webAppContext.getDisplayName(), e);
+        exception = addMultiException(exception, e);
+      }
+    }
     for (ListenerInfo li : listeners) {
       if (!li.isManaged) {
         continue;

hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestSSLHttpServer.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.hbase.http;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
 
 import java.io.ByteArrayOutputStream;
 import java.io.File;
@@ -26,6 +27,9 @@
 import java.net.URI;
 import java.net.URL;
 import java.security.GeneralSecurityException;
+import java.security.KeyPair;
+import java.security.KeyStore;
+import java.security.cert.X509Certificate;
 import javax.net.ssl.HttpsURLConnection;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileUtil;
@@ -37,6 +41,7 @@
 import org.apache.hadoop.hbase.testclassification.MiscTests;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory;
 import org.apache.hadoop.security.ssl.SSLFactory;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -46,6 +51,11 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.hbase.thirdparty.org.eclipse.jetty.server.AbstractConnector;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.server.ConnectionFactory;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.server.SslConnectionFactory;
+import org.apache.hbase.thirdparty.org.eclipse.jetty.util.ssl.SslContextFactory;
+
 /**
  * This testcase issues SSL certificates configures the HttpServer to serve HTTPS using the created
  * certficates and calls an echo servlet using the corresponding HTTPS URL.
@@ -65,6 +75,7 @@ public class TestSSLHttpServer extends HttpServerFunctionalTest {
   private static String sslConfDir;
   private static SSLFactory clientSslFactory;
   private static HBaseCommonTestingUtil HTU;
+  private static long reloadInterval;
 
   @BeforeClass
   public static void setup() throws Exception {
@@ -74,6 +85,9 @@ public static void setup() throws Exception {
 
     serverConf.setInt(HttpServer.HTTP_MAX_THREADS, TestHttpServer.MAX_THREADS);
     serverConf.setBoolean(ServerConfigurationKeys.HBASE_SSL_ENABLED_KEY, true);
+    reloadInterval = 1000;
+    serverConf.setLong(FileBasedKeyStoresFactory.SSL_STORES_RELOAD_INTERVAL_TPL_KEY,
+      reloadInterval);
 
     keystoresDir = new File(HTU.getDataTestDir("keystore").toString());
     keystoresDir.mkdirs();
@@ -131,6 +145,45 @@ public void testSecurityHeaders() throws IOException, GeneralSecurityException {
       conn.getHeaderField("Content-Security-Policy"));
   }
 
+  @Test(timeout = 60000)
+  public void testReloadKeyStore() throws Exception {
+    String serverKS = keystoresDir + "/serverKS.jks";
+    String serverPassword = "serverP";
+
+    KeyStore oldKeyStore = KeyStoreTestUtil.loadKeyStore(serverKS, serverPassword.toCharArray());
+
+    KeyPair sKP = KeyStoreTestUtil.generateKeyPair("RSA");
+    X509Certificate sCert =
+      KeyStoreTestUtil.generateCertificate("CN=localhost, O=server", sKP, 30, "SHA1withRSA");
+    KeyStoreTestUtil.createKeyStore(serverKS, serverPassword, "server", sKP.getPrivate(), sCert);
+    KeyStore newKeyStore = KeyStoreTestUtil.loadKeyStore(serverKS, serverPassword.toCharArray());
+
+    Thread.sleep((reloadInterval + 1000));
+
+    for (AbstractConnector connector : server.getServerConnectors()) {
+      if (connector != null) {
+        for (ConnectionFactory connectionFactory : connector.getConnectionFactories()) {
+          if (connectionFactory instanceof SslConnectionFactory) {
+            SslContextFactory sslContextFactory =
+              ((SslConnectionFactory) connectionFactory).getSslContextFactory();
+            KeyStore currentKeyStore = sslContextFactory.getKeyStore();
+
+            assertNotEquals(currentKeyStore.getCertificate("server"),
+              oldKeyStore.getCertificate("server"));
+            assertNotEquals(currentKeyStore.getKey("server", serverPassword.toCharArray()),
+              oldKeyStore.getKey("server", serverPassword.toCharArray()));
+
+            assertEquals(currentKeyStore.getCertificate("server"),
+              newKeyStore.getCertificate("server"));
+            assertEquals(currentKeyStore.getKey("server", serverPassword.toCharArray()),
+              newKeyStore.getKey("server", serverPassword.toCharArray()));
+
+          }
+        }
+      }
+    }
+  }
+
   private static String readOut(URL url) throws Exception {
     HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
     conn.setSSLSocketFactory(clientSslFactory.createSSLSocketFactory());

hbase-http/src/test/java/org/apache/hadoop/hbase/http/ssl/KeyStoreTestUtil.java

@@ -20,6 +20,7 @@
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.Writer;
 import java.math.BigInteger;
 import java.net.URL;
@@ -173,6 +174,34 @@ public static void createKeyStore(String filename, String password, String keyPa
     saveKeyStore(ks, filename, password);
   }
 
+  /**
+   * Load a keystore from the jks file.
+   * @param filename String filename to load keystore
+   * @param password char array password to load keystore
+   * @throws GeneralSecurityException for any error with the security APIs
+   * @throws IOException              if there is an I/O error saving the file
+   */
+  public static KeyStore loadKeyStore(String filename, char[] password)
+    throws GeneralSecurityException, IOException {
+    return loadKeyStore(filename, password, "JKS");
+  }
+
+  /**
+   * Load a keystore from the file.
+   * @param filename     String filename to load keystore
+   * @param password     char array password to load keystore
+   * @param keystoreType String keystore file type (e.g. "JKS")
+   * @throws GeneralSecurityException for any error with the security APIs
+   * @throws IOException              if there is an I/O error saving the file
+   */
+  public static KeyStore loadKeyStore(String filename, char[] password, String keystoreType)
+    throws GeneralSecurityException, IOException {
+    InputStream inputStream = java.nio.file.Files.newInputStream(new File(filename).toPath());
+    KeyStore ks = KeyStore.getInstance(keystoreType);
+    ks.load(inputStream, password);
+    return ks;
+  }
+
   /**
    * Creates a truststore with a single certificate and saves it to a file. This method uses the
    * default JKS truststore type.

hbase-http/src/test/resources/hbase-default.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+  <property>
+    <name>hbase.defaults.for.version.skip</name>
+    <value>true</value>
+  </property>
+</configuration>