HDDS-16347. Stop expanding _HOST to the bind host in HTTP SPNEGO principals

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/BaseHttpServer.java

@@ -36,10 +36,13 @@
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.net.InetAddresses;
 import java.io.File;
 import java.io.IOException;
+import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.URI;
+import java.net.UnknownHostException;
 import java.util.Map;
 import java.util.Optional;
 import java.util.OptionalInt;
@@ -118,6 +121,11 @@ public BaseHttpServer(MutableConfigurationSource conf, String name)
         // Ozone config prefix must be set to avoid AuthenticationFilter
         // fall back to default one form hadoop.http.authentication.
         builder.authFilterConfigurationPrefix(getHttpAuthConfigPrefix());
+        // Without an explicit host, HttpServer2 expands _HOST in the SPNEGO
+        // principal to the host of its first endpoint, which is the bind
+        // address: a wildcard or interface-specific bind host does not name
+        // this node and matches no keytab entry.
+        builder.hostName(getSpnegoHost());
         if (httpAuthType.equals("kerberos")) {
           builder.setSecurityEnabled(true);
           builder.setUsernameConfKey(getSpnegoPrincipal());
@@ -329,6 +337,28 @@ protected InetSocketAddress getBindAddress(String bindHostKey,
         getHostPortString(hostName, addressPort.orElse(bindPortdefault)));
   }
 
+  /**
+   * Returns the host that replaces {@code _HOST} in this server's SPNEGO
+   * principal: the host of the advertised HTTP or HTTPS address unless it is
+   * a wildcard, otherwise the canonical name of the local host.
+   */
+  protected String getSpnegoHost() throws UnknownHostException {
+    for (String addressKey : new String[] {getHttpAddressKey(), getHttpsAddressKey()}) {
+      Optional<String> host = getHostNameFromConfigKeys(conf, addressKey);
+      if (host.isPresent() && !isWildcardHost(host.get())) {
+        return host.get();
+      }
+    }
+    return InetAddress.getLocalHost().getCanonicalHostName();
+  }
+
+  private static boolean isWildcardHost(String host) {
+    // Judged on the text first: InetAddresses.forString rejects a scoped
+    // literal unless this host owns the named interface.
+    return host.indexOf('%') < 0 && InetAddresses.isInetAddress(host)
+        && InetAddresses.forString(host).isAnyLocalAddress();
+  }
+
   /**
    * Retrieve the socket address that should be used by clients to connect
    * to the  HTTPS web interface.

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2.java

@@ -221,7 +221,7 @@ public final class HttpServer2 implements FilterContainer {
   // gets stored.
   public static final String CONF_CONTEXT_ATTRIBUTE = "hadoop.conf";
   private static final String ADMINS_ACL = "admins.acl";
-  private static final String SPNEGO_FILTER = "SpnegoFilter";
+  static final String SPNEGO_FILTER = "SpnegoFilter";
   private static final String NO_CACHE_FILTER = "NoCacheFilter";
 
   private static final String BIND_ADDRESS = "bind.address";

hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/http/TestBaseHttpServer.java

@@ -27,18 +27,24 @@
 import java.net.InetSocketAddress;
 import java.net.URI;
 import java.nio.file.Path;
-import org.apache.commons.lang3.NotImplementedException;
+import java.util.stream.Stream;
 import org.apache.hadoop.hdds.conf.MutableConfigurationSource;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
 import org.apache.ozone.test.GenericTestUtils.PortAllocator;
+import org.eclipse.jetty.ee8.servlet.FilterHolder;
 import org.eclipse.jetty.ee8.webapp.WebAppContext;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInstance;
 import org.junit.jupiter.api.io.TempDir;
 import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.CsvSource;
 import org.junit.jupiter.params.provider.EnumSource;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
 
 /**
  * Test Common ozone/hdds web methods.
@@ -54,6 +60,10 @@ class TestBaseHttpServer {
   private static final int BIND_PORT_HTTP_DEFAULT = PortAllocator.getFreePort();
   private static final int BIND_PORT_HTTPS_DEFAULT = PortAllocator.getFreePort();
   private static final String ENABLED_KEY = "enabled";
+  private static final String HTTP_AUTH_TYPE_KEY = "http.auth.type";
+  private static final String HTTP_AUTH_CONFIG_PREFIX = "http.auth.";
+  private static final String KEYTAB_KEY = "http.kerberos.keytab";
+  private static final String SPNEGO_PRINCIPAL_KEY = "http.kerberos.principal";
 
   private String hostname;
 
@@ -206,10 +216,7 @@ void usesPerServerTempSubdirectory() throws Exception {
     BaseHttpServer subject = new TestingHttpServer(conf);
     try {
       subject.start();
-      Field field = BaseHttpServer.class.getDeclaredField("httpServer");
-      field.setAccessible(true);
-      WebAppContext webAppContext =
-          ((HttpServer2) field.get(subject)).getWebAppContext();
+      WebAppContext webAppContext = httpServer2Of(subject).getWebAppContext();
       assertEquals(new File(tempDir.toFile(), "testing").getCanonicalFile(),
           webAppContext.getTempDirectory().getCanonicalFile());
       assertTrue(webAppContext.isPersistTempDirectory());
@@ -232,17 +239,87 @@ void honoursOperatorConfiguredTempDir() throws Exception {
     BaseHttpServer subject = new TestingHttpServer(conf);
     try {
       subject.start();
-      Field field = BaseHttpServer.class.getDeclaredField("httpServer");
-      field.setAccessible(true);
-      WebAppContext webAppContext =
-          ((HttpServer2) field.get(subject)).getWebAppContext();
+      WebAppContext webAppContext = httpServer2Of(subject).getWebAppContext();
       assertEquals(operatorTempDir.getCanonicalFile(),
           webAppContext.getTempDirectory().getCanonicalFile());
     } finally {
       subject.stop();
     }
   }
 
+  static Stream<Arguments> advertisedAddressesNamingNoHost() {
+    return Stream.of(
+        Arguments.of(null, null),
+        Arguments.of("0.0.0.0:9874", "0.0.0.0:9875"),
+        Arguments.of("[::]:9874", "[::]:9875"));
+  }
+
+  @ParameterizedTest
+  @MethodSource("advertisedAddressesNamingNoHost")
+  void spnegoHostFallsBackToLocalHostWithoutAdvertisedHost(
+      String httpAddress, String httpsAddress) throws Exception {
+    BaseHttpServer subject = newDisabledServer(httpAddress, httpsAddress);
+
+    assertEquals(InetAddress.getLocalHost().getCanonicalHostName(),
+        subject.getSpnegoHost());
+  }
+
+  @ParameterizedTest
+  @CsvSource({
+      "om1.example.com:9874, , om1.example.com",
+      "0.0.0.0:9874, om1.example.com:9875, om1.example.com",
+      "[2001:db8::1]:9874, , 2001:db8::1",
+  })
+  void spnegoHostIsAdvertisedHost(String httpAddress, String httpsAddress,
+      String expectedHost) throws Exception {
+    BaseHttpServer subject = newDisabledServer(httpAddress, httpsAddress);
+
+    assertEquals(expectedHost, subject.getSpnegoHost());
+  }
+
+  /**
+   * The bind host is what {@link HttpServer2} expands {@code _HOST} to when
+   * it is not told otherwise; none of these name the host the keytab was
+   * issued for.
+   */
+  @ParameterizedTest
+  @ValueSource(strings = {"0.0.0.0", "::", "127.0.0.1"})
+  void spnegoPrincipalNamesAdvertisedHostNotBindHost(String bindHost)
+      throws Exception {
+    MutableConfigurationSource conf = newConfig(HttpConfig.Policy.HTTP_ONLY);
+    conf.set(BIND_HOST_HTTP_KEY, bindHost);
+    conf.set(ADDRESS_HTTP_KEY, "om1.example.com:" + BIND_PORT_HTTP_DEFAULT);
+    conf.set(HTTP_AUTH_TYPE_KEY, "kerberos");
+    conf.set(SPNEGO_PRINCIPAL_KEY, "HTTP/_HOST@EXAMPLE.COM");
+
+    try (BaseHttpServer subject = new SecureTestingHttpServer(conf)) {
+      FilterHolder spnegoFilter = httpServer2Of(subject).getWebAppContext()
+          .getServletHandler().getFilter(HttpServer2.SPNEGO_FILTER);
+      assertEquals("HTTP/om1.example.com@EXAMPLE.COM",
+          spnegoFilter.getInitParameter(KerberosAuthenticationHandler.PRINCIPAL));
+    }
+  }
+
+  private BaseHttpServer newDisabledServer(String httpAddress,
+      String httpsAddress) throws IOException {
+    MutableConfigurationSource conf = newConfig(HttpConfig.Policy.HTTP_ONLY);
+    conf.setBoolean(ENABLED_KEY, false);
+    if (httpAddress != null) {
+      conf.set(ADDRESS_HTTP_KEY, httpAddress);
+    }
+    if (httpsAddress != null) {
+      conf.set(ADDRESS_HTTPS_KEY, httpsAddress);
+    }
+    return new TestingHttpServer(conf);
+  }
+
+  private static HttpServer2 httpServer2Of(BaseHttpServer server)
+      throws ReflectiveOperationException {
+    Field field = BaseHttpServer.class.getDeclaredField("httpServer");
+    field.setAccessible(true);
+    return (HttpServer2) field.get(server);
+  }
+
   private MutableConfigurationSource newConfig(HttpConfig.Policy policy) {
     MutableConfigurationSource conf = new OzoneConfiguration();
     conf.set(OzoneConfigKeys.OZONE_HTTP_BASEDIR, tempDir.toString());
@@ -293,12 +370,12 @@ protected int getHttpsBindPortDefault() {
 
     @Override
     protected String getKeytabFile() {
-      throw new NotImplementedException();
+      return KEYTAB_KEY;
     }
 
     @Override
     protected String getSpnegoPrincipal() {
-      throw new NotImplementedException();
+      return SPNEGO_PRINCIPAL_KEY;
     }
 
     @Override
@@ -308,12 +385,25 @@ protected String getEnabledKey() {
 
     @Override
     protected String getHttpAuthType() {
-      throw new NotImplementedException();
+      return HTTP_AUTH_TYPE_KEY;
     }
 
     @Override
     protected String getHttpAuthConfigPrefix() {
-      throw new NotImplementedException();
+      return HTTP_AUTH_CONFIG_PREFIX;
+    }
+  }
+
+  /** Reports HTTP security as enabled without a Kerberos login. */
+  private static final class SecureTestingHttpServer extends TestingHttpServer {
+
+    SecureTestingHttpServer(MutableConfigurationSource conf) throws IOException {
+      super(conf);
+    }
+
+    @Override
+    public boolean isSecurityEnabled() {
+      return true;
     }
   }
 

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayWebAdminServer.java

@@ -126,7 +126,7 @@ private void addSecretAuthentication(MutableConfigurationSource conf)
             conf.get(OZONE_S3G_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL);
         if (!Strings.isNullOrEmpty(principalInConf)) {
           params.put("kerberos.principal", SecurityUtil.getServerPrincipal(
-              principalInConf, conf.get(OZONE_S3G_WEBADMIN_HTTP_BIND_HOST_KEY)));
+              principalInConf, getSpnegoHost()));
         }
         String httpKeytab = conf.get(OZONE_S3G_KEYTAB_FILE);
         if (!Strings.isNullOrEmpty(httpKeytab)) {

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestS3GatewayHttpServers.java

@@ -23,12 +23,20 @@
 
 import java.io.IOException;
 import java.nio.file.Path;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.hdds.conf.MutableConfigurationSource;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.s3secret.S3SecretConfigKeys;
 import org.apache.hadoop.ozone.s3sts.S3STSConfigKeys;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
 import org.apache.ozone.test.GenericTestUtils.PortAllocator;
+import org.eclipse.jetty.ee8.servlet.FilterHolder;
 import org.eclipse.jetty.ee8.webapp.WebAppContext;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 
@@ -46,6 +54,36 @@ class TestS3GatewayHttpServers {
   @TempDir
   private Path tempDir;
 
+  /**
+   * Switching {@link UserGroupInformation} to Kerberos reads the default
+   * realm, which the JDK resolves from these properties when there is no
+   * krb5.conf, as in CI.
+   */
+  private static String previousKerberosRealm;
+  private static String previousKerberosKdc;
+
+  @BeforeAll
+  static void fakeKerberosRealm() {
+    previousKerberosRealm = System.getProperty("java.security.krb5.realm");
+    previousKerberosKdc = System.getProperty("java.security.krb5.kdc");
+    System.setProperty("java.security.krb5.realm", "EXAMPLE.COM");
+    System.setProperty("java.security.krb5.kdc", "localhost");
+  }
+
+  @AfterAll
+  static void clearKerberosRealm() {
+    if (previousKerberosRealm == null) {
+      System.clearProperty("java.security.krb5.realm");
+    } else {
+      System.setProperty("java.security.krb5.realm", previousKerberosRealm);
+    }
+    if (previousKerberosKdc == null) {
+      System.clearProperty("java.security.krb5.kdc");
+    } else {
+      System.setProperty("java.security.krb5.kdc", previousKerberosKdc);
+    }
+  }
+
   /**
    * An enabled server gets Jetty's CDI integration attribute, without which
    * Weld falls through to its legacy Jetty container and fails to start the
@@ -77,6 +115,36 @@ void disabledStsServerIsCreatedWithoutCdiIntegration() {
     assertDoesNotThrow(() -> new S3STSHttpServer(conf, "s3g-sts"));
   }
 
+  /**
+   * The secret endpoint expands {@code _HOST} in its SPNEGO principal from the
+   * advertised web admin address, not from the bind host, which here is the
+   * IPv6 wildcard that no keytab entry can match.
+   */
+  @Test
+  void secretEndpointPrincipalNamesAdvertisedHost() throws Exception {
+    OzoneConfiguration conf = newConf();
+    conf.set(S3GatewayConfigKeys.OZONE_S3G_WEBADMIN_HTTP_BIND_HOST_KEY, "::");
+    conf.set(S3GatewayConfigKeys.OZONE_S3G_WEBADMIN_HTTP_ADDRESS_KEY,
+        "s3g1.example.com:" + PortAllocator.getFreePort());
+    conf.setBoolean(S3SecretConfigKeys.OZONE_S3G_SECRET_HTTP_ENABLED_KEY, true);
+    conf.set(S3GatewayConfigKeys.OZONE_S3G_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL,
+        "HTTP/_HOST@EXAMPLE.COM");
+
+    Configuration hadoopConf = new Configuration();
+    hadoopConf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+        "kerberos");
+    UserGroupInformation.setConfiguration(hadoopConf);
+    try (ExposedS3GatewayWebAdminServer server =
+        new ExposedS3GatewayWebAdminServer(conf)) {
+      FilterHolder secretFilter = server.context().getServletHandler()
+          .getFilter("secretAuthentication");
+      assertEquals("HTTP/s3g1.example.com@EXAMPLE.COM",
+          secretFilter.getInitParameter(KerberosAuthenticationHandler.PRINCIPAL));
+    } finally {
+      UserGroupInformation.reset();
+    }
+  }
+
   private OzoneConfiguration newConf() {
     OzoneConfiguration conf = new OzoneConfiguration();
     conf.set(OzoneConfigKeys.OZONE_HTTP_BASEDIR,
@@ -97,4 +165,18 @@ WebAppContext context() {
       return getWebAppContext();
     }
   }
+
+  /** Exposes the web app context, which BaseHttpServer keeps protected. */
+  private static final class ExposedS3GatewayWebAdminServer
+      extends S3GatewayWebAdminServer {
+
+    ExposedS3GatewayWebAdminServer(MutableConfigurationSource conf)
+        throws IOException {
+      super(conf, "s3g-web");
+    }
+
+    WebAppContext context() {
+      return getWebAppContext();
+    }
+  }
 }