HDDS-1596. Create service endpoint to download configuration from SCM

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java

@@ -47,19 +47,43 @@ public class OzoneConfiguration extends Configuration {
   }
 
   public OzoneConfiguration() {
+    this(false);
+  }
+
+  private OzoneConfiguration(boolean justTheDefaults) {
     OzoneConfiguration.activate();
     loadDefaults();
+    if (!justTheDefaults) {
+      loadConfigFiles();
+    }
+  }
+
+  private void loadConfigFiles() {
+    addResource("ozone-global.xml");
+    addResource("ozone-site.xml");
   }
 
   public OzoneConfiguration(Configuration conf) {
+    this(conf, false);
+  }
+
+  private OzoneConfiguration(Configuration conf, boolean justTheDefaults) {
     super(conf);
     //load the configuration from the classloader of the original conf.
     setClassLoader(conf.getClassLoader());
     if (!(conf instanceof OzoneConfiguration)) {
       loadDefaults();
+      //here we load the REAL configuration.
+      if (!justTheDefaults) {
+        loadConfigFiles();
+      }
     }
   }
 
+  public static OzoneConfiguration createWithDefaultsOnly() {
+    return new OzoneConfiguration(true);
+  }
+
   private void loadDefaults() {
     try {
       //there could be multiple ozone-default-generated.xml files on the
@@ -74,7 +98,6 @@ private void loadDefaults() {
     } catch (IOException e) {
       e.printStackTrace();
     }
-    addResource("ozone-site.xml");
   }
 
   public List<Property> readPropertyFromXml(URL url) throws JAXBException {
@@ -316,4 +339,9 @@ public Properties getAllPropertiesByTag(String tag) {
     }
     return props;
   }
+
+  @Override
+  public synchronized Properties getProps() {
+    return super.getProps();
+  }
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/discovery/DiscoveryUtil.java

@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.discovery;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.net.URL;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility to download ozone configuration from SCM.
+ */
+public final class DiscoveryUtil {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(DiscoveryUtil.class);
+
+  public static final String OZONE_GLOBAL_XML = "ozone-global.xml";
+
+  private DiscoveryUtil() {
+  }
+
+  /**
+   * Download ozone-global.conf from SCM to the local HADOOP_CONF_DIR.
+   */
+  public static boolean loadGlobalConfig(OzoneConfiguration conf) {
+    String hadoopConfDir = System.getenv("HADOOP_CONF_DIR");
+    if (hadoopConfDir == null || hadoopConfDir.isEmpty()) {
+      LOG.warn(
+          "HADOOP_CONF_DIR is not set, can't download ozone-global.xml from "
+              + "SCM.");
+      return false;
+    }
+    if (conf.get("ozone.scm.names") == null) {
+      LOG.warn("ozone.scm.names is not set. Can't download config from scm.");
+      return false;
+    }
+    for (int i = 0; i < 60; i++) {
+      for (String scmHost : conf.getStrings("ozone.scm.names")) {
+        String configOrigin =
+            String.format("http://%s:9876/discovery/config", scmHost);
+        File destinationFile = new File(hadoopConfDir, OZONE_GLOBAL_XML);
+
+        try {
+          LOG.info("Downloading {} to {}", configOrigin,
+              destinationFile.getAbsolutePath());
+          URL confUrl = new URL(configOrigin);
+          ReadableByteChannel rbc = Channels.newChannel(confUrl.openStream());
+          FileOutputStream fos =
+              new FileOutputStream(
+                  destinationFile);
+          fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
+          return true;
+        } catch (Exception ex) {
+          LOG.error("Can't download config from " + configOrigin, ex);
+        }
+      }
+      LOG.warn(
+          "Configuration download was unsuccessful. Let's wait 5 seconds and"
+              + " retry.");
+      try {
+        Thread.sleep(5000);
+      } catch (InterruptedException e) {
+        LOG.error("Polling the config file upload is interrupted", e);
+      }
+    }
+    return false;
+  }
+}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/discovery/package-info.java

@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.hdds.discovery;
+
+/*
+ * Discovery/config service related classes.
+ */

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java

@@ -25,6 +25,7 @@
 import org.apache.hadoop.hdds.cli.GenericCli;
 import org.apache.hadoop.hdds.cli.HddsVersionProvider;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.discovery.DiscoveryUtil;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
 import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
@@ -140,7 +141,12 @@ public Void call() throws Exception {
       StringUtils
           .startupShutdownMessage(HddsDatanodeService.class, args, LOG);
     }
-    start(createOzoneConfiguration());
+    OzoneConfiguration ozoneConfiguration = createOzoneConfiguration();
+    if (DiscoveryUtil.loadGlobalConfig(ozoneConfiguration)) {
+      //reload the configuration with the downloaded new configs.
+      ozoneConfiguration = createOzoneConfiguration();
+    }
+    start(ozoneConfiguration);
     join();
     return null;
   }

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/BaseHttpServer.java

@@ -33,6 +33,7 @@
 import javax.servlet.http.HttpServlet;
 import java.io.IOException;
 import java.net.InetSocketAddress;
+import java.util.Map;
 import java.util.Optional;
 
 import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys;
@@ -127,6 +128,18 @@ protected void addServlet(String servletName, String pathSpec,
     httpServer.addServlet(servletName, pathSpec, clazz);
   }
 
+  /**
+   * Add a servlet to BaseHttpServer.
+   *
+   * @param servletName The name of the servlet
+   * @param pathSpec    The path spec for the servlet
+   * @param clazz       The servlet class
+   */
+  protected void addInternalServlet(String servletName, String pathSpec,
+      Class<? extends HttpServlet> clazz, Map<String, String> initParams) {
+    httpServer.addInternalServlet(servletName, pathSpec, clazz, initParams);
+  }
+
   /**
    * Returns the WebAppContext associated with this HttpServer.
    *

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/ServerUtils.java

@@ -96,7 +96,17 @@ public static InetSocketAddress updateRPCListenAddress(
         rpcServer.getListenerAddress());
   }
 
-
+  public static InetSocketAddress updateRPCListenPort(
+      OzoneConfiguration conf, String rpcAddressKey,
+      InetSocketAddress listenerAddress) {
+    String originalValue = conf.get(rpcAddressKey);
+    //remove existing port
+    originalValue = originalValue.replaceAll(":.*", "");
+    conf.set(rpcAddressKey,
+        originalValue + ":" + listenerAddress.getPort());
+    return new InetSocketAddress(originalValue,
+        listenerAddress.getPort());
+  }
   /**
    * After starting an server, updates configuration with the actual
    * listening address of that server. The listening address may be different

hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/server/TestServerUtils.java

@@ -23,11 +23,14 @@
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.test.PathUtils;
+
+import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 
 import java.io.File;
+import java.net.InetSocketAddress;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -120,4 +123,18 @@ public void ozoneMetadataDirRejectsList() {
     ServerUtils.getOzoneMetaDirPath(conf);
   }
 
+  @Test
+  public void updateRpcListenPort() {
+    OzoneConfiguration conf = new OzoneConfiguration();
+
+    conf.set("test1", "localhost:0");
+    ServerUtils.updateRPCListenPort(conf, "test1",
+        new InetSocketAddress("0.0.0.0", 1234));
+    Assert.assertEquals("localhost:1234", conf.get("test1"));
+
+    conf.set("test2", "localhost");
+    ServerUtils.updateRPCListenPort(conf, "test2",
+        new InetSocketAddress("0.0.0.0", 1234));
+    Assert.assertEquals("localhost:1234", conf.get("test2"));
+  }
 }

hadoop-hdds/pom.xml

@@ -190,6 +190,27 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
         <version>${bouncycastle.version}</version>
       </dependency>
 
+      <dependency>
+        <groupId>com.sun.xml.bind</groupId>
+        <artifactId>jaxb-impl</artifactId>
+        <version>2.3.0.1</version>
+      </dependency>
+      <dependency>
+        <groupId>com.sun.xml.bind</groupId>
+        <artifactId>jaxb-core</artifactId>
+        <version>2.3.0.1</version>
+      </dependency>
+      <dependency>
+        <groupId>javax.xml.bind</groupId>
+        <artifactId>jaxb-api</artifactId>
+        <version>2.3.0</version>
+      </dependency>
+      <dependency>
+        <groupId>javax.activation</groupId>
+        <artifactId>activation</artifactId>
+        <version>1.1.1</version>
+      </dependency>
+
       <dependency>
         <groupId>org.junit.jupiter</groupId>
         <artifactId>junit-jupiter-api</artifactId>

hadoop-hdds/server-scm/pom.xml

@@ -54,6 +54,24 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
       <artifactId>hadoop-hdds-docs</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>com.sun.xml.bind</groupId>
+      <artifactId>jaxb-impl</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.sun.xml.bind</groupId>
+      <artifactId>jaxb-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>javax.xml.bind</groupId>
+      <artifactId>jaxb-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>javax.activation</groupId>
+      <artifactId>activation</artifactId>
+    </dependency>
+
+
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdds-container-service</artifactId>

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/discovery/ConfigurationEndpoint.java

@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.discovery;
+
+import javax.servlet.ServletContext;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+
+import java.util.Map.Entry;
+import java.util.Properties;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManagerHttpServer;
+
+/**
+ * JAXRS endpoint to publish current ozone configuration.
+ */
+@Path("/config")
+public class ConfigurationEndpoint {
+
+  private Properties defaults =
+      OzoneConfiguration.createWithDefaultsOnly().getProps();
+
+  @javax.ws.rs.core.Context
+  private ServletContext context;
+
+  /**
+   * Returns with the non-default configuration.
+   */
+  @GET
+  public ConfigurationXml getConfiguration() {
+    OzoneConfiguration conf = (OzoneConfiguration) context.getAttribute(
+        StorageContainerManagerHttpServer.CONFIG_CONTEXT_ATTRIBUTE);
+    ConfigurationXml configXml = new ConfigurationXml();
+    for (Entry<Object, Object> entry : conf.getProps().entrySet()) {
+      //return only the non-defaults
+      if (defaults.get(entry.getKey()) != entry.getValue()) {
+        configXml.addConfiguration(entry.getKey().toString(),
+            entry.getValue().toString());
+      }
+    }
+    return configXml;
+  }
+
+}