HADOOP-13587. distcp.map.bandwidth.mb is overwritten even when -bandwidth flag isn't set. Contributed by Zoran Dimitrijevic

Author: raviprak-altiscale

hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java

@@ -71,6 +71,7 @@ public class DistCp extends Configured implements Tool {
   private static final String PREFIX = "_distcp";
   private static final String WIP_PREFIX = "._WIP_";
   private static final String DISTCP_DEFAULT_XML = "distcp-default.xml";
+  private static final String DISTCP_SITE_XML = "distcp-site.xml";
   static final Random rand = new Random();
 
   private boolean submitted;
@@ -86,6 +87,7 @@ public class DistCp extends Configured implements Tool {
   public DistCp(Configuration configuration, DistCpOptions inputOptions) throws Exception {
     Configuration config = new Configuration(configuration);
     config.addResource(DISTCP_DEFAULT_XML);
+    config.addResource(DISTCP_SITE_XML);
     setConf(config);
     this.inputOptions = inputOptions;
     this.metaFolder   = createMetaFolderPath();
@@ -393,10 +395,12 @@ public static void main(String argv[]) {
    * Loads properties from distcp-default.xml into configuration
    * object
    * @return Configuration which includes properties from distcp-default.xml
+   *         and distcp-site.xml
    */
   private static Configuration getDefaultConf() {
     Configuration config = new Configuration();
     config.addResource(DISTCP_DEFAULT_XML);
+    config.addResource(DISTCP_SITE_XML);
     return config;
   }
 

hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java

@@ -47,7 +47,7 @@ public class DistCpOptions {
   public static final int maxNumListstatusThreads = 40;
   private int numListstatusThreads = 0;  // Indicates that flag is not set.
   private int maxMaps = DistCpConstants.DEFAULT_MAPS;
-  private float mapBandwidth = DistCpConstants.DEFAULT_BANDWIDTH_MB;
+  private float mapBandwidth = 0;  // Indicates that we should use the default.
 
   private String copyStrategy = DistCpConstants.UNIFORMSIZE;
 
@@ -609,8 +609,10 @@ public void appendToConf(Configuration conf) {
         String.valueOf(useDiff));
     DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.SKIP_CRC,
         String.valueOf(skipCRC));
-    DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.BANDWIDTH,
-        String.valueOf(mapBandwidth));
+    if (mapBandwidth > 0) {
+      DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.BANDWIDTH,
+          String.valueOf(mapBandwidth));
+    }
     DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.PRESERVE_STATUS,
         DistCpUtils.packAttributes(preserveStatus));
     if (filtersFile != null) {

hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java

@@ -107,7 +107,7 @@ public void testParsebandwidth() {
     DistCpOptions options = OptionsParser.parse(new String[] {
         "hdfs://localhost:9820/source/first",
         "hdfs://localhost:9820/target/"});
-    Assert.assertEquals(options.getMapBandwidth(), DistCpConstants.DEFAULT_BANDWIDTH_MB, DELTA);
+    Assert.assertEquals(options.getMapBandwidth(), 0, DELTA);
 
     options = OptionsParser.parse(new String[] {
         "-bandwidth",
@@ -389,7 +389,7 @@ public void testToString() {
         + "deleteMissing=false, ignoreFailures=false, overwrite=false, "
         + "append=false, useDiff=false, fromSnapshot=null, toSnapshot=null, "
         + "skipCRC=false, blocking=true, numListstatusThreads=0, maxMaps=20, "
-        + "mapBandwidth=100.0, "
+        + "mapBandwidth=0.0, "
         + "copyStrategy='uniformsize', preserveStatus=[], "
         + "preserveRawXattrs=false, atomicWorkPath=null, logPath=null, "
         + "sourceFileListing=abc, sourcePaths=null, targetPath=xyz, "
@@ -572,6 +572,8 @@ public void testOptionsAppendToConf() {
     Configuration conf = new Configuration();
     Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false));
     Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(), false));
+    Assert.assertEquals(
+        conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), null);
     DistCpOptions options = OptionsParser.parse(new String[] {
         "-atomic",
         "-i",
@@ -581,7 +583,7 @@ public void testOptionsAppendToConf() {
     Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false));
     Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(), false));
     Assert.assertEquals(conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1),
-        DistCpConstants.DEFAULT_BANDWIDTH_MB, DELTA);
+        -1.0, DELTA);
 
     conf = new Configuration();
     Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false));
@@ -602,6 +604,62 @@ public void testOptionsAppendToConf() {
     Assert.assertEquals(conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 11.2, DELTA);
   }
 
+  @Test
+  public void testOptionsAppendToConfDoesntOverwriteBandwidth() {
+    Configuration conf = new Configuration();
+    Assert.assertEquals(
+        conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), null);
+    DistCpOptions options = OptionsParser.parse(new String[] {
+        "hdfs://localhost:8020/source/first",
+        "hdfs://localhost:8020/target/"});
+    options.appendToConf(conf);
+    Assert.assertEquals(
+        conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), -1.0,
+        DELTA);
+
+    conf = new Configuration();
+    Assert.assertEquals(
+        conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), null);
+    options = OptionsParser.parse(new String[] {
+        "-update",
+        "-delete",
+        "-pu",
+        "-bandwidth",
+        "77",
+        "hdfs://localhost:8020/source/first",
+        "hdfs://localhost:8020/target/"});
+    options.appendToConf(conf);
+    Assert.assertEquals(
+        conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 77.0,
+        DELTA);
+
+    conf = new Configuration();
+    conf.set(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), "88");
+    Assert.assertEquals(
+        conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), "88");
+    options = OptionsParser.parse(new String[] {
+        "hdfs://localhost:8020/source/first",
+        "hdfs://localhost:8020/target/"});
+    options.appendToConf(conf);
+    Assert.assertEquals(
+        conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 88.0,
+        DELTA);
+
+    conf = new Configuration();
+    conf.set(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), "88.0");
+    Assert.assertEquals(
+        conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), "88.0");
+    options = OptionsParser.parse(new String[] {
+        "-bandwidth",
+        "99",
+        "hdfs://localhost:8020/source/first",
+        "hdfs://localhost:8020/target/"});
+    options.appendToConf(conf);
+    Assert.assertEquals(
+        conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 99.0,
+        DELTA);
+  }
+
   @Test
   public void testAppendOption() {
     Configuration conf = new Configuration();