HBASE-28686 MapReduceBackupCopyJob should support custom DistCp options (#6017)

Co-authored-by: Ray Mattingly <rmattingly@hubspot.com>
Signed-off-by: Duo Zhang <zhangduo@apache.org>
Signed-off-by: Nick Dimiduk <ndimiduk@apache.org>

Author: 2 people

hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyJob.java

@@ -22,6 +22,7 @@
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.math.BigDecimal;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
@@ -45,6 +46,7 @@
 import org.apache.hadoop.tools.CopyListingFileStatus;
 import org.apache.hadoop.tools.DistCp;
 import org.apache.hadoop.tools.DistCpConstants;
+import org.apache.hadoop.tools.DistCpOptionSwitch;
 import org.apache.hadoop.tools.DistCpOptions;
 import org.apache.yetus.audience.InterfaceAudience;
 import org.slf4j.Logger;
@@ -58,6 +60,10 @@
 @InterfaceAudience.Private
 public class MapReduceBackupCopyJob implements BackupCopyJob {
   public static final String NUMBER_OF_LEVELS_TO_PRESERVE_KEY = "num.levels.preserve";
+
+  // This prefix specifies the DistCp options to be used during backup copy
+  public static final String BACKUP_COPY_OPTION_PREFIX = "hbase.backup.copy.";
+
   private static final Logger LOG = LoggerFactory.getLogger(MapReduceBackupCopyJob.class);
 
   private Configuration conf;
@@ -394,7 +400,15 @@ public int copy(BackupInfo context, BackupManager backupManager, Configuration c
         if (!destfs.exists(dest)) {
           destfs.mkdirs(dest);
         }
-        res = distcp.run(newOptions);
+
+        List<String> distCpOptionsFromConf = parseDistCpOptions(conf);
+        String[] finalOptions = new String[newOptions.length + distCpOptionsFromConf.size()];
+        for (int i = 0; i < distCpOptionsFromConf.size(); i++) {
+          finalOptions[i] = distCpOptionsFromConf.get(i);
+        }
+        System.arraycopy(newOptions, 0, finalOptions, distCpOptionsFromConf.size(),
+          newOptions.length);
+        res = distcp.run(finalOptions);
       }
       return res;
 
@@ -425,4 +439,25 @@ public void cancel(String jobId) throws IOException {
     }
   }
 
+  protected static List<String> parseDistCpOptions(Configuration conf) {
+    List<String> extraArgsFromConf = new ArrayList<>();
+
+    for (DistCpOptionSwitch optionSwitch : DistCpOptionSwitch.values()) {
+      String configLabel = BACKUP_COPY_OPTION_PREFIX + optionSwitch.getConfigLabel();
+      if (conf.get(configLabel) != null) {
+        if (optionSwitch.getOption().hasArg()) {
+          extraArgsFromConf.add("-" + optionSwitch.getOption().getOpt());
+          extraArgsFromConf.add(conf.get(configLabel));
+        } else {
+          boolean value = conf.getBoolean(configLabel, false);
+          if (value) {
+            extraArgsFromConf.add("-" + optionSwitch.getOption().getOpt());
+          }
+        }
+      }
+    }
+
+    return extraArgsFromConf;
+  }
+
 }

hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/mapreduce/TestMapReduceBackupCopyJob.java

@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup.mapreduce;
+
+import static org.apache.hadoop.hbase.backup.mapreduce.MapReduceBackupCopyJob.BACKUP_COPY_OPTION_PREFIX;
+import static org.apache.hadoop.tools.DistCpConstants.CONF_LABEL_DIRECT_WRITE;
+import static org.apache.hadoop.tools.DistCpConstants.CONF_LABEL_MAX_MAPS;
+import static org.junit.Assert.assertEquals;
+
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList;
+
+@Category(SmallTests.class)
+public class TestMapReduceBackupCopyJob {
+
+  @ClassRule
+  public static final HBaseClassTestRule CLASS_RULE =
+    HBaseClassTestRule.forClass(TestMapReduceBackupCopyJob.class);
+
+  @Test
+  public void testDistCpOptionParsing() {
+    Configuration conf = new Configuration();
+    conf.setInt(BACKUP_COPY_OPTION_PREFIX + CONF_LABEL_MAX_MAPS, 1000);
+    conf.setBoolean(BACKUP_COPY_OPTION_PREFIX + CONF_LABEL_DIRECT_WRITE, true);
+    List<String> args = MapReduceBackupCopyJob.parseDistCpOptions(conf);
+
+    List<String> expectedArgs =
+      ImmutableList.<String> builder().add("-m", "1000").add("-direct").build();
+
+    assertEquals(args, expectedArgs);
+  }
+
+}