HADOOP-6801. io.sort.mb and io.sort.factor were renamed and moved to …

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java

@@ -250,18 +250,43 @@ public class CommonConfigurationKeysPublic {
    * @deprecated Moved to mapreduce, see mapreduce.task.io.sort.mb
    * in mapred-default.xml
    * See https://issues.apache.org/jira/browse/HADOOP-6801
+   *
+   * For {@link org.apache.hadoop.io.SequenceFile.Sorter} control
+   * instead, see {@link #SEQ_IO_SORT_MB_KEY}.
    */
   public static final String  IO_SORT_MB_KEY = "io.sort.mb";
-  /** Default value for IO_SORT_MB_DEFAULT */
+  /** Default value for {@link #IO_SORT_MB_KEY}. */
   public static final int     IO_SORT_MB_DEFAULT = 100;
   /**
    * @deprecated Moved to mapreduce, see mapreduce.task.io.sort.factor
    * in mapred-default.xml
    * See https://issues.apache.org/jira/browse/HADOOP-6801
+   *
+   * For {@link org.apache.hadoop.io.SequenceFile.Sorter} control
+   * instead, see {@link #SEQ_IO_SORT_FACTOR_KEY}.
    */
   public static final String  IO_SORT_FACTOR_KEY = "io.sort.factor";
-  /** Default value for IO_SORT_FACTOR_DEFAULT */
+  /** Default value for {@link #IO_SORT_FACTOR_KEY}. */
   public static final int     IO_SORT_FACTOR_DEFAULT = 100;
+
+  /**
+   * @see
+   * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+   * core-default.xml</a>
+   */
+  public static final String  SEQ_IO_SORT_MB_KEY = "seq.io.sort.mb";
+  /** Default value for {@link #SEQ_IO_SORT_MB_KEY}. */
+  public static final int     SEQ_IO_SORT_MB_DEFAULT = 100;
+
+  /**
+   * @see
+   * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+   * core-default.xml</a>
+   */
+  public static final String  SEQ_IO_SORT_FACTOR_KEY = "seq.io.sort.factor";
+  /** Default value for {@link #SEQ_IO_SORT_FACTOR_KEY}. */
+  public static final int     SEQ_IO_SORT_FACTOR_DEFAULT = 100;
+
   /**
    * @see
    * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SequenceFile.java

@@ -2816,14 +2816,30 @@ public Sorter(FileSystem fs, RawComparator comparator, Class keyClass,
     }
 
     /** Sort and merge using an arbitrary {@link RawComparator}. */
+    @SuppressWarnings("deprecation")
     public Sorter(FileSystem fs, RawComparator comparator, Class keyClass,
                   Class valClass, Configuration conf, Metadata metadata) {
       this.fs = fs;
       this.comparator = comparator;
       this.keyClass = keyClass;
       this.valClass = valClass;
-      this.memory = conf.getInt("io.sort.mb", 100) * 1024 * 1024;
-      this.factor = conf.getInt("io.sort.factor", 100);
+      // Remember to fall-back on the deprecated MB and Factor keys
+      // until they are removed away permanently.
+      if (conf.get(CommonConfigurationKeys.IO_SORT_MB_KEY) != null) {
+        this.memory = conf.getInt(CommonConfigurationKeys.IO_SORT_MB_KEY,
+          CommonConfigurationKeys.SEQ_IO_SORT_MB_DEFAULT) * 1024 * 1024;
+      } else {
+        this.memory = conf.getInt(CommonConfigurationKeys.SEQ_IO_SORT_MB_KEY,
+          CommonConfigurationKeys.SEQ_IO_SORT_MB_DEFAULT) * 1024 * 1024;
+      }
+      if (conf.get(CommonConfigurationKeys.IO_SORT_FACTOR_KEY) != null) {
+        this.factor = conf.getInt(CommonConfigurationKeys.IO_SORT_FACTOR_KEY,
+            CommonConfigurationKeys.SEQ_IO_SORT_FACTOR_DEFAULT);
+      } else {
+        this.factor = conf.getInt(
+            CommonConfigurationKeys.SEQ_IO_SORT_FACTOR_KEY,
+            CommonConfigurationKeys.SEQ_IO_SORT_FACTOR_DEFAULT);
+      }
       this.conf = conf;
       this.metadata = metadata;
     }

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -2476,4 +2476,23 @@
       in audit logs.
     </description>
   </property>
+<!-- SequenceFile's Sorter properties -->
+  <property>
+    <name>seq.io.sort.mb</name>
+    <value>100</value>
+    <description>
+      The total amount of buffer memory to use while sorting files,
+      while using SequenceFile.Sorter, in megabytes. By default,
+      gives each merge stream 1MB, which should minimize seeks.
+    </description>
+  </property>
+  <property>
+    <name>seq.io.sort.factor</name>
+    <value>100</value>
+    <description>
+      The number of streams to merge at once while sorting
+      files using SequenceFile.Sorter.
+      This determines the number of open file handles.
+    </description>
+  </property>
 </configuration>

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java

@@ -38,6 +38,7 @@
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.fail;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 import org.mockito.Mockito;
 
 
@@ -54,7 +55,72 @@ public void testZlibSequenceFile() throws Exception {
     compressedSeqFileTest(new DefaultCodec());
     LOG.info("Successfully tested SequenceFile with DefaultCodec");
   }
-
+
+  @SuppressWarnings("deprecation")
+  public void testSorterProperties() throws IOException {
+    // Test to ensure that deprecated properties have no default
+    // references anymore.
+    Configuration config = new Configuration();
+    assertNull("The deprecated sort memory property "
+        + CommonConfigurationKeys.IO_SORT_MB_KEY
+        + " must not exist in any core-*.xml files.",
+        config.get(CommonConfigurationKeys.IO_SORT_MB_KEY));
+    assertNull("The deprecated sort factor property "
+        + CommonConfigurationKeys.IO_SORT_FACTOR_KEY
+        + " must not exist in any core-*.xml files.",
+        config.get(CommonConfigurationKeys.IO_SORT_FACTOR_KEY));
+
+    // Test deprecated property honoring
+    // Set different values for old and new property names
+    // and compare which one gets loaded
+    config = new Configuration();
+    FileSystem fs = FileSystem.get(config);
+    config.setInt(CommonConfigurationKeys.IO_SORT_MB_KEY, 10);
+    config.setInt(CommonConfigurationKeys.IO_SORT_FACTOR_KEY, 10);
+    config.setInt(CommonConfigurationKeys.SEQ_IO_SORT_MB_KEY, 20);
+    config.setInt(CommonConfigurationKeys.SEQ_IO_SORT_FACTOR_KEY, 20);
+    SequenceFile.Sorter sorter = new SequenceFile.Sorter(
+        fs, Text.class, Text.class, config);
+    assertEquals("Deprecated memory conf must be honored over newer property",
+        10*1024*1024, sorter.getMemory());
+    assertEquals("Deprecated factor conf must be honored over newer property",
+        10, sorter.getFactor());
+
+    // Test deprecated properties (graceful deprecation)
+    config = new Configuration();
+    fs = FileSystem.get(config);
+    config.setInt(CommonConfigurationKeys.IO_SORT_MB_KEY, 10);
+    config.setInt(CommonConfigurationKeys.IO_SORT_FACTOR_KEY, 10);
+    sorter = new SequenceFile.Sorter(
+        fs, Text.class, Text.class, config);
+    assertEquals("Deprecated memory property "
+        + CommonConfigurationKeys.IO_SORT_MB_KEY
+        + " must get properly applied.",
+        10*1024*1024, // In bytes
+        sorter.getMemory());
+    assertEquals("Deprecated sort factor property "
+        + CommonConfigurationKeys.IO_SORT_FACTOR_KEY
+        + " must get properly applied.",
+        10, sorter.getFactor());
+
+    // Test regular properties (graceful deprecation)
+    config = new Configuration();
+    fs = FileSystem.get(config);
+    config.setInt(CommonConfigurationKeys.SEQ_IO_SORT_MB_KEY, 20);
+    config.setInt(CommonConfigurationKeys.SEQ_IO_SORT_FACTOR_KEY, 20);
+    sorter = new SequenceFile.Sorter(
+        fs, Text.class, Text.class, config);
+    assertEquals("Memory property "
+        + CommonConfigurationKeys.SEQ_IO_SORT_MB_KEY
+        + " must get properly applied if present.",
+        20*1024*1024, // In bytes
+        sorter.getMemory());
+    assertEquals("Merge factor property "
+        + CommonConfigurationKeys.SEQ_IO_SORT_FACTOR_KEY
+        + " must get properly applied if present.",
+        20, sorter.getFactor());
+  }
+
   public void compressedSeqFileTest(CompressionCodec codec) throws Exception {
     int count = 1024 * 10;
     int megabytes = 1;