change transfer unit to bytes per second

Author: Ravindra Dingankar

hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md

@@ -370,9 +370,9 @@ Each metrics record contains tags such as SessionId and Hostname as additional i
 |:---- |:---- |
 | `BytesWritten` | Total number of bytes written to DataNode |
 | `BytesRead` | Total number of bytes read from DataNode |
-| `ReadTransferRateMBsNumOps` | Total number of data read transfers |
-| `ReadTransferRateMBsAvgTime` | Average transfer rate of bytes read from DataNode. |
-| `ReadTransferRateMBs`*num*`s(50/75/90/95/99)thPercentileRate` | The 50/75/90/95/99th percentile of the transfer rate of bytes read from DataNode, measured in megabytes per second. |
+| `ReadTransferRateNumOps` | Total number of data read transfers |
+| `ReadTransferRateAvgTime` | Average transfer rate of bytes read from DataNode, measured in bytes per second. |
+| `ReadTransferRate`*num*`s(50/75/90/95/99)thPercentileRate` | The 50/75/90/95/99th percentile of the transfer rate of bytes read from DataNode, measured in bytes per second. |
 | `BlocksWritten` | Total number of blocks written to DataNode |
 | `BlocksRead` | Total number of blocks read from DataNode |
 | `BlocksReplicated` | Total number of blocks replicated |

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java

@@ -1938,18 +1938,18 @@ public static boolean isParentEntry(final String path, final String parent) {
   }
 
   /**
-   * Calculate the transfer rate in megabytes/second. Return -1 for any negative input.
+   * Calculate the transfer rate in bytes/second. Return -1 for any negative input.
    * @param bytes bytes
    * @param durationMS duration in milliseconds
-   * @return the number of megabytes/second of the transfer rate
+   * @return the number of bytes/second of the transfer rate
   */
-  public static long transferRateMBs(long bytes, long durationMS) {
+  public static long transferRateBytesPerSecond(long bytes, long durationMS) {
     if (bytes < 0 || durationMS < 0) {
       return -1;
     }
     if (durationMS == 0) {
       durationMS = 1;
     }
-    return bytes / (1024 * 1024) * 1000 / durationMS;
+    return bytes * 1000 / durationMS;
   }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java

@@ -633,7 +633,7 @@ public void readBlock(final ExtendedBlock block,
       datanode.metrics.incrBytesRead((int) read);
       datanode.metrics.incrBlocksRead();
       datanode.metrics.incrTotalReadTime(duration);
-      datanode.metrics.addReadTransferRateMBs(DFSUtil.transferRateMBs(read, duration));
+      datanode.metrics.addReadTransferRate(DFSUtil.transferRateBytesPerSecond(read, duration));
     } catch ( SocketException ignored ) {
       LOG.trace("{}:Ignoring exception while serving {} to {}",
           dnR, block, remoteAddress, ignored);
@@ -1124,7 +1124,7 @@ public void copyBlock(final ExtendedBlock block,
       datanode.metrics.incrBytesRead((int) read);
       datanode.metrics.incrBlocksRead();
       datanode.metrics.incrTotalReadTime(duration);
-      datanode.metrics.addReadTransferRateMBs(DFSUtil.transferRateMBs(read, duration));
+      datanode.metrics.addReadTransferRate(DFSUtil.transferRateBytesPerSecond(read, duration));
 
       LOG.info("Copied {} to {}", block, peer.getRemoteAddressString());
     } catch (IOException ioe) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java

@@ -61,8 +61,8 @@ public class DataNodeMetrics {
   @Metric MutableCounterLong bytesRead;
   @Metric("Milliseconds spent reading")
   MutableCounterLong totalReadTime;
-  @Metric private MutableRate readTransferRateMBs;
-  final private MutableQuantiles[] readTransferRateMBsQuantiles;
+  @Metric private MutableRate readTransferRate;
+  final private MutableQuantiles[] readTransferRateQuantiles;
   @Metric MutableCounterLong blocksWritten;
   @Metric MutableCounterLong blocksRead;
   @Metric MutableCounterLong blocksReplicated;
@@ -229,7 +229,7 @@ public DataNodeMetrics(String name, String sessionId, int[] intervals,
     sendDataPacketTransferNanosQuantiles = new MutableQuantiles[len];
     ramDiskBlocksEvictionWindowMsQuantiles = new MutableQuantiles[len];
     ramDiskBlocksLazyPersistWindowMsQuantiles = new MutableQuantiles[len];
-    readTransferRateMBsQuantiles = new MutableQuantiles[len];
+    readTransferRateQuantiles = new MutableQuantiles[len];
 
     for (int i = 0; i < len; i++) {
       int interval = intervals[i];
@@ -258,9 +258,9 @@ public DataNodeMetrics(String name, String sessionId, int[] intervals,
           "ramDiskBlocksLazyPersistWindows" + interval + "s",
           "Time between the RamDisk block write and disk persist in ms",
           "ops", "latency", interval);
-      readTransferRateMBsQuantiles[i] = registry.newQuantiles(
-          "readTransferRateMBs" + interval + "s",
-          "Rate at which bytes are read from datanode calculated in megabytes per second",
+      readTransferRateQuantiles[i] = registry.newQuantiles(
+          "readTransferRate" + interval + "s",
+          "Rate at which bytes are read from datanode calculated in bytes per second",
           "ops", "rate", interval);
     }
   }
@@ -323,10 +323,10 @@ public void addIncrementalBlockReport(long latency,
     }
   }
 
-  public void addReadTransferRateMBs(long transferRateMBs) {
-    readTransferRateMBs.add(transferRateMBs);
-    for (MutableQuantiles q : readTransferRateMBsQuantiles) {
-      q.add(transferRateMBs);
+  public void addReadTransferRate(long readTransferRate) {
+    this.readTransferRate.add(readTransferRate);
+    for (MutableQuantiles q : readTransferRateQuantiles) {
+      q.add(readTransferRate);
     }
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java

@@ -1108,4 +1108,12 @@ public void testErrorMessageForInvalidNameservice() throws Exception {
     LambdaTestUtils.intercept(IOException.class, expectedErrorMessage,
         ()->DFSUtil.getNNServiceRpcAddressesForCluster(conf));
   }
+  
+  @Test
+  public void testTransferRateBytesPerSecond() {
+    assertEquals(9830, DFSUtil.transferRateBytesPerSecond(983, 100));
+    assertEquals(983000, DFSUtil.transferRateBytesPerSecond(983, 0));
+    assertEquals(-1, DFSUtil.transferRateBytesPerSecond(-983, 100));
+    assertEquals(-1, DFSUtil.transferRateBytesPerSecond(983, -100));
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeMetrics.java

@@ -392,7 +392,7 @@ public void testDataNodeTimeSpend() throws Exception {
 
       final long startWriteValue = getLongCounter("TotalWriteTime", rb);
       final long startReadValue = getLongCounter("TotalReadTime", rb);
-      assertCounter("ReadTransferRateMBsNumOps", 0L, rb);
+      assertCounter("ReadTransferRateNumOps", 0L, rb);
       final AtomicInteger x = new AtomicInteger(0);
 
       // Lets Metric system update latest metrics
@@ -412,8 +412,8 @@ public Boolean get() {
           MetricsRecordBuilder rbNew = getMetrics(datanode.getMetrics().name());
           final long endWriteValue = getLongCounter("TotalWriteTime", rbNew);
           final long endReadValue = getLongCounter("TotalReadTime", rbNew);
-          assertCounter("ReadTransferRateMBsNumOps", 1L, rbNew);
-          assertQuantileGauges("ReadTransferRateMBs" + "60s", rbNew, "Rate");
+          assertCounter("ReadTransferRateNumOps", 1L, rbNew);
+          assertQuantileGauges("ReadTransferRate" + "60s", rbNew, "Rate");
           return endWriteValue > startWriteValue
               && endReadValue > startReadValue;
         }