Make Files use transferTo, which can copy files without bringing the …

…bytes into userspace.

-------------
Created by MOE: http://code.google.com/p/moe-java
MOE_MIGRATED_REVID=86744917

guava-tests/test/com/google/common/io/ByteStreamsTest.java

@@ -21,10 +21,13 @@
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.EOFException;
+import java.io.File;
+import java.io.FileOutputStream;
 import java.io.FilterInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.io.RandomAccessFile;
 import java.nio.channels.Channels;
 import java.nio.channels.ReadableByteChannel;
 import java.nio.channels.WritableByteChannel;
@@ -48,6 +51,33 @@ public void testCopyChannel() throws IOException {
     assertEquals(expected, out.toByteArray());
   }
 
+  public void testCopyFileChannel() throws IOException {
+    final int chunkSize = 14407; // Random prime, unlikely to match any internal chunk size
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    WritableByteChannel outChannel = Channels.newChannel(out);
+
+    File testFile = createTempFile();
+    FileOutputStream fos = new FileOutputStream(testFile);
+    byte[] dummyData = newPreFilledByteArray(chunkSize);
+    try {
+      for (int i = 0; i < 500; i++) {
+        fos.write(dummyData);
+      }
+    } finally {
+      fos.close();
+    }
+    ReadableByteChannel inChannel = new RandomAccessFile(testFile, "r").getChannel();
+    try {
+      ByteStreams.copy(inChannel, outChannel);
+    } finally {
+      inChannel.close();
+    }
+    byte[] actual = out.toByteArray();
+    for (int i = 0; i < 500 * chunkSize; i += chunkSize) {
+      assertEquals(dummyData, Arrays.copyOfRange(actual, i, i + chunkSize));
+    }
+  }
+
   public void testReadFully() throws IOException {
     byte[] b = new byte[10];
 
@@ -157,7 +187,7 @@ public void testNewDataInput_readFully() {
     in.readFully(actual);
     assertEquals(bytes, actual);
   }
-  
+
   public void testNewDataInput_readFullyAndThenSome() {
     ByteArrayDataInput in = ByteStreams.newDataInput(bytes);
     byte[] actual = new byte[bytes.length * 2];
@@ -168,7 +198,7 @@ public void testNewDataInput_readFullyAndThenSome() {
       assertTrue(ex.getCause() instanceof EOFException);
     }
   }
-  
+
   public void testNewDataInput_readFullyWithOffset() {
     ByteArrayDataInput in = ByteStreams.newDataInput(bytes);
     byte[] actual = new byte[4];
@@ -178,7 +208,7 @@ public void testNewDataInput_readFullyWithOffset() {
     assertEquals(bytes[0], actual[2]);
     assertEquals(bytes[1], actual[3]);
   }
-  
+
   public void testNewDataInput_readLine() {
     ByteArrayDataInput in = ByteStreams.newDataInput(
         "This is a line\r\nThis too\rand this\nand also this".getBytes(Charsets.UTF_8));
@@ -194,7 +224,7 @@ public void testNewDataInput_readFloat() {
     assertEquals(Float.intBitsToFloat(0x12345678), in.readFloat(), 0.0);
     assertEquals(Float.intBitsToFloat(0x76543210), in.readFloat(), 0.0);
   }
-  
+
   public void testNewDataInput_readDouble() {
     byte[] data = {0x12, 0x34, 0x56, 0x78, 0x76, 0x54, 0x32, 0x10};
     ByteArrayDataInput in = ByteStreams.newDataInput(data);
@@ -216,7 +246,7 @@ public void testNewDataInput_readChar() {
     assertEquals('e', in.readChar());
     assertEquals('d', in.readChar());
   }
-  
+
   public void testNewDataInput_readUnsignedShort() {
     byte[] data = {0, 0, 0, 1, (byte) 0xFF, (byte) 0xFF, 0x12, 0x34};
     ByteArrayDataInput in = ByteStreams.newDataInput(data);
@@ -225,7 +255,7 @@ public void testNewDataInput_readUnsignedShort() {
     assertEquals(65535, in.readUnsignedShort());
     assertEquals(0x1234, in.readUnsignedShort());
   }
-  
+
   public void testNewDataInput_readLong() {
     byte[] data = {0x12, 0x34, 0x56, 0x78, 0x76, 0x54, 0x32, 0x10};
     ByteArrayDataInput in = ByteStreams.newDataInput(data);
@@ -236,7 +266,7 @@ public void testNewDataInput_readBoolean() {
     ByteArrayDataInput in = ByteStreams.newDataInput(bytes);
     assertTrue(in.readBoolean());
   }
-  
+
   public void testNewDataInput_readByte() {
     ByteArrayDataInput in = ByteStreams.newDataInput(bytes);
     for (int i = 0; i < bytes.length; i++) {
@@ -554,7 +584,7 @@ public void testLimit_skip() throws Exception {
     lin.skip(3);
     assertEquals(0, lin.available());
   }
-  
+
   public void testLimit_markNotSet() {
     byte[] big = newPreFilledByteArray(5);
     InputStream bin = new ByteArrayInputStream(big);
@@ -567,7 +597,7 @@ public void testLimit_markNotSet() {
       assertEquals("Mark not set", expected.getMessage());
     }
   }
-  
+
   public void testLimit_markNotSupported() {
     InputStream lin = ByteStreams.limit(new UnmarkableInputStream(), 2);
 
@@ -600,6 +630,7 @@ private static byte[] copyOfRange(byte[] in, int from, int to) {
   }
 
   private static void assertEquals(byte[] expected, byte[] actual) {
-    assertTrue(Arrays.equals(expected, actual));
+    assertEquals("Arrays differed in size", expected.length, actual.length);
+    assertTrue("Array contents were different", Arrays.equals(expected, actual));
   }
 }

guava/src/com/google/common/io/ByteStreams.java

@@ -34,6 +34,7 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
 import java.nio.channels.ReadableByteChannel;
 import java.nio.channels.WritableByteChannel;
 import java.util.Arrays;
@@ -48,6 +49,34 @@
 @Beta
 public final class ByteStreams {
   private static final int BUF_SIZE = 8192;
+  /**
+   * There are three methods to implement {@link FileChannel#transferTo(long, long,
+   *  WritableByteChannel)}:
+   *
+   * <ol>
+   * <li> Use sendfile(2) or equivalent. Requires that both the input channel and the output channel
+   *    have their own file descriptors. Generally this only happens when both channels are files or
+   *    sockets. This performs zero copies - the bytes never enter userspace.</li>
+   * <li> Use mmap(2) or equivalent. Requires that either the input channel or the output channel
+   *    have file descriptors. Bytes are copied from the file into a kernel buffer, then directly
+   *    into the other buffer (userspace). Note that if the file is very large, a naive
+   *    implementation will effectively put the whole file in memory. On many systems with paging
+   *    and virtual memory, this is not a problem - because it is mapped read-only, the kernel can
+   *    always page it to disk "for free". However, on systems where killing processes happens all
+   *    the time in normal conditions (i.e., android) the OS must make a tradeoff between paging
+   *    memory and killing other processes - so allocating a gigantic buffer and then sequentially
+   *    accessing it could result in other processes dying. This is solvable via madvise(2), but
+   *    that obviously doesn't exist in java.</li>
+   * <li> Ordinary copy. Kernel copies bytes into a kernel buffer, from a kernel buffer into a
+   *    userspace buffer (byte[] or ByteBuffer), then copies them from that buffer into the
+   *    destination channel.</li>
+   * </ol>
+   *
+   * This value is intended to be large enough to make the overhead of system calls negligible,
+   * without being so large that it causes problems for systems with atypical memory management if
+   * approaches 2 or 3 are used.
+   */
+  private static final int ZERO_COPY_CHUNK_SIZE = 512 * 1024;
 
   private ByteStreams() {}
 
@@ -90,6 +119,19 @@ public static long copy(ReadableByteChannel from,
       WritableByteChannel to) throws IOException {
     checkNotNull(from);
     checkNotNull(to);
+    if (from instanceof FileChannel) {
+      FileChannel sourceChannel = (FileChannel) from;
+      long oldPosition = sourceChannel.position();
+      long position = oldPosition;
+      long copied;
+      do {
+        copied = sourceChannel.transferTo(position, ZERO_COPY_CHUNK_SIZE, to);
+        position += copied;
+        sourceChannel.position(position);
+      } while (copied > 0 || position < sourceChannel.size());
+      return position - oldPosition;
+    }
+
     ByteBuffer buf = ByteBuffer.allocate(BUF_SIZE);
     long total = 0;
     while (from.read(buf) != -1) {