Optimize gathering write in the epoll transport

Motivation:

While benchmarking the native transport, I noticed that gathering write
is not as fast as expected.  It was due to the fact that we have to do a
lot of array copies to put the buffer addresses into the iovec struct
array.

Modifications:

Introduce a new class called IovArray, which allows to fill buffers
directly into an off-heap array of iovec structs, so that it can be
passed over to JNI without any extra array copies.

Result:

Big performance improvement when doing gathering writes:

Before:

[nmaurer@xxx]~% wrk/wrk -H 'Host: localhost' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' -H 'Connection: keep-alive' -d 120 -c 256 -t 16 --pipeline 256  http://xxx:8080/plaintext
Running 2m test @ http://xxx:8080/plaintext
  16 threads and 256 connections
  Thread Stats   Avg      Stdev     Max   +/- Stdev
    Latency    23.44ms   16.37ms 259.57ms   91.77%
    Req/Sec   181.99k    31.69k  304.60k    78.12%
  346544071 requests in 2.00m, 46.48GB read
Requests/sec: 2887885.09
Transfer/sec:    396.59MB

After:

[nmaurer@xxx]~% wrk/wrk -H 'Host: localhost' -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' -H 'Connection: keep-alive' -d 120 -c 256 -t 16 --pipeline 256  http://xxx:8080/plaintext
Running 2m test @ http://xxx:8080/plaintext
  16 threads and 256 connections
  Thread Stats   Avg      Stdev     Max   +/- Stdev
    Latency    21.93ms   16.33ms 305.73ms   92.34%
    Req/Sec   194.56k    33.75k  309.33k    77.04%
  369617503 requests in 2.00m, 49.57GB read
Requests/sec: 3080169.65
Transfer/sec:    423.00MB

common/src/main/java/io/netty/util/internal/PlatformDependent.java

@@ -80,6 +80,8 @@ public final class PlatformDependent {
 
     private static final int BIT_MODE = bitMode0();
 
+    private static final int ADDRESS_SIZE = addressSize0();
+
     static {
         if (logger.isDebugEnabled()) {
             logger.debug("-Dio.netty.noPreferDirect: {}", !DIRECT_BUFFER_PREFERRED);
@@ -173,6 +175,22 @@ public static int bitMode() {
         return BIT_MODE;
     }
 
+    /**
+     * Return the address size of the OS.
+     * 4 (for 32 bits systems ) and 8 (for 64 bits systems).
+     */
+    public static int addressSize() {
+        return ADDRESS_SIZE;
+    }
+
+    public static long allocateMemory(long size) {
+        return PlatformDependent0.allocateMemory(size);
+    }
+
+    public static void freeMemory(long address) {
+        PlatformDependent0.freeMemory(address);
+    }
+
     /**
      * Raises an exception bypassing compiler checks for checked exceptions.
      */
@@ -815,6 +833,13 @@ private static int bitMode0() {
         }
     }
 
+    private static int addressSize0() {
+        if (!hasUnsafe()) {
+            return -1;
+        }
+        return PlatformDependent0.addressSize();
+    }
+
     private PlatformDependent() {
         // only static method supported
     }

common/src/main/java/io/netty/util/internal/PlatformDependent0.java

@@ -365,6 +365,18 @@ public ClassLoader run() {
         }
     }
 
+    static int addressSize() {
+        return UNSAFE.addressSize();
+    }
+
+    static long allocateMemory(long size) {
+        return UNSAFE.allocateMemory(size);
+    }
+
+    static void freeMemory(long address) {
+        UNSAFE.freeMemory(address);
+    }
+
     private PlatformDependent0() {
     }
 

transport-native-epoll/src/main/c/io_netty_channel_epoll_Native.c

@@ -44,7 +44,7 @@ jfieldID limitFieldId = NULL;
 jfieldID fileChannelFieldId = NULL;
 jfieldID transferedFieldId = NULL;
 jfieldID fdFieldId = NULL;
-jfieldID fileDescriptorFieldId = NULL;
+jfieldID fileDescriptorFieldId = NULL;;
 jmethodID inetSocketAddrMethodId = NULL;
 jmethodID datagramSocketAddrMethodId = NULL;
 jclass runtimeExceptionClass = NULL;
@@ -384,6 +384,7 @@ jint JNI_OnLoad(JavaVM* vm, void* reserved) {
             return JNI_ERR;
         }
         socketType = socket_type();
+
         datagramSocketAddrMethodId = (*env)->GetMethodID(env, datagramSocketAddressClass, "<init>", "(Ljava/lang/String;II)V");
         if (datagramSocketAddrMethodId == NULL) {
             throwRuntimeException(env, "Unable to obtain constructor of DatagramSocketAddress");
@@ -671,6 +672,29 @@ JNIEXPORT jobject JNICALL Java_io_netty_channel_epoll_Native_recvFromAddress(JNI
     return recvFrom0(env, fd, (void*) address, pos, limit);
 }
 
+jlong writev0(JNIEnv * env, jclass clazz, jint fd, struct iovec * iov, jint length) {
+    ssize_t res;
+    int err;
+    do {
+        res = writev(fd, iov, length);
+        // keep on writing if it was interrupted
+    } while(res == -1 && ((err = errno) == EINTR));
+
+    if (res < 0) {
+        if (err == EAGAIN || err == EWOULDBLOCK) {
+            // network stack is saturated we will try again later
+            return 0;
+        }
+        if (err == EBADF) {
+            throwClosedChannelException(env);
+            return -1;
+        }
+        throwIOException(env, exceptionMessage("Error while writev(...): ", err));
+        return -1;
+    }
+    return (jlong) res;
+}
+
 JNIEXPORT jlong JNICALL Java_io_netty_channel_epoll_Native_writev(JNIEnv * env, jclass clazz, jint fd, jobjectArray buffers, jint offset, jint length) {
     struct iovec iov[length];
     int iovidx = 0;
@@ -709,26 +733,12 @@ JNIEXPORT jlong JNICALL Java_io_netty_channel_epoll_Native_writev(JNIEnv * env,
         // See https://github.com/netty/netty/issues/2623
         (*env)->DeleteLocalRef(env, bufObj);
     }
-    ssize_t res;
-    int err;
-    do {
-        res = writev(fd, iov, length);
-        // keep on writing if it was interrupted
-    } while(res == -1 && ((err = errno) == EINTR));
+    return writev0(env, clazz, fd, iov, length);
+}
 
-    if (res < 0) {
-        if (err == EAGAIN || err == EWOULDBLOCK) {
-            // network stack is saturated we will try again later
-            return 0;
-        }
-        if (err == EBADF) {
-            throwClosedChannelException(env);
-            return -1;
-        }
-        throwIOException(env, exceptionMessage("Error while writev(...): ", err));
-        return -1;
-    }
-    return res;
+JNIEXPORT jlong JNICALL Java_io_netty_channel_epoll_Native_writevAddresses(JNIEnv * env, jclass clazz, jint fd, jlong memoryAddress, jint length) {
+    struct iovec * iov = (struct iovec *) memoryAddress;
+    return writev0(env, clazz, fd, iov, length);
 }
 
 jint read0(JNIEnv * env, jclass clazz, jint fd, void *buffer, jint pos, jint limit) {
@@ -1156,4 +1166,4 @@ JNIEXPORT jstring JNICALL Java_io_netty_channel_epoll_Native_kernelVersion(JNIEn
 
 JNIEXPORT jint JNICALL Java_io_netty_channel_epoll_Native_iovMax(JNIEnv *env, jclass clazz) {
     return IOV_MAX;
-}
+}

transport-native-epoll/src/main/c/io_netty_channel_epoll_Native.h

@@ -44,6 +44,7 @@ void  Java_io_netty_channel_epoll_Native_epollCtlDel(JNIEnv * env, jclass clazz,
 jint Java_io_netty_channel_epoll_Native_write(JNIEnv * env, jclass clazz, jint fd, jobject jbuffer, jint pos, jint limit);
 jint Java_io_netty_channel_epoll_Native_writeAddress(JNIEnv * env, jclass clazz, jint fd, jlong address, jint pos, jint limit);
 jlong Java_io_netty_channel_epoll_Native_writev(JNIEnv * env, jclass clazz, jint fd, jobjectArray buffers, jint offset, jint length);
+jlong Java_io_netty_channel_epoll_Native_writevAddresses(JNIEnv * env, jclass clazz, jint fd, jlong memoryAddress, jint length);
 jint Java_io_netty_channel_epoll_Native_sendTo(JNIEnv * env, jclass clazz, jint fd, jobject jbuffer, jint pos, jint limit, jbyteArray address, jint scopeId, jint port);
 jint Java_io_netty_channel_epoll_Native_sendToAddress(JNIEnv * env, jclass clazz, jint fd, jlong memoryAddress, jint pos, jint limit, jbyteArray address, jint scopeId, jint port);