third-party: Nailgun python3 compatibility improvements (#28037)

third_party/nailgun/README.amp

@@ -12,4 +12,9 @@ Files in this directory (copied from https://github.com/facebook/nailgun/release
 - nailgun-runner (copied from ng.py)
 
 Local Modifications:
-None.
+- nailgun-runner: [python3] Fix memoryview to work with python3
+  https://github.com/facebook/nailgun/commit/90167a65969d90094d7997e0308d9ddb1124d4b6
+- nailgun-runner: [python3] Ensure UTF8 strings are sent properly
+  https://github.com/facebook/nailgun/commit/f923d52beff7963f508b808f846f9970f97be0f6
+- nailgun-runner: [python3] Remove bytes -> str conversion
+  https://github.com/facebook/nailgun/commit/878f95db2ec6cb3f69144ec456a4b3fa3f1eb2f9

third_party/nailgun/nailgun-runner

@@ -25,9 +25,10 @@ import select
 import socket
 import struct
 import sys
+from io import BytesIO
 from threading import Condition, Event, Thread, RLock
 
-is_py2 = sys.version[0] == "2"
+is_py2 = sys.version_info[0] == 2
 if is_py2:
     import Queue as Queue
     import __builtin__ as builtin
@@ -45,11 +46,6 @@ else:
         return bytes(s, "utf-8")
 
 
-def bytes_to_str(bytes_to_convert):
-    """Version independent way of converting bytes to string."""
-    return bytes_to_convert if is_py2 else bytes_to_convert.decode("utf-8")
-
-
 # @author <a href="http://www.martiansoftware.com/contact.html">Marty Lamb</a>
 # @author Pete Kirkham (Win32 port)
 # @author Sergey Balabanov, Ben Hamilton (Python port)
@@ -89,6 +85,21 @@ EVENT_STDIN_CLOSED = 1
 EVENT_STDIN_EXCEPTION = 2
 
 
+def compat_memoryview_py2(buf):
+    return memoryview(buf)
+
+
+def compat_memoryview_py3(buf):
+    return memoryview(buf).cast("c")
+
+
+# memoryview in python3, while wrapping ctypes.create_string_buffer has problems with
+# that type's default format (<c) and assignment operators. For python3, cast to
+# a 'c' array. Little endian single byte doesn't make sense anyways. However,
+# 'cast' does not exist for python2. So, we have to toggle a bit.
+compat_memoryview = compat_memoryview_py2 if is_py2 else compat_memoryview_py3
+
+
 class NailgunException(Exception):
     SOCKET_FAILED = 231
     CONNECT_FAILED = 230
@@ -372,6 +383,11 @@ class WindowsNamedPipeTransport(Transport):
             self._raise_win_err("error while waiting for read", err)
 
         nread = nread.value
+        if not is_py2:
+            # Wrap in a memoryview, as python3 does not let you assign from a
+            # ctypes.c_char_array slice directly to a memory view, as one is 'c', and one
+            # is '<c' struct/buffer proto format.
+            buf = compat_memoryview(buf)
         buffer[:nread] = buf[:nread]
         return nread
 
@@ -568,7 +584,7 @@ class NailgunConnection(object):
         """
         Sends a NAILGUN_TTY_# environment variable.
         """
-        if not f or not hasattr(f, "fileno"):
+        if not f or not hasattr(f, "fileno") or isinstance(f, BytesIO):
             return
         try:
             fileno = f.fileno()
@@ -594,12 +610,21 @@ class NailgunConnection(object):
         object. Used to route data to stdout or stderr on the client.
         """
         bytes_read = 0
+        dest_fd = dest_file
+        flush = False
+        if dest_file and hasattr(dest_file, 'buffer'):
+            dest_fd = dest_file.buffer
+            flush = True
+            # Make sure we've written anything that already existed in the buffer
+            dest_fd.flush()
 
         while bytes_read < num_bytes:
             bytes_to_read = min(len(self.buf), num_bytes - bytes_read)
             bytes_received = self.transport.recv_into(self.buf, bytes_to_read)
-            if dest_file:
-                dest_file.write(bytes_to_str(self.buf[:bytes_received]))
+            if dest_fd:
+                dest_fd.write(self.buf[:bytes_received])
+                if flush:
+                    dest_fd.flush()
             bytes_read += bytes_received
 
     def _recv_to_buffer(self, num_bytes, buf):
@@ -611,7 +636,7 @@ class NailgunConnection(object):
         # only way to provide an offset to recv_into() is to use
         # memoryview(), which doesn't exist until Python 2.7.
         if HAS_MEMORYVIEW:
-            self._recv_into_memoryview(num_bytes, memoryview(buf))
+            self._recv_into_memoryview(num_bytes, compat_memoryview(buf))
         else:
             self._recv_to_buffer_with_copy(num_bytes, buf)
 
@@ -797,8 +822,8 @@ def send_thread_main(conn):
             while not conn.send_queue.empty():
                 # only this thread can deplete the queue, so it is safe to use blocking get()
                 (chunk_type, buf) = conn.send_queue.get()
-                struct.pack_into(">ic", header_buf, 0, len(buf), chunk_type)
                 bbuf = to_bytes(buf)
+                struct.pack_into(">ic", header_buf, 0, len(bbuf), chunk_type)
 
                 # these chunk types are not required for server to accept and process and server may terminate
                 # any time without waiting for them