Merge pull request #830 from segevfiner/not-enough-space-workaround

Wrap stdout/stderr to avoid "Not enough space" in Python 2 on Windows 7

CHANGES.rst

@@ -87,6 +87,8 @@ Unreleased
 -   Raw strings added so correct escaping occurs. (`#807`_)
 -   Fix 16k character limit of ``click.echo`` on Windows. (`#816`_,
     `#819`_)
+-   Overcome 64k character limit when writing to binary stream on
+    Windows 7. (`#825`_, `#830`_)
 -   Add bool conversion for "t" and "f". (`#842`_)
 -   ``NoSuchOption`` errors take ``ctx`` so that ``--help`` hint gets
     printed in error output. (`#860`_)
@@ -212,6 +214,8 @@ Unreleased
 .. _#819: https://github.com/pallets/click/pull/819
 .. _#821: https://github.com/pallets/click/issues/821
 .. _#822: https://github.com/pallets/click/issues/822
+.. _#825: https://github.com/pallets/click/issues/825
+.. _#830: https://github.com/pallets/click/pull/830
 .. _#842: https://github.com/pallets/click/pull/842
 .. _#860: https://github.com/pallets/click/issues/860
 .. _#862: https://github.com/pallets/click/issues/862

click/_compat.py

@@ -224,9 +224,11 @@ def get_binary_stdin():
         return set_binary_mode(sys.stdin)
 
     def get_binary_stdout():
+        _wrap_std_stream('stdout')
         return set_binary_mode(sys.stdout)
 
     def get_binary_stderr():
+        _wrap_std_stream('stderr')
         return set_binary_mode(sys.stderr)
 
     def get_text_stdin(encoding=None, errors=None):
@@ -237,13 +239,15 @@ def get_text_stdin(encoding=None, errors=None):
                                  force_readable=True)
 
     def get_text_stdout(encoding=None, errors=None):
+        _wrap_std_stream('stdout')
         rv = _get_windows_console_stream(sys.stdout, encoding, errors)
         if rv is not None:
             return rv
         return _make_text_stream(sys.stdout, encoding, errors,
                                  force_writable=True)
 
     def get_text_stderr(encoding=None, errors=None):
+        _wrap_std_stream('stderr')
         rv = _get_windows_console_stream(sys.stderr, encoding, errors)
         if rv is not None:
             return rv
@@ -582,7 +586,7 @@ def should_strip_ansi(stream=None, color=None):
     # Windows has a smaller terminal
     DEFAULT_COLUMNS = 79
 
-    from ._winconsole import _get_windows_console_stream
+    from ._winconsole import _get_windows_console_stream, _wrap_std_stream
 
     def _get_argv_encoding():
         import locale
@@ -644,6 +648,7 @@ def _get_argv_encoding():
         return getattr(sys.stdin, 'encoding', None) or get_filesystem_encoding()
 
     _get_windows_console_stream = lambda *x: None
+    _wrap_std_stream = lambda *x: None
 
 
 def term_len(x):
@@ -669,6 +674,7 @@ def func():
             return rv
         rv = wrapper_func()
         try:
+            stream = src_func()  # In case wrapper_func() modified the stream
             cache[stream] = rv
         except Exception:
             pass

click/_winconsole.py

@@ -201,6 +201,40 @@ def __repr__(self):
         )
 
 
+class WindowsChunkedWriter(object):
+    """
+    Wraps a stream (such as stdout), acting as a transparent proxy for all
+    attribute access apart from method 'write()' which we wrap to write in
+    limited chunks due to a Windows limitation on binary console streams.
+    """
+    def __init__(self, wrapped):
+        # double-underscore everything to prevent clashes with names of
+        # attributes on the wrapped stream object.
+        self.__wrapped = wrapped
+
+    def __getattr__(self, name):
+        return getattr(self.__wrapped, name)
+
+    def write(self, text):
+        total_to_write = len(text)
+        written = 0
+
+        while written < total_to_write:
+            to_write = min(total_to_write - written, MAX_BYTES_WRITTEN)
+            self.__wrapped.write(text[written:written+to_write])
+            written += to_write
+
+
+_wrapped_std_streams = set()
+
+
+def _wrap_std_stream(name):
+    # Python 2 & Windows 7 and below
+    if PY2 and sys.getwindowsversion()[:2] <= (6, 1) and name not in _wrapped_std_streams:
+        setattr(sys, name, WindowsChunkedWriter(getattr(sys, name)))
+        _wrapped_std_streams.add(name)
+
+
 def _get_text_stdin(buffer_stream):
     text_stream = _NonClosingTextIOWrapper(
         io.BufferedReader(_WindowsConsoleReader(STDIN_HANDLE)),

docs/wincmd.rst

@@ -59,14 +59,21 @@ This hackery is used on both Python 2 and Python 3 as neither version of
 Python has native support for cmd.exe with unicode characters.  There are
 some limitations you need to be aware of:
 
-*   this unicode support is limited to ``click.echo``, ``click.prompt`` as
+*   This unicode support is limited to ``click.echo``, ``click.prompt`` as
     well as ``click.get_text_stream``.
-*   depending on if unicode values or byte strings are passed the control
+*   Depending on if unicode values or byte strings are passed the control
     flow goes completely different places internally which can have some
     odd artifacts if data partially ends up being buffered.  Click
     attempts to protect against that by manually always flushing but if
     you are mixing and matching different string types to ``stdout`` or
     ``stderr`` you will need to manually flush.
+*   The raw output stream is set to binary mode, which is a global
+    operation on Windows, so ``print`` calls will be affected. Prefer
+    ``click.echo`` over ``print``.
+*   On Windows 7 and below, there is a limitation where at most 64k
+    characters can be written in one call in binary mode. In this
+    situation, ``sys.stdout`` and ``sys.stderr`` are replaced with
+    wrappers that work around the limitation.
 
 Another important thing to note is that the Windows console's default
 fonts do not support a lot of characters which means that you are mostly