expose chunk_size and document its significance

This commit addresses reviewer comment
<datalad#596 (comment)>.
It exposes `chunk_size` in `shell_connection` and
documents its significance for the size of the
`stderr`-queue that is accessible via the
`stderr_deque`-attribute of a
`ShellCommandResponseGenerator`-object.

Author: christian-monch

datalad_next/utils/shell_connection.py

@@ -17,6 +17,7 @@
 
 from more_itertools import consume
 
+from datalad_next.consts import COPY_BUFSIZE
 from datalad_next.itertools import align_pattern
 from datalad_next.runners.iter_subproc import (
     OutputFrom,
@@ -29,6 +30,8 @@
 
 @contextmanager
 def shell_connection(shell_cmd: list[str],
+                     *,
+                     chunk_size: int = COPY_BUFSIZE
                      ) -> Generator[ShellCommandExecutor, None, None]:
     """Context manager that provides an interactive connection to a shell
 
@@ -64,6 +67,7 @@ def train(queue: Queue):
     subprocess_inputs: Queue = Queue()
     with iter_subproc(shell_cmd,
                       input=train(subprocess_inputs),
+                      chunk_size=chunk_size,
                       bufsize=0) as shell_output:
         cmd_executor = ShellCommandExecutor(subprocess_inputs, shell_output)
         try:
@@ -170,7 +174,9 @@ def upload(self,
         -------
         tuple[int, bytes]
             A tuple containing the return code of the command that was executed
-            to upload the content and the last 64k bytes of stderr output
+            to upload the file and the last ``chunk_size`` (defined by the
+            ``chunk_size`` keyword argument to func:`shell_connection`) bytes
+            of stderr output.
         """
         file_size = local_path.stat().st_size
         cmd_line = f'dd bs=1 of="{remote_path.as_posix()}" count={file_size}'
@@ -199,7 +205,9 @@ def download(self,
         -------
         tuple[int, bytes]
             A tuple containing the return code of the command that was executed
-            to upload the content and the last 64k bytes of stderr output
+            to download the file and the last ``chunk_size`` (defined by the
+            ``chunk_size`` keyword argument to func:`shell_connection`) bytes
+            of stderr output.
         """
         # We use 7z in gzip-mode to compress the data that is sent over the
         # wire. That ensures that the end-marker is not in the downloaded data,
@@ -231,7 +239,9 @@ def delete(self,
         -------
         tuple[int, bytes]
             A tuple containing the return code of the command that was executed
-            to delete the files and the last 64k bytes of stderr output.
+            to delete the files and the last ``chunk_size`` (defined by the
+            ``chunk_size`` keyword argument to func:`shell_connection`) bytes
+            of stderr output.
         """
         cmd_line = \
             'rm ' \
@@ -261,7 +271,11 @@ class ShellCommandResponseGenerator(ABCGenerator):
     :class:`ShellCommandExecutor`. It yields all bytes that the executed
     command writes to its ``stdout``. Once the command is finished, the
     generator will be exhausted. At this time, the return code of the command
-    can be retrieved from the ``code``-attribute of the generator.
+    can be retrieved from the ``code``-attribute of the generator. The last
+    ``chunk_size`` (defined by the ``chunk_size`` keyword argument to
+    func:`shell_connection`) bytes of the stderr-output of the command are
+    available in the ``stderr_dequeue``-attribute, which contains a
+    ``deque``-object that holds individual stderr chunks.
     """
     def __init__(self,
                  stdout: OutputFrom,