Use 128 KB as default value for DEFAULT_BUFFER_SIZE (#926)

* Add `TlsInterception` acceptance test

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Generate CA cert if not available

* Fix lint and tests

* Fix args

* Remove acceptance tests for now

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: abhinavsingh

Makefile

@@ -28,7 +28,7 @@ endif
 
 .PHONY: all https-certificates sign-https-certificates ca-certificates
 .PHONY: lib-check lib-clean lib-test lib-package lib-coverage lib-lint lib-pytest
-.PHONY: lib-release-test lib-release lib-profile lib-doc
+.PHONY: lib-release-test lib-release lib-profile lib-doc lib-pre-commit
 .PHONY: lib-dep lib-flake8 lib-mypy lib-speedscope container-buildx-all-platforms
 .PHONY: container container-run container-release container-build container-buildx
 .PHONY: devtools dashboard dashboard-clean container-without-openssl
@@ -91,6 +91,9 @@ lib-clean:
 	rm -rf proxy.py.egg-info
 	rm -rf .pytest_cache
 	rm -rf .hypothesis
+	# Doc RST files are cached and can cause issues
+	# See https://github.com/abhinavsingh/proxy.py/issues/642#issuecomment-1003444578
+	rm docs/pkg/*.rst
 
 lib-dep:
 	pip install --upgrade pip && \
@@ -101,6 +104,9 @@ lib-dep:
 		-r requirements-tunnel.txt && \
 	pip install "setuptools>=42"
 
+lib-pre-commit:
+	python -m pre_commit run --hook-stage manual --all-files -v
+
 lib-lint:
 	python -m tox -e lint
 

README.md

@@ -2232,7 +2232,7 @@ usage: -m [-h] [--enable-events] [--enable-conn-pool] [--threadless]
           [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG]
           [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE]
 
-proxy.py v2.4.0rc5.dev26+gb2b1bdc.d20211230
+proxy.py v2.4.0rc5.dev31+gc998255
 
 options:
   -h, --help            show this help message and exit
@@ -2290,21 +2290,17 @@ options:
                         Default: False. If used, will enable proxy protocol.
                         Only version 1 is currently supported.
   --client-recvbuf-size CLIENT_RECVBUF_SIZE
-                        Default: 1 MB. Maximum amount of data received from
-                        the client in a single recv() operation. Bump this
-                        value for faster uploads at the expense of increased
-                        RAM.
+                        Default: 128 KB. Maximum amount of data received from
+                        the client in a single recv() operation.
   --key-file KEY_FILE   Default: None. Server key file to enable end-to-end
                         TLS encryption with clients. If used, must also pass
                         --cert-file.
   --timeout TIMEOUT     Default: 10.0. Number of seconds after which an
                         inactive connection must be dropped. Inactivity is
                         defined by no data sent or received by the client.
   --server-recvbuf-size SERVER_RECVBUF_SIZE
-                        Default: 1 MB. Maximum amount of data received from
-                        the server in a single recv() operation. Bump this
-                        value for faster downloads at the expense of increased
-                        RAM.
+                        Default: 128 KB. Maximum amount of data received from
+                        the server in a single recv() operation.
   --disable-http-proxy  Default: False. Whether to disable
                         proxy.HttpProxyPlugin.
   --disable-headers DISABLE_HEADERS

proxy/common/constants.py

@@ -74,7 +74,7 @@ def _env_threadless_compliant() -> bool:
 # Defaults
 DEFAULT_BACKLOG = 100
 DEFAULT_BASIC_AUTH = None
-DEFAULT_BUFFER_SIZE = 1024 * 1024
+DEFAULT_BUFFER_SIZE = 128 * 1024
 DEFAULT_CA_CERT_DIR = None
 DEFAULT_CA_CERT_FILE = None
 DEFAULT_CA_KEY_FILE = None

proxy/http/handler.py

@@ -39,10 +39,9 @@
     '--client-recvbuf-size',
     type=int,
     default=DEFAULT_CLIENT_RECVBUF_SIZE,
-    help='Default: 1 MB. Maximum amount of data received from the '
-    'client in a single recv() operation. Bump this '
-    'value for faster uploads at the expense of '
-    'increased RAM.',
+    help='Default: ' + str(int(DEFAULT_CLIENT_RECVBUF_SIZE / 1024)) +
+    ' KB. Maximum amount of data received from the '
+    'client in a single recv() operation.',
 )
 flags.add_argument(
     '--key-file',

proxy/http/proxy/server.py

@@ -56,10 +56,9 @@
     '--server-recvbuf-size',
     type=int,
     default=DEFAULT_SERVER_RECVBUF_SIZE,
-    help='Default: 1 MB. Maximum amount of data received from the '
-    'server in a single recv() operation. Bump this '
-    'value for faster downloads at the expense of '
-    'increased RAM.',
+    help='Default: ' + str(int(DEFAULT_SERVER_RECVBUF_SIZE / 1024)) +
+    ' KB. Maximum amount of data received from the '
+    'server in a single recv() operation.',
 )
 
 flags.add_argument(

requirements-testing.txt

@@ -20,3 +20,4 @@ httpx==0.20.0
 h2==4.1.0
 hpack==4.0.0
 hyperframe==6.0.1
+pre-commit==2.16.0

tests/integration/test_integration.py

@@ -69,8 +69,8 @@ def proxy_py_subprocess(request: Any) -> Generator[int, None, None]:
     reason='OSError: [WinError 193] %1 is not a valid Win32 application',
     raises=OSError,
 )  # type: ignore[misc]
-def test_curl(proxy_py_subprocess: int) -> None:
-    """An acceptance test with using ``curl`` through proxy.py."""
+def test_integration(proxy_py_subprocess: int) -> None:
+    """An acceptance test using ``curl`` through proxy.py."""
     this_test_module = Path(__file__)
     shell_script_test = this_test_module.with_suffix('.sh')
     check_output([str(shell_script_test), str(proxy_py_subprocess)])