Transfer-Encoding: chunked causing php_fastcgi to hang

[arabcoders]

Hello, I was attempting to support Transfer-Encoding: chunked header into my application, and noticed that the server never responses if the header is used.

$ caddy version
v2.6.2 h1:wKoFIxpmOJLGl3QXoo6PNbYvGW4xLEgo32GPBEjWL8o=

Caddyfile

{
	debug
	log stdout
	admin 0.0.0.0:2018
	grace_period 1s
}

:8888 {
	log
	root * /home/user/test
	php_fastcgi unix//var/run/php/php8.1-fpm.sock {
		#buffer_requests
		#flush_interval -1
	}
}

php file (index.php)

<?php
echo "i am alive\n";

Example of working request/response

$ curl -k http://localhost:8888/ --header "Pragma: no-cache" -X POST -d file=test

i am alive

Caddy Logs

2022/12/06 14:41:18.348 DEBUG   http.handlers.rewrite   rewrote request {"request": {"remote_ip": "127.0.0.1", "remote_port": "49996", "proto": "HTTP/1.1", "method": "POST", "host": "localhost:8888", "uri": "/", "headers": {"Content-Length": ["9"], "Content-Type": ["application/x-www-form-urlencoded"], "Cache-Control": ["no-cache"], "User-Agent": ["curl/7.81.0"], "Accept": ["*/*"], "Pragma": ["no-cache"]}}, "method": "POST", "uri": "/index.php"}
2022/12/06 14:41:18.348 DEBUG   http.handlers.reverse_proxy     selected upstream       {"dial": "/var/run/php/php8.1-fpm.sock", "total_upstreams": 1}
2022/12/06 14:41:18.348 DEBUG   http.reverse_proxy.transport.fastcgi    roundtrip       {"request": {"remote_ip": "127.0.0.1", "remote_port": "49996", "proto": "HTTP/1.1", "method": "POST", "host": "localhost:8888", "uri": "/index.php", "headers": {"User-Agent": ["curl/7.81.0"], "Pragma": ["no-cache"], "X-Forwarded-For": ["127.0.0.1"], "X-Forwarded-Proto": ["http"], "X-Forwarded-Host": ["localhost:8888"], "Cache-Control": ["no-cache"], "Accept": ["*/*"], "Content-Length": ["9"], "Content-Type": ["application/x-www-form-urlencoded"]}}, "env": {"GATEWAY_INTERFACE": "CGI/1.1", "REMOTE_IDENT": "", "CONTENT_LENGTH": "9", "SERVER_PORT": "8888", "HTTP_CONTENT_LENGTH": "9", "REMOTE_HOST": "127.0.0.1", "REMOTE_PORT": "49996", "HTTP_HOST": "localhost:8888", "HTTP_CACHE_CONTROL": "no-cache", "HTTP_PRAGMA": "no-cache", "REQUEST_METHOD": "POST", "DOCUMENT_ROOT": "/home/user/test", "REQUEST_URI": "/", "REMOTE_ADDR": "127.0.0.1", "REMOTE_USER": "", "HTTP_CONTENT_TYPE": "application/x-www-form-urlencoded", "PATH_INFO": "", "SERVER_PROTOCOL": "HTTP/1.1", "HTTP_ACCEPT": "*/*", "CONTENT_TYPE": "application/x-www-form-urlencoded", "AUTH_TYPE": "", "REQUEST_SCHEME": "http", "SERVER_NAME": "localhost", "SCRIPT_FILENAME": "/home/user/testindex.php", "HTTP_X_FORWARDED_PROTO": "http", "HTTP_X_FORWARDED_HOST": "localhost:8888", "HTTP_USER_AGENT": "curl/7.81.0", "QUERY_STRING": "", "SERVER_SOFTWARE": "Caddy/v2.6.2", "DOCUMENT_URI": "/index.php", "SCRIPT_NAME": "/index.php", "HTTP_X_FORWARDED_FOR": "127.0.0.1"}, "dial": "/var/run/php/php8.1-fpm.sock", "env": {"REQUEST_METHOD": "POST", "DOCUMENT_ROOT": "/home/user/test", "REQUEST_URI": "/", "REMOTE_ADDR": "127.0.0.1", "REMOTE_USER": "", "HTTP_CONTENT_TYPE": "application/x-www-form-urlencoded", "PATH_INFO": "", "SERVER_PROTOCOL": "HTTP/1.1", "HTTP_ACCEPT": "*/*", "CONTENT_TYPE": "application/x-www-form-urlencoded", "SCRIPT_FILENAME": "/home/user/testindex.php", "HTTP_X_FORWARDED_PROTO": "http", "HTTP_X_FORWARDED_HOST": "localhost:8888", "HTTP_USER_AGENT": "curl/7.81.0", "AUTH_TYPE": "", "REQUEST_SCHEME": "http", "SERVER_NAME": "localhost", "SCRIPT_NAME": "/index.php", "HTTP_X_FORWARDED_FOR": "127.0.0.1", "QUERY_STRING": "", "SERVER_SOFTWARE": "Caddy/v2.6.2", "DOCUMENT_URI": "/index.php", "SERVER_PORT": "8888", "HTTP_CONTENT_LENGTH": "9", "GATEWAY_INTERFACE": "CGI/1.1", "REMOTE_IDENT": "", "CONTENT_LENGTH": "9", "HTTP_CACHE_CONTROL": "no-cache", "HTTP_PRAGMA": "no-cache", "REMOTE_HOST": "127.0.0.1", "REMOTE_PORT": "49996", "HTTP_HOST": "localhost:8888"}, "request": {"remote_ip": "127.0.0.1", "remote_port": "49996", "proto": "HTTP/1.1", "method": "POST", "host": "localhost:8888", "uri": "/index.php", "headers": {"Cache-Control": ["no-cache"], "Accept": ["*/*"], "Content-Length": ["9"], "Content-Type": ["application/x-www-form-urlencoded"], "User-Agent": ["curl/7.81.0"], "Pragma": ["no-cache"], "X-Forwarded-For": ["127.0.0.1"], "X-Forwarded-Proto": ["http"], "X-Forwarded-Host": ["localhost:8888"]}}}
2022/12/06 14:41:18.349 DEBUG   http.handlers.reverse_proxy     upstream roundtrip      {"upstream": "unix//var/run/php/php8.1-fpm.sock", "duration": 0.000485466, "request": {"remote_ip": "127.0.0.1", "remote_port": "49996", "proto": "HTTP/1.1", "method": "POST", "host": "localhost:8888", "uri": "/index.php", "headers": {"Accept": ["*/*"], "Content-Length": ["9"], "Content-Type": ["application/x-www-form-urlencoded"], "Cache-Control": ["no-cache"], "Pragma": ["no-cache"], "X-Forwarded-For": ["127.0.0.1"], "X-Forwarded-Proto": ["http"], "X-Forwarded-Host": ["localhost:8888"], "User-Agent": ["curl/7.81.0"]}}, "headers": {"Content-Type": ["text/html; charset=UTF-8"]}, "status": 200}
2022/12/06 14:41:18.349 INFO    http.log.access handled request {"request": {"remote_ip": "127.0.0.1", "remote_port": "49996", "proto": "HTTP/1.1", "method": "POST", "host": "localhost:8888", "uri": "/", "headers": {"Cache-Control": ["no-cache"], "User-Agent": ["curl/7.81.0"], "Accept": ["*/*"], "Pragma": ["no-cache"], "Content-Length": ["9"], "Content-Type": ["application/x-www-form-urlencoded"]}}, "user_id": "", "duration": 0.000779217, "size": 1788, "status": 200, "resp_headers": {"Server": ["Caddy"], "Content-Type": ["text/html; charset=UTF-8"]}}

Example of not working request/response

$ curl -k http://localhost:8888/ --header "Pragma: no-cache" -X POST -d file=test --header "Transfer-Encoding: chunked" -vvv
hangs forever. no response.

Caddy logs.

2022/12/06 14:46:23.786 DEBUG   http.handlers.rewrite   rewrote request {"request": {"remote_ip": "127.0.0.1", "remote_port": "43034", "proto": "HTTP/1.1", "method": "POST", "host": "localhost:8888", "uri": "/", "headers": {"Accept": ["*/*"], "User-Agent": ["curl/7.81.0"], "Pragma": ["no-cache"], "Content-Type": ["application/x-www-form-urlencoded"], "Cache-Control": ["no-cache"]}}, "method": "POST", "uri": "/index.php"}
2022/12/06 14:46:23.786 DEBUG   http.handlers.reverse_proxy     selected upstream       {"dial": "/var/run/php/php8.1-fpm.sock", "total_upstreams": 1}
2022/12/06 14:46:23.786 DEBUG   http.reverse_proxy.transport.fastcgi    roundtrip       {"request": {"remote_ip": "127.0.0.1", "remote_port": "43034", "proto": "HTTP/1.1", "method": "POST", "host": "localhost:8888", "uri": "/index.php", "headers": {"Pragma": ["no-cache"], "Content-Type": ["application/x-www-form-urlencoded"], "Cache-Control": ["no-cache"], "User-Agent": ["curl/7.81.0"], "X-Forwarded-For": ["127.0.0.1"], "X-Forwarded-Proto": ["http"], "X-Forwarded-Host": ["localhost:8888"], "Accept": ["*/*"]}}, "env": {"SCRIPT_FILENAME": "/home/user/test/index.php", "HTTP_CONTENT_TYPE": "application/x-www-form-urlencoded", "HTTP_USER_AGENT": "curl/7.81.0", "HTTP_X_FORWARDED_PROTO": "http", "AUTH_TYPE": "", "REMOTE_IDENT": "", "REMOTE_USER": "", "REQUEST_SCHEME": "http", "DOCUMENT_ROOT": "/home/user/test", "SERVER_NAME": "localhost", "HTTP_X_FORWARDED_FOR": "127.0.0.1", "REMOTE_ADDR": "127.0.0.1", "HTTP_X_FORWARDED_HOST": "localhost:8888", "PATH_INFO": "", "QUERY_STRING": "", "SERVER_SOFTWARE": "Caddy/v2.6.2", "GATEWAY_INTERFACE": "CGI/1.1", "SERVER_PROTOCOL": "HTTP/1.1", "REQUEST_URI": "/", "HTTP_ACCEPT": "*/*", "HTTP_PRAGMA": "no-cache", "HTTP_CACHE_CONTROL": "no-cache", "CONTENT_LENGTH": "", "REMOTE_HOST": "127.0.0.1", "DOCUMENT_URI": "/index.php", "CONTENT_TYPE": "application/x-www-form-urlencoded", "REMOTE_PORT": "43034", "REQUEST_METHOD": "POST", "HTTP_HOST": "localhost:8888", "SCRIPT_NAME": "/index.php", "SERVER_PORT": "8888"}, "dial": "/var/run/php/php8.1-fpm.sock", "env": {"REMOTE_ADDR": "127.0.0.1", "HTTP_X_FORWARDED_HOST": "localhost:8888", "PATH_INFO": "", "QUERY_STRING": "", "SERVER_SOFTWARE": "Caddy/v2.6.2", "HTTP_ACCEPT": "*/*", "HTTP_PRAGMA": "no-cache", "HTTP_CACHE_CONTROL": "no-cache", "GATEWAY_INTERFACE": "CGI/1.1", "SERVER_PROTOCOL": "HTTP/1.1", "REQUEST_URI": "/", "CONTENT_LENGTH": "", "REMOTE_HOST": "127.0.0.1", "DOCUMENT_URI": "/index.php", "HTTP_HOST": "localhost:8888", "SCRIPT_NAME": "/index.php", "SERVER_PORT": "8888", "CONTENT_TYPE": "application/x-www-form-urlencoded", "REMOTE_PORT": "43034", "REQUEST_METHOD": "POST", "HTTP_X_FORWARDED_PROTO": "http", "SCRIPT_FILENAME": "/home/user/test/index.php", "HTTP_CONTENT_TYPE": "application/x-www-form-urlencoded", "HTTP_USER_AGENT": "curl/7.81.0", "REQUEST_SCHEME": "http", "DOCUMENT_ROOT": "/home/user/test", "AUTH_TYPE": "", "REMOTE_IDENT": "", "REMOTE_USER": "", "SERVER_NAME": "localhost", "HTTP_X_FORWARDED_FOR": "127.0.0.1"}, "request": {"remote_ip": "127.0.0.1", "remote_port": "43034", "proto": "HTTP/1.1", "method": "POST", "host": "localhost:8888", "uri": "/index.php", "headers": {"Content-Type": ["application/x-www-form-urlencoded"], "Cache-Control": ["no-cache"], "User-Agent": ["curl/7.81.0"], "X-Forwarded-For": ["127.0.0.1"], "X-Forwarded-Proto": ["http"], "X-Forwarded-Host": ["localhost:8888"], "Accept": ["*/*"], "Pragma": ["no-cache"]}}}

i also tried with buffer_requests and flush_interval -1 and it seems to have no effect.

This behavior led me to believe it was an issue with PHP itself, however digging deeper i was able to find this issue php bug #60826, reading the replies this suggest the problem is not conforming to fastcgi spec

Essentially, PHP is following the spec, and not reading beyond the FastCGI CONTENT_LENGTH header value - so correct fixes are necessarily above PHP, in the FastCGI or web server implementation.

In a strict sense this is indeed not a PHP bug, because the CGI specification (which is only informational, though), mandates[1]:

As transfer-codings are not supported on the request-body, the server MUST remove any such codings from the message-body, and recalculate the CONTENT_LENGTH. If this is not possible (for example, because of large buffering requirements), the server SHOULD reject the client request.

It seems apache also had the bug and they have somewhat working fix at https://bz.apache.org/bugzilla/show_bug.cgi?id=57087

I tried to see if the request itself is valid i used python gunicorn to test

curl http://httpbin/post --header "Pragma: no-cache" -d file=test --header "Transfer-Encoding: chunked" -vvv
*   Trying 172.17.0.2:80...
* Connected to httpbin (172.17.0.2) port 80 (#0)
> POST /post HTTP/1.1
> Host: httpbin
> User-Agent: curl/7.81.0
> Accept: */*
> Pragma: no-cache
> Transfer-Encoding: chunked
> Content-Type: application/x-www-form-urlencoded
>
* Mark bundle as not supporting multiuse
< HTTP/1.1 200 OK
< Server: gunicorn/19.9.0
< Date: Tue, 06 Dec 2022 15:01:59 GMT
< Connection: keep-alive
< Content-Type: application/json
< Content-Length: 383
< Access-Control-Allow-Origin: *
< Access-Control-Allow-Credentials: true
<
{
  "args": {},
  "data": "",
  "files": {},
  "form": {
    "file": "test"
  },
  "headers": {
    "Accept": "*/*",
    "Content-Type": "application/x-www-form-urlencoded",
    "Host": "httpbin",
    "Pragma": "no-cache",
    "Transfer-Encoding": "chunked",
    "User-Agent": "curl/7.81.0"
  },
  "json": null,
  "origin": "172.17.0.1",
  "url": "http://httpbin/post"
}
* Connection #0 to host httpbin left intact

[mholt]

This is a good bug report, thanks for the investigation and the details. I'm a little behind on things currently, but anyone is welcome to look into this for faster resolution!

[u5surf]

@arabcoders @mholt
Hi, I would like to work on this issue.
I consider that it can be solved the issue to enable BufferRequests in reverse proxy when the request header has a Transfer-Encoding: chunked.
And then it has better returned buffered body size from bufferedBody to set content length.

My dirty hack seems to work fine in this scenario.

diff --git a/modules/caddyhttp/reverseproxy/reverseproxy.go b/modules/caddyhttp/reverseproxy/reverseproxy.go
index 55d3aa8c..fb1bb411 100644
--- a/modules/caddyhttp/reverseproxy/reverseproxy.go
+++ b/modules/caddyhttp/reverseproxy/reverseproxy.go
@@ -622,8 +622,17 @@ func (h Handler) prepareRequest(req *http.Request, repl *caddy.Replacer) (*http.
        // attacks, so it is strongly recommended to only use this
        // feature if absolutely required, if read timeouts are
        // set, and if body size is limited
-       if h.BufferRequests && req.Body != nil {
-               req.Body = h.bufferedBody(req.Body)
+       var chunkedRequest bool
+       var size int64
+       for _, transferEncoding := range req.TransferEncoding {
+               if transferEncoding == "chunked" {
+                       chunkedRequest = true
+               }
+       }
+
+       if (h.BufferRequests || chunkedRequest) && req.Body != nil {
+               req.Body, size = h.bufferedBody(req.Body)
+               req.ContentLength = size
        }

        if req.ContentLength == 0 {
@@ -854,7 +863,7 @@ func (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, origRe

        // if enabled, buffer the response body
        if h.BufferResponses {
-               res.Body = h.bufferedBody(res.Body)
+               res.Body, _ = h.bufferedBody(res.Body)
        }

        // see if any response handler is configured for this response from the backend
@@ -1125,7 +1134,8 @@ func (h Handler) provisionUpstream(upstream *Upstream) {

 // bufferedBody reads originalBody into a buffer, then returns a reader for the buffer.
 // Always close the return value when done with it, just like if it was the original body!
-func (h Handler) bufferedBody(originalBody io.ReadCloser) io.ReadCloser {
+func (h Handler) bufferedBody(originalBody io.ReadCloser) (io.ReadCloser, int64) {
+       var written int64
        buf := bufPool.Get().(*bytes.Buffer)
        buf.Reset()
        if h.MaxBufferSize > 0 {
@@ -1135,16 +1145,16 @@ func (h Handler) bufferedBody(originalBody io.ReadCloser) io.ReadCloser {
                                Reader: io.MultiReader(buf, originalBody),
                                buf:    buf,
                                body:   originalBody,
-                       }
+                       }, n
                }
        } else {
-               _, _ = io.Copy(buf, originalBody)
+               written, _ = io.Copy(buf, originalBody)
        }
        originalBody.Close() // no point in keeping it open
        return bodyReadCloser{
                Reader: buf,
                buf:    buf,
-       }
+       }, written
 }

 // cloneRequest makes a semi-deep clone of origReq.

[francislavoie]

Sounds good @u5surf, PR is welcome! If you open a PR, it would probably make it easier for @arabcoders to test and confirm the fix.

[arabcoders]

@u5surf Thank you for for your patch.

Indeed it would be great if you can send PR so i can test it out, sadly i don't have the infra to build caddy at the moment.

[francislavoie]

It doesn't really take any "infrastructure", you just install Go (download a tar/zip, extract it to your PATH) and then download xcaddy and run xcaddy build. But yes, a PR would make that easy.

[arabcoders]

It doesn't really take any "infrastructure", you just install Go (download a tar/zip, extract it to your PATH) and then download xcaddy and run xcaddy build. But yes, a PR would make that easy.

Thanks, i tried to patch caddy but it keep failing with this error

$ git apply ff.patch -vvv
Checking patch modules/caddyhttp/reverseproxy/reverseproxy.go...
error: while searching for:
       // attacks, so it is strongly recommended to only use this
       // feature if absolutely required, if read timeouts are
       // set, and if body size is limited
       if h.BufferRequests && req.Body != nil {
               req.Body = h.bufferedBody(req.Body)
       }

       if req.ContentLength == 0 {

error: patch failed: modules/caddyhttp/reverseproxy/reverseproxy.go:622
error: modules/caddyhttp/reverseproxy/reverseproxy.go: patch does not apply

im tying to manually add the changes and will see how it goes.

---

Edit: nevermind, ignoring whitespace did the trick --ignore-whitespace building now.

---

Edit2: Build somewhat works, however it seems there is slight problem not sure if it's a problem or not checking the headers caddy sending to fastcgi server it seems CONTENT_LENGTH to be empty string e.g.

2023/01/06 20:37:53.257 INFO    using provided configuration    {"config_file": "./Caddyfile", "config_adapter": ""}
2023/01/06 20:37:53.258 INFO    admin   admin endpoint started  {"address": "0.0.0.0:2018", "enforce_origin": false, "origins": ["//0.0.0.0:2018"]}
2023/01/06 20:37:53.258 WARN    admin   admin endpoint on open interface; host checking disabled        {"address": "0.0.0.0:2018"}
2023/01/06 20:37:53.258 INFO    tls.cache.maintenance   started background certificate maintenance      {"cache": "0xc000512d90"}
2023/01/06 20:37:53.258 DEBUG   http    starting server loop    {"address": "[::]:8888", "tls": false, "http3": false}
2023/01/06 20:37:53.258 INFO    http.log        server running  {"name": "srv0", "protocols": ["h1", "h2", "h3"]}
2023/01/06 20:37:53.258 INFO    tls     cleaning storage unit   {"description": "FileStorage:/home/user/.local/share/caddy"}
2023/01/06 20:37:53.258 INFO    autosaved config (load with --resume flag)      {"file": "/home/user/.config/caddy/autosave.json"}
2023/01/06 20:37:53.258 INFO    tls     finished cleaning storage units
2023/01/06 20:37:53.258 INFO    serving initial configuration
2023/01/06 20:37:56.216 DEBUG   http.handlers.rewrite   rewrote request {"request": {"remote_ip": "127.0.0.1", "remote_port": "49906", "proto": "HTTP/1.1", "method": "POST", "host": "localhost:8888", "uri": "/", "headers": {"Accept": ["*/*"], "Pragma": ["no-cache"], "Content-Type": ["application/x-www-form-urlencoded"], "User-Agent": ["curl/7.81.0"], "Cache-Control": ["no-cache"]}}, "method": "POST", "uri": "/index.php"}
2023/01/06 20:37:56.216 DEBUG   http.handlers.reverse_proxy     selected upstream       {"dial": "/var/run/php/php8.1-fpm.sock", "total_upstreams": 1}
2023/01/06 20:37:56.216 DEBUG   http.reverse_proxy.transport.fastcgi    roundtrip       {"request": {"remote_ip": "127.0.0.1", "remote_port": "49906", "proto": "HTTP/1.1", "method": "POST", "host": "localhost:8888", "uri": "/index.php", "headers": {"X-Forwarded-For": ["127.0.0.1"], "X-Forwarded-Proto": ["http"], "X-Forwarded-Host": ["localhost:8888"], "Accept": ["*/*"], "Pragma": ["no-cache"], "Content-Type": ["application/x-www-form-urlencoded"], "User-Agent": ["curl/7.81.0"], "Cache-Control": ["no-cache"]}}, "env": {"REMOTE_PORT": "49906", "SERVER_NAME": "localhost", "SERVER_PROTOCOL": "HTTP/1.1", "HTTP_X_FORWARDED_HOST": "localhost:8888", "CONTENT_TYPE": "application/x-www-form-urlencoded", "PATH_INFO": "", "REMOTE_HOST": "127.0.0.1", "SCRIPT_NAME": "/index.php", "HTTP_CACHE_CONTROL": "no-cache", "REMOTE_USER": "", "REQUEST_METHOD": "POST", "DOCUMENT_URI": "/index.php", "HTTP_HOST": "localhost:8888", "GATEWAY_INTERFACE": "CGI/1.1", "QUERY_STRING": "", "REMOTE_ADDR": "127.0.0.1", "SCRIPT_FILENAME": "/home/user/tests/test_caddy/test/index.php", "HTTP_USER_AGENT": "curl/7.81.0", "AUTH_TYPE": "", "SERVER_PORT": "8888", "HTTP_X_FORWARDED_PROTO": "http", "HTTP_CONTENT_TYPE": "application/x-www-form-urlencoded", "HTTP_PRAGMA": "no-cache", "REMOTE_IDENT": "", "SERVER_SOFTWARE": "Caddy/(devel)_custom", "HTTP_ACCEPT": "*/*", "REQUEST_URI": "/", "HTTP_X_FORWARDED_FOR": "127.0.0.1", "CONTENT_LENGTH": "", "REQUEST_SCHEME": "http", "DOCUMENT_ROOT": "/home/user/tests/test_caddy/test"}, "dial": "/var/run/php/php8.1-fpm.sock", "env": {"AUTH_TYPE": "", "SERVER_PORT": "8888", "HTTP_X_FORWARDED_PROTO": "http", "HTTP_CONTENT_TYPE": "application/x-www-form-urlencoded", "REMOTE_IDENT": "", "SERVER_SOFTWARE": "Caddy/(devel)_custom", "HTTP_ACCEPT": "*/*", "HTTP_PRAGMA": "no-cache", "CONTENT_LENGTH": "", "REQUEST_SCHEME": "http", "DOCUMENT_ROOT": "/home/user/tests/test_caddy/test", "REQUEST_URI": "/", "HTTP_X_FORWARDED_FOR": "127.0.0.1", "CONTENT_TYPE": "application/x-www-form-urlencoded", "PATH_INFO": "", "REMOTE_HOST": "127.0.0.1", "REMOTE_PORT": "49906", "SERVER_NAME": "localhost", "SERVER_PROTOCOL": "HTTP/1.1", "HTTP_X_FORWARDED_HOST": "localhost:8888", "SCRIPT_NAME": "/index.php", "HTTP_CACHE_CONTROL": "no-cache", "GATEWAY_INTERFACE": "CGI/1.1", "QUERY_STRING": "", "REMOTE_ADDR": "127.0.0.1", "REMOTE_USER": "", "REQUEST_METHOD": "POST", "DOCUMENT_URI": "/index.php", "HTTP_HOST": "localhost:8888", "SCRIPT_FILENAME": "/home/user/tests/test_caddy/test/index.php", "HTTP_USER_AGENT": "curl/7.81.0"}, "request": {"remote_ip": "127.0.0.1", "remote_port": "49906", "proto": "HTTP/1.1", "method": "POST", "host": "localhost:8888", "uri": "/index.php", "headers": {"X-Forwarded-For": ["127.0.0.1"], "X-Forwarded-Proto": ["http"], "X-Forwarded-Host": ["localhost:8888"], "Accept": ["*/*"], "Pragma": ["no-cache"], "Content-Type": ["application/x-www-form-urlencoded"], "User-Agent": ["curl/7.81.0"], "Cache-Control": ["no-cache"]}}}
2023/01/06 20:37:56.216 DEBUG   http.handlers.reverse_proxy     upstream roundtrip      {"upstream": "unix//var/run/php/php8.1-fpm.sock", "duration": 0.000424846, "request": {"remote_ip": "127.0.0.1", "remote_port": "49906", "proto": "HTTP/1.1", "method": "POST", "host": "localhost:8888", "uri": "/index.php", "headers": {"Content-Type": ["application/x-www-form-urlencoded"], "User-Agent": ["curl/7.81.0"], "Cache-Control": ["no-cache"], "X-Forwarded-For": ["127.0.0.1"], "X-Forwarded-Proto": ["http"], "X-Forwarded-Host": ["localhost:8888"], "Accept": ["*/*"], "Pragma": ["no-cache"]}}, "headers": {"Content-Type": ["text/html; charset=UTF-8"]}, "status": 200}
2023/01/06 20:37:56.216 INFO    http.log.access handled request {"request": {"remote_ip": "127.0.0.1", "remote_port": "49906", "proto": "HTTP/1.1", "method": "POST", "host": "localhost:8888", "uri": "/", "headers": {"User-Agent": ["curl/7.81.0"], "Cache-Control": ["no-cache"], "Accept": ["*/*"], "Pragma": ["no-cache"], "Content-Type": ["application/x-www-form-urlencoded"]}}, "user_id": "", "duration": 0.00065616, "size": 11, "status": 200, "resp_headers": {"Server": ["Caddy"], "Content-Type": ["text/html; charset=UTF-8"]}}

However, the server does response correctly now

curl -k http://localhost:8888/ --header "Pragma: no-cache" -X POST -d file=test --header "Transfer-Encoding: chunked" -vvv
Note: Unnecessary use of -X or --request, POST is already inferred.
*   Trying 127.0.0.1:8888...
* Connected to localhost (127.0.0.1) port 8888 (#0)
> POST / HTTP/1.1
> Host: localhost:8888
> User-Agent: curl/7.81.0
> Accept: */*
> Pragma: no-cache
> Transfer-Encoding: chunked
> Content-Type: application/x-www-form-urlencoded
>
* Mark bundle as not supporting multiuse
< HTTP/1.1 200 OK
< Content-Type: text/html; charset=UTF-8
< Server: Caddy
< Date: Fri, 06 Jan 2023 20:37:56 GMT
< Content-Length: 11
<
i am alive
* Connection #0 to host localhost left intact

[u5surf]

@arabcoders

if it's a problem or not checking the headers caddy sending to fastcgi server it seems CONTENT_LENGTH to be empty string e.g.

I find that it should add the content-length header after h.bufferedBody because Transfer-Encoding: chunked andContent-Length are exclusive. I suppose to reconsider that point.

[u5surf]

Added req.Header.Set("Content-Length", strconv.FormatInt(req.ContentLength, 10))
https://github.com/caddyserver/caddy/pull/5289/files#diff-a248c9a1ec018edea8b377d155bc1df1a642bf79d00ababb5cdacc6b86c5733dR627

[arabcoders]

@u5surf Thank you it seems the Content-Length is populated now, although it seems there is an extra header as well

023/01/07 17:54:09.853 DEBUG   http.reverse_proxy.transport.fastcgi    roundtrip       {"request": {"remote_ip": "127.0.0.1", "remote_port": "42242", "proto": "HTTP/1.1", "method": "POST", "host": "localhost:8888", "uri": "/index.php", "headers": {"Content-Type": ["application/x-www-form-urlencoded"], "Cache-Control": ["no-cache"], "Pragma": ["no-cache"], "Content-Length": ["9"], "Accept": ["*/*"], "User-Agent": ["curl/7.81.0"], "X-Forwarded-For": ["127.0.0.1"], "X-Forwarded-Proto": ["http"], "X-Forwarded-Host": ["localhost:8888"]}}, "env": {"SERVER_PROTOCOL": "HTTP/1.1", "REQUEST_URI": "/", "HTTP_X_FORWARDED_PROTO": "http", "GATEWAY_INTERFACE": "CGI/1.1", "QUERY_STRING": "", "REMOTE_HOST": "127.0.0.1", "REQUEST_SCHEME": "http", "REMOTE_PORT": "42242", "REMOTE_USER": "", "SCRIPT_FILENAME": "/home/test/tests/test_caddy/test/index.php", "HTTP_CONTENT_LENGTH": "9", "HTTP_CACHE_CONTROL": "no-cache", "REMOTE_ADDR": "127.0.0.1", "SERVER_NAME": "localhost", "HTTP_HOST": "localhost:8888", "HTTP_X_FORWARDED_FOR": "127.0.0.1", "SERVER_SOFTWARE": "Caddy/16eed7fc-20230107", "DOCUMENT_ROOT": "/home/test/tests/test_caddy/test", "DOCUMENT_URI": "/index.php", "SCRIPT_NAME": "/index.php", "AUTH_TYPE": "", "CONTENT_TYPE": "application/x-www-form-urlencoded", "PATH_INFO": "", "REQUEST_METHOD": "POST", "HTTP_USER_AGENT": "curl/7.81.0", "HTTP_PRAGMA": "no-cache", "SERVER_PORT": "8888", "HTTP_CONTENT_TYPE": "application/x-www-form-urlencoded", "HTTP_X_FORWARDED_HOST": "localhost:8888", "REMOTE_IDENT": "", "CONTENT_LENGTH": "9", "HTTP_ACCEPT": "*/*"}, "dial": "/var/run/php/php8.1-fpm.sock", "env": {"SERVER_PORT": "8888", "HTTP_CONTENT_TYPE": "application/x-www-form-urlencoded", "HTTP_X_FORWARDED_HOST": "localhost:8888", "REMOTE_IDENT": "", "CONTENT_LENGTH": "9", "HTTP_ACCEPT": "*/*", "SERVER_PROTOCOL": "HTTP/1.1", "GATEWAY_INTERFACE": "CGI/1.1", "QUERY_STRING": "", "REMOTE_HOST": "127.0.0.1", "REQUEST_SCHEME": "http", "REQUEST_URI": "/", "HTTP_X_FORWARDED_PROTO": "http", "REMOTE_PORT": "42242", "REMOTE_USER": "", "SCRIPT_FILENAME": "/home/test/tests/test_caddy/test/index.php", "HTTP_CONTENT_LENGTH": "9", "REMOTE_ADDR": "127.0.0.1", "SERVER_NAME": "localhost", "HTTP_HOST": "localhost:8888", "HTTP_X_FORWARDED_FOR": "127.0.0.1", "HTTP_CACHE_CONTROL": "no-cache", "AUTH_TYPE": "", "CONTENT_TYPE": "application/x-www-form-urlencoded", "PATH_INFO": "", "REQUEST_METHOD": "POST", "SERVER_SOFTWARE": "Caddy/16eed7fc-20230107", "DOCUMENT_ROOT": "/home/test/tests/test_caddy/test", "DOCUMENT_URI": "/index.php", "SCRIPT_NAME": "/index.php", "HTTP_USER_AGENT": "curl/7.81.0", "HTTP_PRAGMA": "no-cache"}, "request": {"remote_ip": "127.0.0.1", "remote_port": "42242", "proto": "HTTP/1.1", "method": "POST", "host": "localhost:8888", "uri": "/index.php", "headers": {"Accept": ["*/*"], "User-Agent": ["curl/7.81.0"], "X-Forwarded-For": ["127.0.0.1"], "X-Forwarded-Proto": ["http"], "X-Forwarded-Host": ["localhost:8888"], "Content-Type": ["application/x-www-form-urlencoded"], "Cache-Control": ["no-cache"], "Pragma": ["no-cache"], "Content-Length": ["9"]}}}

It seems the fix did indeed fix CONTENT_LENGTH however somehow there are two headers now HTTP_CONTENT_LENGTH and CONTENT_LENGTH

[francislavoie]

It seems the fix did indeed fix CONTENT_LENGTH however somehow there are two headers now HTTP_CONTENT_LENGTH and CONTENT_LENGTH

That's normal -- HTTP_CONTENT_LENGTH is the HTTP header, and CONTENT_LENGTH is the CGI variable. See here:

caddy/modules/caddyhttp/reverseproxy/fastcgi/fastcgi.go

Line 299 in e450a73

"CONTENT_LENGTH": r.Header.Get("Content-Length"),