refactor: UnwrapSubdomainTests → proxy-gateway

This replaces magic helpers.UnwrapSubdomainTests with explicit
spec preset named proxy-gateway, which can be enabled/disabled
the same way as other presets.

While it was nice to run every test in both proxy modes,
it was extremely noisy and hard to debug once things went wrong.

By having explicit tests in *_proxy_test.go it is easy to skip them
where proxy functionality is not relevant.

Other:

- Send meaningful User-Agent
- Sent detached path in HTTP Connect proxy tunnel tests (instead of full
  URL – in tunnel mode we pretend we talk to remote server on the other
  end of tunnel, and not send full URL like it is done in plain proxy
  mode)

Makefile

@@ -19,7 +19,7 @@ test-cargateway: provision-cargateway fixtures.car gateway-conformance
 
 test-kubo-subdomains: provision-kubo gateway-conformance
 	./kubo-config.example.sh
-	./gateway-conformance test --json reports/output.json --gateway-url http://127.0.0.1:8080 --subdomain-url http://localhost:8080
+	./gateway-conformance test --json reports/output.json --gateway-url http://127.0.0.1:8080 --subdomain-url http://example.com:8080
 
 test-kubo: provision-kubo gateway-conformance
 	./gateway-conformance test --json reports/output.json --gateway-url http://127.0.0.1:8080 --specs -subdomain-gateway
@@ -31,13 +31,16 @@ provision-kubo:
 	find ./fixtures -name '*.car' -exec ipfs dag import --stats --pin-roots=false {} \;
 	find ./fixtures -name '*.ipns-record' -exec sh -c 'ipfs routing put --allow-offline /ipns/$$(basename -s .ipns-record "{}" | cut -d'_' -f1) "{}"' \;
 
-start-kubo-docker: stop-kubo-docker gateway-conformance
-	./gateway-conformance extract-fixtures --dnslink=true --car=false --ipns=false --dir=.temp
-	docker pull ipfs/kubo:$(KUBO_VERSION)
-	docker run -d --rm --net=host --name $(KUBO_DOCKER_NAME) -e IPFS_NS_MAP="$(shell cat ./.temp/dnslinks.IPFS_NS_MAP)" -v ./fixtures:/fixtures ipfs/kubo:$(KUBO_VERSION) daemon --init --offline
-	@until docker exec $(KUBO_DOCKER_NAME) ipfs --api=/ip4/127.0.0.1/tcp/5001 dag stat /ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn >/dev/null 2>&1; do sleep 0.1; done
-	find ./fixtures -name '*.car' -exec docker exec $(KUBO_DOCKER_NAME) ipfs dag import --stats --pin-roots=false {} \;
-	find ./fixtures -name '*.ipns-record' -exec docker exec $(KUBO_DOCKER_NAME) sh -c 'ipfs routing put --allow-offline /ipns/$$(basename -s .ipns-record "{}" | cut -d'_' -f1) "{}"' \;
+#start-kubo-docker: stop-kubo-docker gateway-conformance
+#	./gateway-conformance extract-fixtures --dir=.temp/fixtures
+#	docker pull ipfs/kubo:$(KUBO_VERSION)
+#	docker run -d --rm --net=host --name $(KUBO_DOCKER_NAME) -v "$(shell realpath .temp/fixtures)":/fixtures -v kubo-config.example.sh:/container-init.d/001-config.sh ipfs/kubo:$(KUBO_VERSION) daemon --init --offline
+#	@until docker exec $(KUBO_DOCKER_NAME) ipfs --api=/ip4/127.0.0.1/tcp/5001 dag stat /ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn >/dev/null 2>&1; do sleep 0.1; done
+#	docker exec $(KUBO_DOCKER_NAME) find /fixtures -name '*.car' -exec ipfs dag import --stats --pin-roots=false {} \;
+#	docker exec $(KUBO_DOCKER_NAME) find /fixtures -name '*.ipns-record' -exec sh -c 'ipfs routing put --allow-offline /ipns/$$(basename -s .ipns-record "{}" | cut -d'_' -f1) "{}"' \;
+#	TODO: provision Kubo config at Gateway.PublicGateways  to have subdomain gateway on example.com and also enable inlining on localhost
+#	      See: https://github.com/ipfs/kubo/blob/a07852a3f0294974b802923fb136885ad077384e/.github/workflows/gateway-conformance.yml#L22-L34
+#	      (this is not as trivial as it sounds because Kubo does not apply config inrealtime, and a restart is required.)
 
 stop-kubo-docker: clean
 	docker stop $(KUBO_DOCKER_NAME) || true

cmd/gateway-conformance/main.go

@@ -91,7 +91,7 @@ func main() {
 						Name:    "subdomain-url",
 						EnvVars: []string{"SUBDOMAIN_GATEWAY_URL"},
 						Usage:   "URL of the HTTP Host that should be used when testing https://specs.ipfs.tech/http-gateways/subdomain-gateway/ functionality",
-						Value:   "http://example.com:8080", // TODO: ideally, make these empty by default, and opt-in
+						Value:   "http://example.com:8080",
 					},
 					&cli.StringFlag{
 						Name:    "json-output",
@@ -108,7 +108,7 @@ func main() {
 					&cli.StringFlag{
 						Name:    "specs",
 						EnvVars: []string{"SPECS"},
-						Usage:   "Optional explicit scope of tests to run. Accepts a 'spec' (test only this spec), a '+spec' (test also this immature spec), or a '-spec' (do not test this mature spec). Available spec presets: " + strings.Join(getAvailableSpecPresets(), ","),
+						Usage:   "Adjust the scope of tests to run. Accepts a 'spec' (test only this spec), a '+spec' (test also this immature spec), or a '-spec' (do not test this mature spec). Available spec presets: " + strings.Join(getAvailableSpecPresets(), ","),
 						Value:   "",
 					},
 					&cli.BoolFlag{
@@ -120,13 +120,24 @@ func main() {
 				Action: func(cctx *cli.Context) error {
 					env := os.Environ()
 					verbose := cctx.Bool("verbose")
+
+					// Set gateway URLs
 					gatewayURL := cctx.String("gateway-url")
 					subdomainGatewayURL := cctx.String("subdomain-url")
-					env = append(env, fmt.Sprintf("GATEWAY_URL=%s", gatewayURL))
+					envGwURL := fmt.Sprintf("GATEWAY_URL=%s", gatewayURL)
+					if verbose {
+						fmt.Println(envGwURL)
+					}
+					env = append(env, envGwURL)
 					if subdomainGatewayURL != "" {
-						env = append(env, fmt.Sprintf("SUBDOMAIN_GATEWAY_URL=%s", subdomainGatewayURL))
+						envSubdomainGwURL := fmt.Sprintf("SUBDOMAIN_GATEWAY_URL=%s", subdomainGatewayURL)
+						if verbose {
+							fmt.Println(envSubdomainGwURL)
+						}
+						env = append(env, envSubdomainGwURL)
 					}
 
+					// Set other parameters
 					args := []string{"test", "./tests", "-test.v=test2json"}
 
 					specs := cctx.String("specs")
@@ -140,8 +151,8 @@ func main() {
 					args = append(args, cctx.Args().Slice()...)
 
 					fmt.Println("go " + strings.Join(args, " "))
-					fmt.Println("ENV " + strings.Join(env, " "))
 
+					// Execute tests against URLs
 					output := &bytes.Buffer{}
 					cmd := exec.Command("go", args...)
 					cmd.Dir = tooling.Home()

kubo-config.example.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/usr/bin/env bash
 
 FIXTURES_PATH=${1:-$(pwd)}
 
@@ -15,7 +15,7 @@ ipfs config --json Gateway.PublicGateways '{
 	}
 }'
 
-export IPFS_NS_MAP="$(cat "${FIXTURES_PATH}/dnslinks.json" | jq -r '.domains | to_entries | map("\(.key):\(.value)") | join(",")')"
+export IPFS_NS_MAP="$(cat "${FIXTURES_PATH}/dnslinks.IPFS_NS_MAP")"
 
 echo "Set the following IPFS_NS_MAP before starting the kubo daemon:"
 echo "IPFS_NS_MAP=${IPFS_NS_MAP}"

tests/dnslink_gateway_test.go

@@ -7,7 +7,6 @@ import (
 	"github.com/ipfs/gateway-conformance/tooling/car"
 	. "github.com/ipfs/gateway-conformance/tooling/check"
 	"github.com/ipfs/gateway-conformance/tooling/dnslink"
-	"github.com/ipfs/gateway-conformance/tooling/helpers"
 	"github.com/ipfs/gateway-conformance/tooling/specs"
 	. "github.com/ipfs/gateway-conformance/tooling/test"
 )
@@ -74,5 +73,5 @@ func TestDNSLinkGatewayUnixFSDirectoryListing(t *testing.T) {
 		},
 	}
 
-	RunWithSpecs(t, helpers.UnwrapSubdomainTests(t, tests), specs.DNSLinkGateway)
+	RunWithSpecs(t, tests, specs.DNSLinkGateway)
 }

tests/metadata_test.go

@@ -12,8 +12,8 @@ func logGatewayURL(t *testing.T) {
 		GatewayURL          string `json:"gateway_url"`
 		SubdomainGatewayURL string `json:"subdomain_gateway_url"`
 	}{
-		GatewayURL:          test.GatewayURL,
-		SubdomainGatewayURL: test.SubdomainGatewayURL,
+		GatewayURL:          test.GatewayURL().String(),
+		SubdomainGatewayURL: test.SubdomainGatewayURL().String(),
 	})
 }