test: t0118-gateway-car.sh

core/corehttp/gateway_handler.go

@@ -327,6 +327,9 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 
 	// Support custom response formats passed via ?format or Accept HTTP header
 	switch contentType := getExplicitContentType(r); contentType {
+	case "":
+		// nothing we should special-case, skip
+		break
 	case "application/vnd.ipld.raw":
 		logger.Debugw("serving raw block", "path", parsedPath)
 		i.serveRawBlock(w, r, resolvedPath.Cid(), parsedPath)

core/corehttp/gateway_handler_car.go

@@ -20,32 +20,31 @@ func (i *gatewayHandler) serveCar(w http.ResponseWriter, r *http.Request, rootCi
 	name := rootCid.String() + ".car"
 	setContentDispositionHeader(w, name, "attachment")
 
-	// Set remaining headers
-	/* TODO modtime := addCacheControlHeaders(w, r, contentPath, rootCid)
-	- how does cache-control look like, given car can fail mid-stream?
-	  - we don't want clients to cache partial/interrupted CAR
-	  - we may document that client should verify that all blocks were dowloaded,
-	    or we may leverage content-length to hint something went wrong
-	*/
-
-	/* TODO: content-length (so user agents show % of remaining download)
-	- introduce max-car-size  limit in go-ipfs-config and pre-compute CAR first, and then get size and use lazySeeker?
-	- are we able to provide length for Unixfs DAGs? (CumulativeSize+CARv0 header+envelopes)
-	*/
+	// Weak Etag W/ because we can't guarantee byte-for-byte identical  responses
+	// (CAR is streamed, blocks arrive from datastore in non-deterministic order)
+	w.Header().Set("Etag", `W/"`+rootCid.String()+`.car"`)
+
+	// Explicit Cache-Control to ensure fresh stream on retry.
+	// CAR stream could be interrupted, and client should be able to resume and get full response, not the truncated one
+	w.Header().Set("Cache-Control", "no-cache, no-transform")
 
 	w.Header().Set("Content-Type", "application/vnd.ipld.car; version=1")
 	w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^)
 
 	// Same go-car settings as dag.export command
 	store := dagStore{dag: i.api.Dag(), ctx: ctx}
+
+	// TODO: support selectors passed as request param: https://github.com/ipfs/go-ipfs/issues/8769
 	dag := gocar.Dag{Root: rootCid, Selector: selectorparse.CommonSelector_ExploreAllRecursively}
 	car := gocar.NewSelectiveCar(ctx, store, []gocar.Dag{dag}, gocar.TraverseLinksOnlyOnce())
 
-	w.WriteHeader(http.StatusOK)
-
 	if err := car.Write(w); err != nil {
-		// TODO: can we do any error handling here?
-		// TODO: idea: add best-effort proxy reader which  will set http.StatusOK only if the first block is yielded correctly
+		// We return error as a trailer, however it is not something browsers can access
+		// (https://github.com/mdn/browser-compat-data/issues/14703)
+		// Due to this, we suggest client always verify that
+		// the received CAR stream response is matching requested DAG selector
+		w.Header().Set("X-Stream-Error", err.Error())
+		return
 	}
 }
 

test/sharness/lib/test-lib.sh

@@ -520,3 +520,16 @@ findprovs_expect() {
     test_cmp findprovsOut expected
   '
 }
+
+purge_blockstore() {
+  ipfs pin ls --quiet --type=recursive | ipfs pin rm &>/dev/null
+  ipfs repo gc --silent &>/dev/null
+
+  test_expect_success "pinlist empty" '
+    [[ -z "$( ipfs pin ls )" ]]
+  '
+  test_expect_success "nothing left to gc" '
+    [[ -z "$( ipfs repo gc )" ]]
+  '
+}
+

test/sharness/t0117-gateway-block.sh

@@ -5,12 +5,12 @@ test_description="Test HTTP Gateway Raw Block (application/vnd.ipld.raw) Support
 . lib/test-lib.sh
 
 test_init_ipfs
-test_launch_ipfs_daemon
+test_launch_ipfs_daemon_without_network
 
 test_expect_success "Create text fixtures" '
   mkdir -p dir &&
-  echo "hello" > dir/ascii.txt &&
-  ROOT_DIR_CID=$(ipfs add -Qrw --cid-version 1 dir)
+  echo "hello application/vnd.ipld.raw" > dir/ascii.txt &&
+  ROOT_DIR_CID=$(ipfs add -Qrw --cid-version 1 dir) &&
   FILE_CID=$(ipfs resolve -r /ipfs/$ROOT_DIR_CID/dir/ascii.txt | cut -d "/" -f3)
 '
 

test/sharness/t0118-gateway-car.sh

@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+
+test_description="Test HTTP Gateway CAR (application/vnd.ipld.car) Support"
+
+. lib/test-lib.sh
+
+test_init_ipfs
+test_launch_ipfs_daemon_without_network
+
+# CAR stream is not deterministic, as blocks can arrive in random order,
+# but if we have a small file that fits into a single block, and export its CID
+# we will get a CAR that is a deterministic array of bytes.
+
+test_expect_success "Create a deterministic CAR for testing" '
+  mkdir -p subdir &&
+  echo "hello application/vnd.ipld.car" > subdir/ascii.txt &&
+  ROOT_DIR_CID=$(ipfs add -Qrw --cid-version 1 subdir) &&
+  FILE_CID=$(ipfs resolve -r /ipfs/$ROOT_DIR_CID/subdir/ascii.txt | cut -d "/" -f3) &&
+  ipfs dag export $ROOT_DIR_CID > test-dag.car &&
+  ipfs dag export $FILE_CID > deterministic.car &&
+  purge_blockstore
+'
+
+# GET unixfs file as CAR
+# (by using a single file we ensure deterministic result that can be compared byte-for-byte)
+
+    test_expect_success "GET with format=car param returns a CARv1 stream" '
+    ipfs dag import test-dag.car &&
+    curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt?format=car" -o gateway-param.car &&
+    test_cmp deterministic.car gateway-param.car
+    '
+
+    test_expect_success "GET for application/vnd.ipld.car returns a CARv1 stream" '
+    ipfs dag import test-dag.car &&
+    curl -sX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" -o gateway-header.car &&
+    test_cmp deterministic.car gateway-header.car
+    '
+
+    # explicit version=1
+    test_expect_success "GET for application/vnd.ipld.raw version=1 returns a CARv1 stream" '
+    ipfs dag import test-dag.car &&
+    curl -sX GET -H "Accept: application/vnd.ipld.car; version=1" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" -o gateway-header-v1.car &&
+    test_cmp deterministic.car gateway-header-v1.car
+    '
+
+# GET unixfs directory as a CAR with DAG and some selector
+
+    # TODO: this is basic test for "full" selector, we will add support for custom ones in https://github.com/ipfs/go-ipfs/issues/8769
+    test_expect_success "GET for application/vnd.ipld.car with unixfs dir returns a CARv1 stream with full DAG" '
+    ipfs dag import test-dag.car &&
+    curl -sX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID" -o gateway-dir.car &&
+    purge_blockstore &&
+    ipfs dag import gateway-dir.car &&
+    ipfs dag stat --offline $ROOT_DIR_CID
+    '
+
+# Make sure expected HTTP headers are returned with the block bytes
+
+    test_expect_success "GET response for application/vnd.ipld.car has expected Content-Type" '
+    ipfs dag import test-dag.car &&
+    curl -svX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" >/dev/null 2>curl_output &&
+    cat curl_output &&
+    grep "< Content-Type: application/vnd.ipld.car; version=1" curl_output
+    '
+
+    # CAR is streamed, gateway may not have the entire thing, unable to calculate total size
+    test_expect_success "GET response for application/vnd.ipld.car includes no Content-Length" '
+    grep -qv "< Content-Length:" curl_output
+    '
+
+    test_expect_success "GET response for application/vnd.ipld.car includes Content-Disposition" '
+    grep "< Content-Disposition: attachment\; filename=\"${FILE_CID}.car\"" curl_output
+    '
+
+    test_expect_success "GET response for application/vnd.ipld.car includes nosniff hint" '
+    grep "< X-Content-Type-Options: nosniff" curl_output
+    '
+
+# Cache control HTTP headers
+
+    test_expect_success "GET response for application/vnd.ipld.car includes a weak Etag" '
+    grep "< Etag: W/\"${FILE_CID}.car\"" curl_output
+    '
+
+    # (basic checks, detailed behavior for some fields is tested in  t0116-gateway-cache.sh)
+    test_expect_success "GET response for application/vnd.ipld.car includes X-Ipfs-Path and X-Ipfs-Roots" '
+    grep "< X-Ipfs-Path" curl_output &&
+    grep "< X-Ipfs-Roots" curl_output
+    '
+
+    test_expect_success "GET response for application/vnd.ipld.raw includes expected Cache-Control" '
+    grep "< Cache-Control: no-cache, no-transform" curl_output
+    '
+
+test_kill_ipfs_daemon
+
+test_done