feat: add specs and user taxonomies

CHANGELOG.md

@@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Metadata logging used to associate tests with custom data like versions, specs identifiers, etc.
 - Output Github's workflow URL with metadata. [PR](https://github.com/ipfs/gateway-conformance/pull/145)
 - Basic Dashboard Output with content generation. [PR](https://github.com/ipfs/gateway-conformance/pull/152)
+- Test Group Metadata on Tests. [PR](https://github.com/ipfs/gateway-conformance/pull/156)
+- Specs Metadata on Tests. [PR](https://github.com/ipfs/gateway-conformance/pull/159)
 
 ## [0.3.0] - 2023-07-31
 ### Added

aggregate-into-table.js

@@ -1,6 +1,7 @@
 const fs = require("fs");
 
 const TestMetadata = "TestMetadata";
+const METADATA_TEST_GROUP = "group";
 
 // retrieve the list of input files from the command line
 const files = process.argv.slice(2);
@@ -10,24 +11,85 @@ const inputs = files.map((file) => {
     return JSON.parse(fs.readFileSync(file, 'utf8'));
 });
 
-// merge all the unique keys from all the inputs
-let keys = new Set();
+// merge all the unique keys & metadata from all the inputs
+const metadata = {}
 inputs.forEach((input) => {
     Object.keys(input).forEach((key) => {
-        keys.add(key);
+        metadata[key] = { ...metadata[key], ...input[key]["meta"] || {} };
     });
 });
-keys.delete(TestMetadata); // Extract TestMetadata which is a special case
-keys = Array.from(keys).sort();
+delete metadata[TestMetadata]; // Extract TestMetadata which is a special case
+
+// generate groups: an array of {group, key} objects
+// where group is the group name (or undefined), and key is the test key name (or undefined)
+// It represents the table leftmost column.
+//
+// Group1
+//  Group1 - Test1
+//  Group1 - Test2
+// Group2
+// ...
+const groups = []
+const groupsAdded = new Set();
+Object.entries(metadata).forEach(([key, value]) => {
+    const group = value[METADATA_TEST_GROUP] || undefined;
+
+    if (!groupsAdded.has(group)) {
+        groups.push({ group, key: undefined });
+        groupsAdded.add(group);
+    }
+
+    groups.push({ group, key });
+});
+
+// sort the groups so that the tests are ordered by group, then by key.
+// undefined groups are always at the end.
+groups.sort((a, b) => {
+    if (a.group === b.group) {
+        if (a.key === undefined) {
+            return -1;
+        }
+        if (b.key === undefined) {
+            return 1;
+        }
+        return a.key.localeCompare(b.key);
+    }
+
+    if (a.group === undefined) {
+        return 1;
+    }
+
+    if (b.group === undefined) {
+        return -1;
+    }
+
+    return a.group.localeCompare(b.group);
+});
 
 // generate a table
 const columns = [];
 
-// add the leading column ("gateway", "version", "key1", "key2", ... "keyN")
+// add the leading column ("gateway", "version", "group1", "test11", ... "test42")
 const leading = ["gateway", "version"];
-keys.forEach((key) => {
+groups.forEach(({ group, key }) => {
+    if (key === undefined) {
+        leading.push(`**${group || 'Other'}**`);
+        return;
+    }
+
     // Skip the "Test" prefix
-    const niceKey = key.replace(/^Test/, '');
+    let niceKey = key.replace(/^Test/, '');
+
+    const m = metadata[key];
+    if (m.specs && m.specs.length > 0) {
+        if (m.specs.length === 1) {
+            niceKey = `[${niceKey}](https://${m.specs[0]})`
+        } else {
+            const urls = m.specs.map((url, index) => `[${index}](https://${url})`);
+            niceKey = `${niceKey} (${urls.join(', ')})`;
+        }
+    }
+
     leading.push(niceKey);
 });
 columns.push(leading);
@@ -73,7 +135,11 @@ inputs.forEach((input, index) => {
     const col = [name, versionCell];
 
     // extract results
-    keys.forEach((key) => {
+    groups.forEach(({ group, key }) => {
+        if (key === undefined) {
+            col.push(null);
+            return;
+        }
         col.push(cellRender(input[key] || null));
     });
     columns.push(col);

aggregate.js

@@ -16,7 +16,7 @@ lines = lines.filter((line) => {
 // # extract test metadata
 //   action is output, and starts with ".* --- META: (.*)"
 //   see details in https://github.com/ipfs/gateway-conformance/pull/125
-const getMetadata = (line) => {
+const extractMetadata = (line) => {
   const { Action, Output } = line;
 
   if (Action !== "output") {
@@ -34,7 +34,7 @@ const getMetadata = (line) => {
 }
 
 lines = lines.map((line) => {
-  const metadata = getMetadata(line);
+  const metadata = extractMetadata(line);
 
   if (!metadata) {
     return line;
@@ -87,6 +87,44 @@ lines.forEach((line) => {
   });
 })
 
+// prepare metadata up the tree
+const metadataTree = {};
+
+// sort lines so that the one with the longest path is processed first
+const sortedLines = lines.sort((a, b) => {
+  return b.Path.length - a.Path.length;
+});
+
+sortedLines.forEach((line) => {
+  const { Path, Action, Metadata } = line;
+  let current = metadataTree;
+
+  if (Action !== "meta") {
+    return;
+  }
+
+  Path.forEach((path) => {
+    if (!current[path]) {
+      current[path] = {};
+    }
+    current = current[path];
+    current["meta"] = { ...current["meta"], ...Metadata };
+  });
+});
+
+const getMetadata = (path) => {
+  let current = metadataTree;
+
+  path.forEach((path) => {
+    if (!current[path]) {
+      return null;
+    }
+    current = current[path];
+  });
+
+  return current["meta"];
+}
+
 // # Drop all lines where the Test "Path" does not point to a leaf
 //   if the test has children then we don't really care about it's pass / fail / skip status,
 //   we'll aggregate its children results'
@@ -118,7 +156,7 @@ lines.forEach((line) => {
   const key = path.join(" > ");
 
   if (!current[key]) {
-    current[key] = { Path: path, "pass": 0, "fail": 0, "skip": 0, "total": 0, "meta": {} };
+    current[key] = { Path: path, "pass": 0, "fail": 0, "skip": 0, "total": 0, "meta": getMetadata(path) || {} };
   }
   current = current[key];
 

tests/dnslink_gateway_test.go

@@ -4,6 +4,7 @@ import (
 	"net/url"
 	"testing"
 
+	"github.com/ipfs/gateway-conformance/tooling"
 	"github.com/ipfs/gateway-conformance/tooling/car"
 	. "github.com/ipfs/gateway-conformance/tooling/check"
 	"github.com/ipfs/gateway-conformance/tooling/dnslink"
@@ -14,6 +15,8 @@ import (
 )
 
 func TestDNSLinkGatewayUnixFSDirectoryListing(t *testing.T) {
+	tooling.LogTestGroup(t, GroupDNSLink)
+
 	fixture := car.MustOpenUnixfsCar("dir_listing/fixtures.car")
 	file := fixture.MustGetNode("ą", "ę", "file-źł.txt")
 

tests/metadata_test.go

@@ -4,10 +4,32 @@ import (
 	"testing"
 
 	"github.com/ipfs/gateway-conformance/tooling"
+	"github.com/ipfs/gateway-conformance/tooling/test"
 )
 
+func logGatewayURL(t *testing.T) {
+	tooling.LogMetadata(t, struct {
+		GatewayURL          string `json:"gateway_url"`
+		SubdomainGatewayURL string `json:"subdomain_gateway_url"`
+	}{
+		GatewayURL:          test.GatewayURL,
+		SubdomainGatewayURL: test.SubdomainGatewayURL,
+	})
+}
+
 func TestMetadata(t *testing.T) {
 	tooling.LogVersion(t)
 	tooling.LogJobURL(t)
-	tooling.LogGatewayURL(t)
+	logGatewayURL(t)
 }
+
+const (
+	GroupSubdomains = "Subdomains"
+	GroupCORS       = "CORS"
+	GroupIPNS       = "IPNS"
+	GroupDNSLink    = "DNSLink"
+	GroupJSONCbor   = "JSON-CBOR"
+	GroupBlockCar   = "Block-CAR"
+	GroupTar        = "Tar"
+	GroupUnixFS     = "UnixFS"
+)

tests/path_gateway_cors_test.go

@@ -3,11 +3,14 @@ package tests
 import (
 	"testing"
 
+	"github.com/ipfs/gateway-conformance/tooling"
 	"github.com/ipfs/gateway-conformance/tooling/specs"
 	. "github.com/ipfs/gateway-conformance/tooling/test"
 )
 
 func TestCors(t *testing.T) {
+	tooling.LogTestGroup(t, GroupCORS)
+
 	cidHello := "bafkqabtimvwgy3yk" // hello
 
 	tests := SugarTests{

tests/path_gateway_dag_test.go

@@ -3,6 +3,7 @@ package tests
 import (
 	"testing"
 
+	"github.com/ipfs/gateway-conformance/tooling"
 	"github.com/ipfs/gateway-conformance/tooling/car"
 	. "github.com/ipfs/gateway-conformance/tooling/check"
 	"github.com/ipfs/gateway-conformance/tooling/ipns"
@@ -12,6 +13,8 @@ import (
 )
 
 func TestGatewayJsonCbor(t *testing.T) {
+	tooling.LogTestGroup(t, GroupJSONCbor)
+
 	fixture := car.MustOpenUnixfsCar("path_gateway_dag/gateway-json-cbor.car")
 
 	fileJSON := fixture.MustGetNode("ą", "ę", "t.json")
@@ -39,6 +42,7 @@ func TestGatewayJsonCbor(t *testing.T) {
 		},
 		{
 			Name: "GET UnixFS file with JSON bytes is returned with application/json Content-Type - with headers",
+			Spec: "specs.ipfs.tech/http-gateways/path-gateway/#accept-request-header",
 			Hint: `
 			## Quick regression check for JSON stored on UnixFS:
 			## it has nothing to do with DAG-JSON and JSON codecs,
@@ -66,6 +70,8 @@ func TestGatewayJsonCbor(t *testing.T) {
 // ## Reading UnixFS (data encoded with dag-pb codec) as DAG-CBOR and DAG-JSON
 // ## (returns representation defined in https://ipld.io/specs/codecs/dag-pb/spec/#logical-format)
 func TestDagPbConversion(t *testing.T) {
+	tooling.LogTestGroup(t, GroupJSONCbor)
+
 	fixture := car.MustOpenUnixfsCar("path_gateway_dag/gateway-json-cbor.car")
 
 	dir := fixture.MustGetRoot()
@@ -205,6 +211,8 @@ func TestDagPbConversion(t *testing.T) {
 // # Requesting CID with plain json (0x0200) and cbor (0x51) codecs
 // # (note these are not UnixFS, not DAG-* variants, just raw block identified by a CID with a special codec)
 func TestPlainCodec(t *testing.T) {
+	tooling.LogTestGroup(t, GroupJSONCbor)
+
 	table := []struct {
 		Name        string
 		Format      string
@@ -308,6 +316,8 @@ func TestPlainCodec(t *testing.T) {
 
 // ## Pathing, traversal over DAG-JSON and DAG-CBOR
 func TestPathing(t *testing.T) {
+	tooling.LogTestGroup(t, GroupJSONCbor)
+
 	dagJSONTraversal := car.MustOpenUnixfsCar("path_gateway_dag/dag-json-traversal.car").MustGetRoot()
 	dagCBORTraversal := car.MustOpenUnixfsCar("path_gateway_dag/dag-cbor-traversal.car").MustGetRoot()
 
@@ -381,6 +391,8 @@ func TestPathing(t *testing.T) {
 // ## NATIVE TESTS for DAG-JSON (0x0129) and DAG-CBOR (0x71):
 // ## DAG- regression tests for core behaviors when native DAG-(CBOR|JSON) is requested
 func TestNativeDag(t *testing.T) {
+	tooling.LogTestGroup(t, GroupJSONCbor)
+
 	missingCID := car.RandomCID()
 
 	table := []struct {
@@ -467,7 +479,7 @@ func TestNativeDag(t *testing.T) {
 				Response: Expect().
 					Headers(
 						Header("Content-Type").Hint("expected Content-Type").Equals("application/vnd.ipld.dag-{{format}}", row.Format),
-						Header("Content-Length").Hint("includes Content-Length").Equals("{{length}}", len(dagTraversal.RawData())),
+						Header("Content-Length").Spec("specs.ipfs.tech/http-gateways/path-gateway/#content-disposition-response-header").Hint("includes Content-Length").Equals("{{length}}", len(dagTraversal.RawData())),
 						Header("Content-Disposition").Hint("includes Content-Disposition").Contains(`{{disposition}}; filename="{{cid}}.{{format}}"`, row.Disposition, dagTraversalCID, row.Format),
 						Header("X-Content-Type-Options").Hint("includes nosniff hint").Contains("nosniff"),
 					),
@@ -541,6 +553,7 @@ func TestNativeDag(t *testing.T) {
 			},
 			{
 				Name: Fmt("HEAD {{name}} with only-if-cached for missing block returns HTTP 412 Precondition Failed", row.Name),
+				Spec: "specs.ipfs.tech/http-gateways/path-gateway/#only-if-cached",
 				Request: Request().
 					Path("/ipfs/{{cid}}", missingCID).
 					Header("Cache-Control", "only-if-cached").
@@ -570,6 +583,8 @@ func TestNativeDag(t *testing.T) {
 }
 
 func TestGatewayJSONCborAndIPNS(t *testing.T) {
+	tooling.LogTestGroup(t, GroupIPNS)
+
 	ipnsIdDagJSON := "k51qzi5uqu5dhjghbwdvbo6mi40htrq6e2z4pwgp15pgv3ho1azvidttzh8yy2"
 	ipnsIdDagCBOR := "k51qzi5uqu5dghjous0agrwavl8vzl64xckoqzwqeqwudfr74kfd11zcyk3b7l"
 

tests/path_gateway_ipns_test.go

@@ -3,11 +3,14 @@ package tests
 import (
 	"testing"
 
+	"github.com/ipfs/gateway-conformance/tooling"
 	"github.com/ipfs/gateway-conformance/tooling/specs"
 	. "github.com/ipfs/gateway-conformance/tooling/test"
 )
 
 func TestRedirectCanonicalIPNS(t *testing.T) {
+	tooling.LogTestGroup(t, GroupIPNS)
+
 	tests := SugarTests{
 		{
 			Name: "GET for /ipns/{b58-multihash-of-ed25519-key} redirects to /ipns/{cidv1-libp2p-key-base36}",

tests/path_gateway_raw_test.go

@@ -5,12 +5,15 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/ipfs/gateway-conformance/tooling"
 	"github.com/ipfs/gateway-conformance/tooling/car"
 	"github.com/ipfs/gateway-conformance/tooling/specs"
 	. "github.com/ipfs/gateway-conformance/tooling/test"
 )
 
 func TestGatewayBlock(t *testing.T) {
+	tooling.LogTestGroup(t, GroupBlockCar)
+
 	fixture := car.MustOpenUnixfsCar("gateway-raw-block.car")
 
 	tests := SugarTests{