From 77e34fc115cbfb82585fd954bcf389ecebf655bc Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 28 Feb 2024 10:27:15 +0000 Subject: [PATCH] fix: update project deps and docs --- .release-please-manifest.json | 3 +- .release-please.json | 3 +- README.md | 22 +- packages/block-brokers/README.md | 2 + packages/block-brokers/package.json | 10 +- packages/car/README.md | 15 + packages/car/package.json | 10 +- packages/dag-cbor/README.md | 15 + packages/dag-cbor/package.json | 8 +- packages/dag-json/README.md | 15 + packages/dag-json/package.json | 8 +- packages/helia/README.md | 17 + packages/helia/package.json | 48 +- packages/http/README.md | 17 + packages/http/package.json | 10 +- packages/interface/README.md | 17 + packages/interface/package.json | 12 +- packages/interop/README.md | 2 + packages/interop/package.json | 28 +- .../interop/src/verified-fetch-json.spec.ts | 47 -- .../src/verified-fetch-unixfs-dir.spec.ts | 94 --- .../src/verified-fetch-websites.spec.ts | 85 --- packages/interop/tsconfig.json | 3 - packages/ipns/README.md | 17 +- packages/ipns/package.json | 20 +- packages/json/README.md | 15 + packages/json/package.json | 6 +- packages/mfs/README.md | 15 + packages/mfs/package.json | 16 +- packages/routers/README.md | 17 + packages/routers/package.json | 12 +- packages/strings/README.md | 15 + packages/strings/package.json | 8 +- packages/unixfs/README.md | 15 + packages/unixfs/package.json | 16 +- packages/utils/README.md | 17 + packages/utils/package.json | 30 +- packages/verified-fetch/.aegir.js | 8 - packages/verified-fetch/LICENSE | 4 - packages/verified-fetch/LICENSE-APACHE | 5 - packages/verified-fetch/LICENSE-MIT | 19 - packages/verified-fetch/README.md | 531 --------------- packages/verified-fetch/package.json | 197 ------ packages/verified-fetch/src/index.ts | 632 ------------------ packages/verified-fetch/src/singleton.ts | 20 - packages/verified-fetch/src/types.ts | 1 - .../src/utils/dag-cbor-to-safe-json.ts | 44 -- .../utils/get-content-disposition-filename.ts | 18 - .../verified-fetch/src/utils/get-e-tag.ts | 36 - .../utils/get-stream-from-async-iterable.ts | 45 -- .../src/utils/get-tar-stream.ts | 68 -- .../src/utils/parse-resource.ts | 40 -- .../src/utils/parse-url-string.ts | 154 ----- .../verified-fetch/src/utils/responses.ts | 29 - .../src/utils/select-output-type.ts | 167 ----- packages/verified-fetch/src/utils/tlru.ts | 52 -- .../verified-fetch/src/utils/walk-path.ts | 34 - packages/verified-fetch/src/verified-fetch.ts | 510 -------------- .../verified-fetch/test/accept-header.spec.ts | 249 ------- packages/verified-fetch/test/car.spec.ts | 69 -- .../test/content-type-parser.spec.ts | 135 ---- .../test/custom-dns-resolvers.spec.ts | 52 -- packages/verified-fetch/test/fixtures/cids.ts | 18 - .../test/fixtures/create-offline-helia.ts | 20 - .../test/fixtures/memory-car.ts | 33 - .../verified-fetch/test/get-e-tag.spec.ts | 33 - .../get-stream-from-async-iterable.spec.ts | 74 -- packages/verified-fetch/test/index.spec.ts | 53 -- .../verified-fetch/test/ipns-record.spec.ts | 89 --- .../test/parse-resource.spec.ts | 32 - .../test/parse-url-string.spec.ts | 323 --------- packages/verified-fetch/test/tar.spec.ts | 149 ----- .../get-content-disposition-filename.spec.ts | 16 - .../test/utils/select-output-type.spec.ts | 41 -- .../test/verified-fetch.spec.ts | 554 --------------- packages/verified-fetch/tsconfig.json | 42 -- packages/verified-fetch/typedoc.json | 5 - 77 files changed, 344 insertions(+), 4967 deletions(-) delete mode 100644 packages/interop/src/verified-fetch-json.spec.ts delete mode 100644 packages/interop/src/verified-fetch-unixfs-dir.spec.ts delete mode 100644 packages/interop/src/verified-fetch-websites.spec.ts delete mode 100644 packages/verified-fetch/.aegir.js delete mode 100644 packages/verified-fetch/LICENSE delete mode 100644 packages/verified-fetch/LICENSE-APACHE delete mode 100644 packages/verified-fetch/LICENSE-MIT delete mode 100644 packages/verified-fetch/README.md delete mode 100644 packages/verified-fetch/package.json delete mode 100644 packages/verified-fetch/src/index.ts delete mode 100644 packages/verified-fetch/src/singleton.ts delete mode 100644 packages/verified-fetch/src/types.ts delete mode 100644 packages/verified-fetch/src/utils/dag-cbor-to-safe-json.ts delete mode 100644 packages/verified-fetch/src/utils/get-content-disposition-filename.ts delete mode 100644 packages/verified-fetch/src/utils/get-e-tag.ts delete mode 100644 packages/verified-fetch/src/utils/get-stream-from-async-iterable.ts delete mode 100644 packages/verified-fetch/src/utils/get-tar-stream.ts delete mode 100644 packages/verified-fetch/src/utils/parse-resource.ts delete mode 100644 packages/verified-fetch/src/utils/parse-url-string.ts delete mode 100644 packages/verified-fetch/src/utils/responses.ts delete mode 100644 packages/verified-fetch/src/utils/select-output-type.ts delete mode 100644 packages/verified-fetch/src/utils/tlru.ts delete mode 100644 packages/verified-fetch/src/utils/walk-path.ts delete mode 100644 packages/verified-fetch/src/verified-fetch.ts delete mode 100644 packages/verified-fetch/test/accept-header.spec.ts delete mode 100644 packages/verified-fetch/test/car.spec.ts delete mode 100644 packages/verified-fetch/test/content-type-parser.spec.ts delete mode 100644 packages/verified-fetch/test/custom-dns-resolvers.spec.ts delete mode 100644 packages/verified-fetch/test/fixtures/cids.ts delete mode 100644 packages/verified-fetch/test/fixtures/create-offline-helia.ts delete mode 100644 packages/verified-fetch/test/fixtures/memory-car.ts delete mode 100644 packages/verified-fetch/test/get-e-tag.spec.ts delete mode 100644 packages/verified-fetch/test/get-stream-from-async-iterable.spec.ts delete mode 100644 packages/verified-fetch/test/index.spec.ts delete mode 100644 packages/verified-fetch/test/ipns-record.spec.ts delete mode 100644 packages/verified-fetch/test/parse-resource.spec.ts delete mode 100644 packages/verified-fetch/test/parse-url-string.spec.ts delete mode 100644 packages/verified-fetch/test/tar.spec.ts delete mode 100644 packages/verified-fetch/test/utils/get-content-disposition-filename.spec.ts delete mode 100644 packages/verified-fetch/test/utils/select-output-type.spec.ts delete mode 100644 packages/verified-fetch/test/verified-fetch.spec.ts delete mode 100644 packages/verified-fetch/tsconfig.json delete mode 100644 packages/verified-fetch/typedoc.json diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 3bab71b9..fe2a5bbc 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -13,6 +13,5 @@ "packages/unixfs": "3.0.0", "packages/utils": "0.0.1", "packages/http": "1.0.1", - "packages/routers": "1.0.0", - "packages/verified-fetch": "0.0.0" + "packages/routers": "1.0.0" } diff --git a/.release-please.json b/.release-please.json index f38bc1fb..10d16aa8 100644 --- a/.release-please.json +++ b/.release-please.json @@ -23,7 +23,6 @@ "packages/routers": {}, "packages/strings": {}, "packages/unixfs": {}, - "packages/utils": {}, - "packages/verified-fetch": {} + "packages/utils": {} } } diff --git a/README.md b/README.md index 172fd321..839986b6 100644 --- a/README.md +++ b/README.md @@ -167,22 +167,23 @@ The basic Helia API is defined in: The API is implemented by: -- [`/packages/helia`](./packages/helia) An implementation of the Helia API +- [`/packages/helia`](./packages/helia) An peer to peer implementation that uses [bitswap](https://docs.ipfs.tech/concepts/bitswap/), [libp2p](https://www.npmjs.com/package/libp2p) and [HTTP gateways](https://docs.ipfs.tech/reference/http/gateway/) as fallback +- [`/packages/http`](./packages/http) A lightweight implementation that uses [HTTP gateways](https://docs.ipfs.tech/reference/http/gateway/) exclusively -Helia also ships a number of supplemental libraries and tools. +Helia also ships a number of supplemental libraries and tools that can be combined with Helia API implementations to accomplish tasks in distributed and trustless ways. These libraries are not intended to be the "one true implementation" of any given API, but are made available for users to include depending on the need of their particular application: -- [./packages/unixfs](./packages/unixfs) The `@helia/unixfs` module +- [./packages/car](./packages/car) The `@helia/car` module +- [./packages/dag-cbor](./packages/dag-cbor) The `@helia/dag-cbor` module +- [./packages/dag-json](./packages/dag-json) The `@helia/dag-json` module +- [./packages/ipns](./packages/ipns) The `@helia/ipns` module +- [./packages/json](./packages/json) The `@helia/json` module - [./packages/mfs](./packages/mfs) The `@helia/mfs` module -- [./packages/ipns](./packages/ipns) `@helia/ipns` -- [./packages/car](./packages/car) `@helia/car` -- [./packages/strings](./packages/strings) `@helia/strings` -- [./packages/json](./packages/json) `@helia/json` -- [./packages/dag-json](./packages/dag-json) `@helia/dag-json` -- [./packages/dag-cbor](./packages/dag-cbor) `@helia/dag-cbor` +- [./packages/strings](./packages/strings) The `@helia/strings` module +- [./packages/unixfs](./packages/unixfs) The `@helia/unixfs` module -An interface suite ensures everything is compatible: +An interop suite ensures everything is compatible: - [`/packages/interop`](./packages/interop) Interop tests for Helia @@ -190,6 +191,7 @@ An interface suite ensures everything is compatible: There are several other modules available outside this repo: +- [`@helia/verified-fetch`](https://github.com/ipfs/helia-verified-fetch) A [fetch](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API)-like API for retrieving trustless, verified content from the distributed web - [`@helia/delegated-routing-v1-http-api`](https://github.com/ipfs/helia-delegated-routing-v1-http-api) An implementation of the [Delegated Routing v1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/) including a server and a client - [Helia WNFS](https://github.com/shovelers/helia-wnfs) a [WNFS](https://guide.fission.codes/developers/webnative/file-system-wnfs) implementation built on top of Helia - [`@helia/remote-pinning`](https://github.com/ipfs/helia-remote-pinning) A Helia client for communicating with [IPFS Pinning Services](https://ipfs.github.io/pinning-services-api-spec/) diff --git a/packages/block-brokers/README.md b/packages/block-brokers/README.md index a00f1546..3415ed81 100644 --- a/packages/block-brokers/README.md +++ b/packages/block-brokers/README.md @@ -4,6 +4,8 @@

+# @helia/block-brokers + [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) [![codecov](https://img.shields.io/codecov/c/github/ipfs/helia.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia) diff --git a/packages/block-brokers/package.json b/packages/block-brokers/package.json index d51aef04..3a237e5d 100644 --- a/packages/block-brokers/package.json +++ b/packages/block-brokers/package.json @@ -54,16 +54,16 @@ }, "dependencies": { "@helia/interface": "^4.0.0", - "@libp2p/interface": "^1.1.2", - "interface-blockstore": "^5.2.9", + "@libp2p/interface": "^1.1.3", + "interface-blockstore": "^5.2.10", "ipfs-bitswap": "^20.0.2", - "multiformats": "^13.0.1", + "multiformats": "^13.1.0", "progress-events": "^1.0.0" }, "devDependencies": { - "@libp2p/logger": "^4.0.5", + "@libp2p/logger": "^4.0.6", "@types/sinon": "^17.0.3", - "aegir": "^42.2.2", + "aegir": "^42.2.5", "sinon": "^17.0.1", "sinon-ts": "^2.0.0" } diff --git a/packages/car/README.md b/packages/car/README.md index bb39fea7..40829455 100644 --- a/packages/car/README.md +++ b/packages/car/README.md @@ -15,6 +15,21 @@ # About + + `@helia/car` provides `import` and `export` methods to read/write Car files to Helia's blockstore. See the Car interface for all available operations. diff --git a/packages/car/package.json b/packages/car/package.json index dfa5ce43..1ace3b1c 100644 --- a/packages/car/package.json +++ b/packages/car/package.json @@ -140,20 +140,20 @@ }, "dependencies": { "@helia/interface": "^4.0.0", - "@ipld/car": "^5.2.6", + "@ipld/car": "^5.3.0", "@libp2p/interfaces": "^3.3.2", - "interface-blockstore": "^5.2.9", + "interface-blockstore": "^5.2.10", "it-drain": "^3.0.5", "it-map": "^3.0.5", - "multiformats": "^13.0.1", + "multiformats": "^13.1.0", "p-defer": "^4.0.0", "p-queue": "^8.0.1", "progress-events": "^1.0.0" }, "devDependencies": { "@helia/unixfs": "^3.0.0", - "@ipld/dag-pb": "^4.0.8", - "aegir": "^42.2.2", + "@ipld/dag-pb": "^4.1.0", + "aegir": "^42.2.5", "blockstore-core": "^4.4.0", "ipfs-unixfs-importer": "^15.2.4", "it-to-buffer": "^4.0.5" diff --git a/packages/dag-cbor/README.md b/packages/dag-cbor/README.md index e158ce42..cb7586e1 100644 --- a/packages/dag-cbor/README.md +++ b/packages/dag-cbor/README.md @@ -15,6 +15,21 @@ # About + + `@helia/dag-cbor` makes working with DAG-JSON Helia simple & straightforward. See the DAGCBOR interface for all available operations. diff --git a/packages/dag-cbor/package.json b/packages/dag-cbor/package.json index 68527fce..060347c0 100644 --- a/packages/dag-cbor/package.json +++ b/packages/dag-cbor/package.json @@ -141,14 +141,14 @@ }, "dependencies": { "@helia/interface": "^4.0.0", - "@ipld/dag-cbor": "^9.1.0", + "@ipld/dag-cbor": "^9.2.0", "@libp2p/interfaces": "^3.3.2", - "interface-blockstore": "^5.2.9", - "multiformats": "^13.0.1", + "interface-blockstore": "^5.2.10", + "multiformats": "^13.1.0", "progress-events": "^1.0.0" }, "devDependencies": { - "aegir": "^42.2.2", + "aegir": "^42.2.5", "blockstore-core": "^4.4.0" } } diff --git a/packages/dag-json/README.md b/packages/dag-json/README.md index 64eb3857..1d364dd1 100644 --- a/packages/dag-json/README.md +++ b/packages/dag-json/README.md @@ -15,6 +15,21 @@ # About + + `@helia/dag-json` makes working with DAG-JSON Helia simple & straightforward. See the DAGJSON interface for all available operations. diff --git a/packages/dag-json/package.json b/packages/dag-json/package.json index a44ec2ea..cdfe3549 100644 --- a/packages/dag-json/package.json +++ b/packages/dag-json/package.json @@ -140,14 +140,14 @@ }, "dependencies": { "@helia/interface": "^4.0.0", - "@ipld/dag-json": "^10.1.7", + "@ipld/dag-json": "^10.2.0", "@libp2p/interfaces": "^3.3.2", - "interface-blockstore": "^5.2.9", - "multiformats": "^13.0.1", + "interface-blockstore": "^5.2.10", + "multiformats": "^13.1.0", "progress-events": "^1.0.0" }, "devDependencies": { - "aegir": "^42.2.2", + "aegir": "^42.2.5", "blockstore-core": "^4.4.0" } } diff --git a/packages/helia/README.md b/packages/helia/README.md index 8586be0f..be75a1d6 100644 --- a/packages/helia/README.md +++ b/packages/helia/README.md @@ -4,6 +4,8 @@

+# helia + [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) [![codecov](https://img.shields.io/codecov/c/github/ipfs/helia.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia) @@ -13,6 +15,21 @@ # About + + Exports a `createHelia` function that returns an object that implements the Helia API. Pass it to other modules like @helia/unixfs to make files available on the distributed web. diff --git a/packages/helia/package.json b/packages/helia/package.json index 5c191dde..4400c285 100644 --- a/packages/helia/package.json +++ b/packages/helia/package.json @@ -55,41 +55,41 @@ }, "dependencies": { "@chainsafe/libp2p-noise": "^15.0.0", - "@chainsafe/libp2p-yamux": "^6.0.1", + "@chainsafe/libp2p-yamux": "^6.0.2", "@helia/block-brokers": "^2.0.1", "@helia/delegated-routing-v1-http-api-client": "^3.0.0", "@helia/interface": "^4.0.0", "@helia/routers": "^1.0.0", "@helia/utils": "^0.0.1", - "@libp2p/autonat": "^1.0.10", - "@libp2p/bootstrap": "^10.0.13", - "@libp2p/circuit-relay-v2": "^1.0.13", - "@libp2p/dcutr": "^1.0.10", - "@libp2p/identify": "^1.0.12", - "@libp2p/interface": "^1.1.2", - "@libp2p/kad-dht": "^12.0.5", - "@libp2p/keychain": "^4.0.7", - "@libp2p/logger": "^4.0.5", - "@libp2p/mdns": "^10.0.13", - "@libp2p/mplex": "^10.0.13", - "@libp2p/ping": "^1.0.10", - "@libp2p/tcp": "^9.0.13", - "@libp2p/upnp-nat": "^1.0.11", - "@libp2p/webrtc": "^4.0.17", - "@libp2p/websockets": "^8.0.13", - "@libp2p/webtransport": "^4.0.17", + "@libp2p/autonat": "^1.0.12", + "@libp2p/bootstrap": "^10.0.15", + "@libp2p/circuit-relay-v2": "^1.0.15", + "@libp2p/dcutr": "^1.0.12", + "@libp2p/identify": "^1.0.14", + "@libp2p/interface": "^1.1.3", + "@libp2p/kad-dht": "^12.0.7", + "@libp2p/keychain": "^4.0.8", + "@libp2p/logger": "^4.0.6", + "@libp2p/mdns": "^10.0.15", + "@libp2p/mplex": "^10.0.15", + "@libp2p/ping": "^1.0.11", + "@libp2p/tcp": "^9.0.15", + "@libp2p/upnp-nat": "^1.0.13", + "@libp2p/webrtc": "^4.0.19", + "@libp2p/websockets": "^8.0.15", + "@libp2p/webtransport": "^4.0.19", "blockstore-core": "^4.4.0", - "datastore-core": "^9.2.7", - "interface-blockstore": "^5.2.9", - "interface-datastore": "^8.2.10", + "datastore-core": "^9.2.9", + "interface-blockstore": "^5.2.10", + "interface-datastore": "^8.2.11", "ipns": "^9.0.0", - "libp2p": "^1.2.1", - "multiformats": "^13.0.1" + "libp2p": "^1.2.3", + "multiformats": "^13.1.0" }, "devDependencies": { "@multiformats/mafmt": "^12.1.6", "@multiformats/multiaddr": "^12.1.14", - "aegir": "^42.2.2", + "aegir": "^42.2.5", "it-all": "^3.0.4", "it-drain": "^3.0.5" }, diff --git a/packages/http/README.md b/packages/http/README.md index c28f1eeb..73734048 100644 --- a/packages/http/README.md +++ b/packages/http/README.md @@ -4,6 +4,8 @@

+# @helia/http + [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) [![codecov](https://img.shields.io/codecov/c/github/ipfs/helia.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia) @@ -13,6 +15,21 @@ # About + + Exports a `createHeliaHTTP` function that returns an object that implements a lightweight version of the Helia API that functions only over HTTP. By default, content and peer routing are requests are resolved using the [Delegated HTTP Routing API](https://specs.ipfs.tech/routing/http-routing-v1/) and blocks are fetched from [Trustless Gateways](https://specs.ipfs.tech/http-gateways/trustless-gateway/). diff --git a/packages/http/package.json b/packages/http/package.json index 9be64aeb..f4cc8736 100644 --- a/packages/http/package.json +++ b/packages/http/package.json @@ -58,13 +58,13 @@ "@helia/routers": "^1.0.0", "@helia/utils": "^0.0.1", "blockstore-core": "^4.4.0", - "datastore-core": "^9.2.7" + "datastore-core": "^9.2.9" }, "devDependencies": { - "@libp2p/interface": "^1.1.2", - "aegir": "^42.2.2", - "interface-datastore": "^8.2.10", - "multiformats": "^13.0.1", + "@libp2p/interface": "^1.1.3", + "aegir": "^42.2.5", + "interface-datastore": "^8.2.11", + "multiformats": "^13.1.0", "sinon": "^17.0.1", "sinon-ts": "^2.0.0" } diff --git a/packages/interface/README.md b/packages/interface/README.md index e50862db..b2146562 100644 --- a/packages/interface/README.md +++ b/packages/interface/README.md @@ -4,6 +4,8 @@

+# @helia/interface + [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) [![codecov](https://img.shields.io/codecov/c/github/ipfs/helia.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia) @@ -13,6 +15,21 @@ # About + + The API defined by a Helia node ## Example diff --git a/packages/interface/package.json b/packages/interface/package.json index 79d8162a..ba42f133 100644 --- a/packages/interface/package.json +++ b/packages/interface/package.json @@ -70,14 +70,14 @@ "build": "aegir build" }, "dependencies": { - "@libp2p/interface": "^1.1.2", - "interface-blockstore": "^5.2.9", - "interface-datastore": "^8.2.10", - "interface-store": "^5.1.7", - "multiformats": "^13.0.1", + "@libp2p/interface": "^1.1.3", + "interface-blockstore": "^5.2.10", + "interface-datastore": "^8.2.11", + "interface-store": "^5.1.8", + "multiformats": "^13.1.0", "progress-events": "^1.0.0" }, "devDependencies": { - "aegir": "^42.2.2" + "aegir": "^42.2.5" } } diff --git a/packages/interop/README.md b/packages/interop/README.md index 43880993..932c7c87 100644 --- a/packages/interop/README.md +++ b/packages/interop/README.md @@ -4,6 +4,8 @@

+# @helia/interop + [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) [![codecov](https://img.shields.io/codecov/c/github/ipfs/helia.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia) diff --git a/packages/interop/package.json b/packages/interop/package.json index e1c938a2..67f58b46 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -70,17 +70,16 @@ "@helia/routers": "^1.0.0", "@helia/strings": "^3.0.0", "@helia/unixfs": "^3.0.0", - "@helia/verified-fetch": "^0.0.0", - "@ipld/car": "^5.2.6", - "@ipld/dag-cbor": "^9.1.0", - "@libp2p/interface": "^1.1.2", - "@libp2p/kad-dht": "^12.0.5", - "@libp2p/keychain": "^4.0.7", - "@libp2p/peer-id": "^4.0.5", - "@libp2p/peer-id-factory": "^4.0.5", - "@libp2p/websockets": "^8.0.13", + "@ipld/car": "^5.3.0", + "@ipld/dag-cbor": "^9.2.0", + "@libp2p/interface": "^1.1.3", + "@libp2p/kad-dht": "^12.0.7", + "@libp2p/keychain": "^4.0.8", + "@libp2p/peer-id": "^4.0.6", + "@libp2p/peer-id-factory": "^4.0.6", + "@libp2p/websockets": "^8.0.15", "@multiformats/sha3": "^3.0.2", - "aegir": "^42.2.2", + "aegir": "^42.2.5", "helia": "^4.0.1", "ipfs-core-types": "^0.14.1", "ipfs-unixfs-importer": "^15.2.4", @@ -91,12 +90,11 @@ "it-map": "^3.0.5", "it-to-buffer": "^4.0.5", "kubo": "^0.26.0", - "kubo-rpc-client": "^3.0.2", - "libp2p": "^1.2.1", - "magic-bytes.js": "^1.8.0", - "multiformats": "^13.0.1", + "kubo-rpc-client": "^3.0.3", + "libp2p": "^1.2.3", + "multiformats": "^13.1.0", "p-defer": "^4.0.0", - "uint8arrays": "^5.0.1", + "uint8arrays": "^5.0.2", "wherearewe": "^2.0.1" }, "browser": { diff --git a/packages/interop/src/verified-fetch-json.spec.ts b/packages/interop/src/verified-fetch-json.spec.ts deleted file mode 100644 index 4536faab..00000000 --- a/packages/interop/src/verified-fetch-json.spec.ts +++ /dev/null @@ -1,47 +0,0 @@ -/* eslint-env mocha */ -import { createVerifiedFetch } from '@helia/verified-fetch' -import { expect } from 'aegir/chai' -import { CID } from 'multiformats/cid' -import { createKuboNode } from './fixtures/create-kubo.js' -import { loadFixtureDataCar } from './fixtures/load-fixture-data.js' -import type { Controller } from 'ipfsd-ctl' - -describe('@helia/verified-fetch - json', () => { - describe('unixfs - multiblock', () => { - let controller: Controller<'go'> - let verifiedFetch: Awaited> - - before(async () => { - controller = await createKuboNode() - await controller.start() - // As of 2024-01-18, https://cloudflare-ipfs.com/ipns/tokens.uniswap.org resolves to: - // root: QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr - // child1: QmNik5N4ryNwzzXYq5hCYKGcRjAf9QtigxtiJh9o8aXXbG // partial JSON - // child2: QmWNBJX6fZyNTLWNYBHxAHpBctCP43R2zeqV2G8uavqFZn // partial JSON - await loadFixtureDataCar(controller, 'QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr-tokens.uniswap.org-2024-01-18.car') - verifiedFetch = await createVerifiedFetch({ - gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`], - routers: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`] - }) - }) - - after(async () => { - await controller.stop() - await verifiedFetch.stop() - }) - - it('handles UnixFS-chunked JSON file', async () => { - const resp = await verifiedFetch(CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr')) - expect(resp).to.be.ok() - const jsonObj = await resp.json() - expect(jsonObj).to.be.ok() - expect(jsonObj).to.have.property('name').equal('Uniswap Labs Default') - expect(jsonObj).to.have.property('timestamp').equal('2023-12-13T18:25:25.830Z') - expect(jsonObj).to.have.property('version').to.deep.equal({ major: 11, minor: 11, patch: 0 }) - expect(jsonObj).to.have.property('tags') - expect(jsonObj).to.have.property('logoURI').equal('ipfs://QmNa8mQkrNKp1WEEeGjFezDmDeodkWRevGFN8JCV7b4Xir') - expect(jsonObj).to.have.property('keywords').to.deep.equal(['uniswap', 'default']) - expect(jsonObj.tokens).to.be.an('array').of.length(767) - }) - }) -}) diff --git a/packages/interop/src/verified-fetch-unixfs-dir.spec.ts b/packages/interop/src/verified-fetch-unixfs-dir.spec.ts deleted file mode 100644 index 2f2db818..00000000 --- a/packages/interop/src/verified-fetch-unixfs-dir.spec.ts +++ /dev/null @@ -1,94 +0,0 @@ -/* eslint-env mocha */ -import { createVerifiedFetch } from '@helia/verified-fetch' -import { expect } from 'aegir/chai' -import { filetypemime } from 'magic-bytes.js' -import { createKuboNode } from './fixtures/create-kubo.js' -import { loadFixtureDataCar } from './fixtures/load-fixture-data.js' -import type { VerifiedFetch } from '@helia/verified-fetch' -import type { Controller } from 'ipfsd-ctl' - -describe('@helia/verified-fetch - unixfs directory', () => { - let controller: Controller - let verifiedFetch: VerifiedFetch - - before(async () => { - controller = await createKuboNode() - await controller.start() - - verifiedFetch = await createVerifiedFetch({ - gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`], - routers: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`] - }) - }) - - after(async () => { - await controller.stop() - await verifiedFetch.stop() - }) - - describe('XKCD Barrel Part 1', () => { - before(async () => { - // This is the content of https://explore.ipld.io/#/explore/QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1%20-%20Barrel%20-%20Part%201 - await loadFixtureDataCar(controller, 'QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR-xkcd-Barrel-part-1.car') - }) - - it('fails to load when passed the root', async () => { - // The spec says we should generate HTML with directory listings, but we don't do that yet, so expect a failure - const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR') - expect(resp).to.be.ok() - expect(resp.status).to.equal(501) // TODO: we should do a directory listing instead - }) - - it('can return a string for unixfs pathed data', async () => { - const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR/1 - Barrel - Part 1 - alt.txt') - expect(resp).to.be.ok() - const text = await resp.text() - expect(text).to.equal('Don\'t we all.') - }) - - it('can return an image for unixfs pathed data', async () => { - const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR/1 - Barrel - Part 1.png') - expect(resp).to.be.ok() - const imgData = await resp.blob() - expect(imgData).to.be.ok() - expect(imgData.size).to.equal(24848) - }) - }) - - describe('content type parser', () => { - before(async () => { - await verifiedFetch.stop() - verifiedFetch = await createVerifiedFetch({ - gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`], - routers: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`] - }, { - contentTypeParser: (bytes) => { - return filetypemime(bytes)?.[0] - } - }) - }) - - it('can return an image content-type for unixfs pathed data', async () => { - const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR/1 - Barrel - Part 1.png') - // tediously this is actually a jpeg file with a .png extension - expect(resp.headers.get('content-type')).to.equal('image/jpeg') - }) - }) - - describe('HAMT-sharded directory', () => { - before(async () => { - // from https://github.com/ipfs/gateway-conformance/blob/193833b91f2e9b17daf45c84afaeeae61d9d7c7e/fixtures/trustless_gateway_car/single-layer-hamt-with-multi-block-files.car - await loadFixtureDataCar(controller, 'bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i-single-layer-hamt-with-multi-block-files.car') - }) - - it('loads path /ipfs/bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i/685.txt', async () => { - const resp = await verifiedFetch('ipfs://bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i/685.txt') - expect(resp).to.be.ok() - const text = await resp.text() - // npx kubo@0.25.0 cat '/ipfs/bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i/685.txt' - expect(text).to.equal(`Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc non imperdiet nunc. Proin ac quam ut nibh eleifend aliquet. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Sed ligula dolor, imperdiet sagittis arcu et, semper tincidunt urna. Donec et tempor augue, quis sollicitudin metus. Curabitur semper ullamcorper aliquet. Mauris hendrerit sodales lectus eget fermentum. Proin sollicitudin vestibulum commodo. Vivamus nec lectus eu augue aliquet dignissim nec condimentum justo. In hac habitasse platea dictumst. Mauris vel sem neque. - -Vivamus finibus, enim at lacinia semper, arcu erat gravida lacus, sit amet gravida magna orci sit amet est. Sed non leo lacus. Nullam viverra ipsum a tincidunt dapibus. Nulla pulvinar ligula sit amet ante ultrices tempus. Proin purus urna, semper sed lobortis quis, gravida vitae ipsum. Aliquam mi urna, pulvinar eu bibendum quis, convallis ac dolor. In gravida justo sed risus ullamcorper, vitae luctus massa hendrerit. Pellentesque habitant amet.`) - }) - }) -}) diff --git a/packages/interop/src/verified-fetch-websites.spec.ts b/packages/interop/src/verified-fetch-websites.spec.ts deleted file mode 100644 index 287e6638..00000000 --- a/packages/interop/src/verified-fetch-websites.spec.ts +++ /dev/null @@ -1,85 +0,0 @@ -/* eslint-env mocha */ -import { createVerifiedFetch } from '@helia/verified-fetch' -import { expect } from 'aegir/chai' -import { createKuboNode } from './fixtures/create-kubo.js' -import { loadFixtureDataCar } from './fixtures/load-fixture-data.js' -import type { Controller } from 'ipfsd-ctl' - -describe('@helia/verified-fetch - websites', () => { - describe('helia-identify.on.fleek.co', () => { - let controller: Controller<'go'> - let verifiedFetch: Awaited> - - before(async () => { - controller = await createKuboNode() - await controller.start() - // 2024-01-22 CID for _dnslink.helia-identify.on.fleek.co - await loadFixtureDataCar(controller, 'QmbxpRxwKXxnJQjnPqm1kzDJSJ8YgkLxH23mcZURwPHjGv-helia-identify-website.car') - verifiedFetch = await createVerifiedFetch({ - gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`], - routers: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`] - }) - }) - - after(async () => { - await controller.stop() - await verifiedFetch.stop() - }) - - it('loads index.html when passed helia-identify.on.fleek.co root CID', async () => { - const resp = await verifiedFetch('ipfs://QmbxpRxwKXxnJQjnPqm1kzDJSJ8YgkLxH23mcZURwPHjGv') - expect(resp).to.be.ok() - const html = await resp.text() - expect(html).to.be.ok() - expect(html).to.include('Run Identify on a remote node with Helia') - }) - - it('loads helia-identify.on.fleek.co index.html directly ', async () => { - const resp = await verifiedFetch('ipfs://QmbxpRxwKXxnJQjnPqm1kzDJSJ8YgkLxH23mcZURwPHjGv/index.html') - expect(resp).to.be.ok() - const html = await resp.text() - expect(html).to.be.ok() - expect(html).to.include('Run Identify on a remote node with Helia') - }) - }) - - /** - * - * Created on 2024-01-23. /ipns/blog.libp2p.io/index.html resolved to QmVZNGy6SPvUbvQCXXaGDdp8kvfJm9MMozjU12dyzH6hKf - * - * ```shell - * mkdir fake-blog.libp2p.io - * npx kubo@0.25.0 cat '/ipfs/QmVZNGy6SPvUbvQCXXaGDdp8kvfJm9MMozjU12dyzH6hKf' > fake-blog.libp2p.io/index.html - * npx kubo@0.25.0 add -r fake-blog.libp2p.io - * npx kubo@0.25.0 dag export QmeiDMLtPUS3RT2xAcUwsNyZz169wPke2q7im9vZpVLSYw > QmeiDMLtPUS3RT2xAcUwsNyZz169wPke2q7im9vZpVLSYw-fake-blog.libp2p.io.car - * ``` - */ - describe('fake blog.libp2p.io', () => { - let controller: Controller<'go'> - let verifiedFetch: Awaited> - - before(async () => { - controller = await createKuboNode() - await controller.start() - await loadFixtureDataCar(controller, 'QmeiDMLtPUS3RT2xAcUwsNyZz169wPke2q7im9vZpVLSYw-fake-blog.libp2p.io.car') - verifiedFetch = await createVerifiedFetch({ - gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`], - routers: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`] - }) - }) - - after(async () => { - await controller.stop() - await verifiedFetch.stop() - }) - - it('loads index.html when passed fake-blog.libp2p.io root CID', async () => { - const resp = await verifiedFetch('ipfs://QmeiDMLtPUS3RT2xAcUwsNyZz169wPke2q7im9vZpVLSYw') - expect(resp).to.be.ok() - const html = await resp.text() - expect(html).to.be.ok() - expect(html).to.include('Home | libp2p Blog & News') - expect(html).to.include('') - }) - }) -}) diff --git a/packages/interop/tsconfig.json b/packages/interop/tsconfig.json index 79ff2ae8..bc845a1b 100644 --- a/packages/interop/tsconfig.json +++ b/packages/interop/tsconfig.json @@ -46,9 +46,6 @@ }, { "path": "../unixfs" - }, - { - "path": "../verified-fetch" } ] } diff --git a/packages/ipns/README.md b/packages/ipns/README.md index 4901d49d..0e2e3f7c 100644 --- a/packages/ipns/README.md +++ b/packages/ipns/README.md @@ -4,7 +4,7 @@

-# @helia/ipns +# @helia/ipns [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) @@ -15,6 +15,21 @@ # About + + IPNS operations using a Helia node ## Example - Getting started diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 4151a1ca..03dce44a 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -165,25 +165,25 @@ }, "dependencies": { "@helia/interface": "^4.0.0", - "@libp2p/interface": "^1.1.2", - "@libp2p/kad-dht": "^12.0.5", - "@libp2p/logger": "^4.0.5", - "@libp2p/peer-id": "^4.0.5", + "@libp2p/interface": "^1.1.3", + "@libp2p/kad-dht": "^12.0.7", + "@libp2p/logger": "^4.0.6", + "@libp2p/peer-id": "^4.0.6", "dns-over-http-resolver": "^3.0.2", "dns-packet": "^5.6.1", "hashlru": "^2.3.0", - "interface-datastore": "^8.2.10", + "interface-datastore": "^8.2.11", "ipns": "^9.0.0", - "multiformats": "^13.0.1", + "multiformats": "^13.1.0", "p-queue": "^8.0.1", "progress-events": "^1.0.0", - "uint8arrays": "^5.0.1" + "uint8arrays": "^5.0.2" }, "devDependencies": { - "@libp2p/peer-id-factory": "^4.0.5", + "@libp2p/peer-id-factory": "^4.0.6", "@types/dns-packet": "^5.6.5", - "aegir": "^42.2.2", - "datastore-core": "^9.2.7", + "aegir": "^42.2.5", + "datastore-core": "^9.2.9", "sinon": "^17.0.1", "sinon-ts": "^2.0.0" }, diff --git a/packages/json/README.md b/packages/json/README.md index fa841821..21debde4 100644 --- a/packages/json/README.md +++ b/packages/json/README.md @@ -15,6 +15,21 @@ # About + + `@helia/json` makes working with JSON in Helia simple & straightforward. See the JSON interface for all available operations. diff --git a/packages/json/package.json b/packages/json/package.json index 153524fd..7e3d2b84 100644 --- a/packages/json/package.json +++ b/packages/json/package.json @@ -141,12 +141,12 @@ "dependencies": { "@helia/interface": "^4.0.0", "@libp2p/interfaces": "^3.3.2", - "interface-blockstore": "^5.2.9", - "multiformats": "^13.0.1", + "interface-blockstore": "^5.2.10", + "multiformats": "^13.1.0", "progress-events": "^1.0.0" }, "devDependencies": { - "aegir": "^42.2.2", + "aegir": "^42.2.5", "blockstore-core": "^4.4.0" } } diff --git a/packages/mfs/README.md b/packages/mfs/README.md index 4c69e132..6b3d04e6 100644 --- a/packages/mfs/README.md +++ b/packages/mfs/README.md @@ -15,6 +15,21 @@ # About + + `@helia/mfs` is an implementation of a Mutable File System powered by Helia. See the [API docs](https://ipfs.github.io/helia/modules/_helia_mfs.html) for all available operations. diff --git a/packages/mfs/package.json b/packages/mfs/package.json index a132778b..b1d405c6 100644 --- a/packages/mfs/package.json +++ b/packages/mfs/package.json @@ -141,25 +141,25 @@ "dependencies": { "@helia/unixfs": "^3.0.0", "@libp2p/interfaces": "^3.3.2", - "@libp2p/logger": "^4.0.5", - "interface-blockstore": "^5.2.9", - "interface-datastore": "^8.2.10", + "@libp2p/logger": "^4.0.6", + "interface-blockstore": "^5.2.10", + "interface-datastore": "^8.2.11", "ipfs-unixfs": "^11.1.3", "ipfs-unixfs-exporter": "^13.5.0", "ipfs-unixfs-importer": "^15.2.4", - "multiformats": "^13.0.1" + "multiformats": "^13.1.0" }, "devDependencies": { - "@ipld/dag-pb": "^4.0.8", - "aegir": "^42.2.2", + "@ipld/dag-pb": "^4.1.0", + "aegir": "^42.2.5", "blockstore-core": "^4.4.0", - "datastore-core": "^9.2.7", + "datastore-core": "^9.2.9", "delay": "^6.0.0", "it-all": "^3.0.4", "it-drain": "^3.0.5", "it-first": "^3.0.4", "it-last": "^3.0.4", "it-to-buffer": "^4.0.5", - "uint8arrays": "^5.0.1" + "uint8arrays": "^5.0.2" } } diff --git a/packages/routers/README.md b/packages/routers/README.md index 22378715..bfa9aa6e 100644 --- a/packages/routers/README.md +++ b/packages/routers/README.md @@ -4,6 +4,8 @@

+# @helia/routers + [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) [![codecov](https://img.shields.io/codecov/c/github/ipfs/helia.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia) @@ -13,6 +15,21 @@ # About + + Abstraction layer over different content and peer routing mechanisms. # Install diff --git a/packages/routers/package.json b/packages/routers/package.json index 309c2618..8df13ee2 100644 --- a/packages/routers/package.json +++ b/packages/routers/package.json @@ -55,17 +55,17 @@ "dependencies": { "@helia/delegated-routing-v1-http-api-client": "^3.0.0", "@helia/interface": "^4.0.0", - "@libp2p/interface": "^1.1.2", + "@libp2p/interface": "^1.1.3", "ipns": "^9.0.0", "it-first": "^3.0.4", "it-map": "^3.0.5", - "multiformats": "^13.0.1", - "uint8arrays": "^5.0.1" + "multiformats": "^13.1.0", + "uint8arrays": "^5.0.2" }, "devDependencies": { - "@libp2p/peer-id": "^4.0.5", - "@libp2p/peer-id-factory": "^4.0.5", - "aegir": "^42.2.2", + "@libp2p/peer-id": "^4.0.6", + "@libp2p/peer-id-factory": "^4.0.6", + "aegir": "^42.2.5", "it-drain": "^3.0.5", "sinon-ts": "^2.0.0" }, diff --git a/packages/strings/README.md b/packages/strings/README.md index 515ff4a7..01a7b437 100644 --- a/packages/strings/README.md +++ b/packages/strings/README.md @@ -15,6 +15,21 @@ # About + + `@helia/strings` makes working with strings Helia simple & straightforward. See the [API docs](https://ipfs.github.io/helia/modules/_helia_strings.html) for all available operations. diff --git a/packages/strings/package.json b/packages/strings/package.json index 7a4b326c..d538949e 100644 --- a/packages/strings/package.json +++ b/packages/strings/package.json @@ -143,13 +143,13 @@ "dependencies": { "@helia/interface": "^4.0.0", "@libp2p/interfaces": "^3.3.2", - "interface-blockstore": "^5.2.9", - "multiformats": "^13.0.1", + "interface-blockstore": "^5.2.10", + "multiformats": "^13.1.0", "progress-events": "^1.0.0", - "uint8arrays": "^5.0.1" + "uint8arrays": "^5.0.2" }, "devDependencies": { - "aegir": "^42.2.2", + "aegir": "^42.2.5", "blockstore-core": "^4.4.0" } } diff --git a/packages/unixfs/README.md b/packages/unixfs/README.md index 7a446ad0..c3b6ca03 100644 --- a/packages/unixfs/README.md +++ b/packages/unixfs/README.md @@ -15,6 +15,21 @@ # About + + `@helia/unixfs` is an implementation of a filesystem compatible with Helia. See the [API docs](https://ipfs.github.io/helia/modules/_helia_unixfs.html) for all available operations. diff --git a/packages/unixfs/package.json b/packages/unixfs/package.json index c4ba3e4d..a04d27ad 100644 --- a/packages/unixfs/package.json +++ b/packages/unixfs/package.json @@ -160,12 +160,12 @@ }, "dependencies": { "@helia/interface": "^4.0.0", - "@ipld/dag-pb": "^4.0.8", - "@libp2p/interface": "^1.1.2", - "@libp2p/logger": "^4.0.5", + "@ipld/dag-pb": "^4.1.0", + "@libp2p/interface": "^1.1.3", + "@libp2p/logger": "^4.0.6", "@multiformats/murmur3": "^2.1.8", - "hamt-sharding": "^3.0.2", - "interface-blockstore": "^5.2.9", + "hamt-sharding": "^3.0.6", + "interface-blockstore": "^5.2.10", "ipfs-unixfs": "^11.1.3", "ipfs-unixfs-exporter": "^13.5.0", "ipfs-unixfs-importer": "^15.2.4", @@ -174,13 +174,13 @@ "it-last": "^3.0.4", "it-pipe": "^3.0.1", "merge-options": "^3.0.4", - "multiformats": "^13.0.1", + "multiformats": "^13.1.0", "progress-events": "^1.0.0", "sparse-array": "^1.3.2", - "uint8arrays": "^5.0.1" + "uint8arrays": "^5.0.2" }, "devDependencies": { - "aegir": "^42.2.2", + "aegir": "^42.2.5", "blockstore-core": "^4.4.0", "delay": "^6.0.0", "iso-url": "^1.2.1", diff --git a/packages/utils/README.md b/packages/utils/README.md index d6da7a93..4ae0cd82 100644 --- a/packages/utils/README.md +++ b/packages/utils/README.md @@ -4,6 +4,8 @@

+# @helia/utils + [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) [![codecov](https://img.shields.io/codecov/c/github/ipfs/helia.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia) @@ -13,6 +15,21 @@ # About + + Exports a `Helia` class that implements the HeliaInterface API. In general you should use the `helia` or `@helia/http` modules instead which diff --git a/packages/utils/package.json b/packages/utils/package.json index 110bf2f7..0da2d349 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -54,32 +54,32 @@ }, "dependencies": { "@helia/interface": "^4.0.0", - "@ipld/dag-cbor": "^9.1.0", - "@ipld/dag-json": "^10.1.7", - "@ipld/dag-pb": "^4.0.8", - "@libp2p/interface": "^1.1.2", - "@libp2p/logger": "^4.0.5", - "@libp2p/peer-collections": "^5.1.5", - "@libp2p/utils": "^5.2.3", + "@ipld/dag-cbor": "^9.2.0", + "@ipld/dag-json": "^10.2.0", + "@ipld/dag-pb": "^4.1.0", + "@libp2p/interface": "^1.1.3", + "@libp2p/logger": "^4.0.6", + "@libp2p/peer-collections": "^5.1.6", + "@libp2p/utils": "^5.2.5", "any-signal": "^4.1.1", "blockstore-core": "^4.4.0", - "cborg": "^4.0.8", - "interface-blockstore": "^5.2.9", - "interface-datastore": "^8.2.10", - "interface-store": "^5.1.7", + "cborg": "^4.0.9", + "interface-blockstore": "^5.2.10", + "interface-datastore": "^8.2.11", + "interface-store": "^5.1.8", "it-drain": "^3.0.5", "it-filter": "^3.0.4", "it-foreach": "^2.0.6", "it-merge": "^3.0.3", "mortice": "^3.0.4", - "multiformats": "^13.0.1", + "multiformats": "^13.1.0", "progress-events": "^1.0.0", - "uint8arrays": "^5.0.1" + "uint8arrays": "^5.0.2" }, "devDependencies": { "@types/sinon": "^17.0.3", - "aegir": "^42.2.2", - "datastore-core": "^9.2.7", + "aegir": "^42.2.5", + "datastore-core": "^9.2.9", "delay": "^6.0.0", "it-all": "^3.0.4", "sinon": "^17.0.1", diff --git a/packages/verified-fetch/.aegir.js b/packages/verified-fetch/.aegir.js deleted file mode 100644 index 89b56338..00000000 --- a/packages/verified-fetch/.aegir.js +++ /dev/null @@ -1,8 +0,0 @@ -/** @type {import('aegir').PartialOptions} */ -const options = { - build: { - bundlesizeMax: '132KB' - } -} - -export default options diff --git a/packages/verified-fetch/LICENSE b/packages/verified-fetch/LICENSE deleted file mode 100644 index 20ce483c..00000000 --- a/packages/verified-fetch/LICENSE +++ /dev/null @@ -1,4 +0,0 @@ -This project is dual licensed under MIT and Apache-2.0. - -MIT: https://www.opensource.org/licenses/mit -Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/verified-fetch/LICENSE-APACHE b/packages/verified-fetch/LICENSE-APACHE deleted file mode 100644 index 14478a3b..00000000 --- a/packages/verified-fetch/LICENSE-APACHE +++ /dev/null @@ -1,5 +0,0 @@ -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/verified-fetch/LICENSE-MIT b/packages/verified-fetch/LICENSE-MIT deleted file mode 100644 index 72dc60d8..00000000 --- a/packages/verified-fetch/LICENSE-MIT +++ /dev/null @@ -1,19 +0,0 @@ -The MIT License (MIT) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/packages/verified-fetch/README.md b/packages/verified-fetch/README.md deleted file mode 100644 index 966f04de..00000000 --- a/packages/verified-fetch/README.md +++ /dev/null @@ -1,531 +0,0 @@ -

- - Helia logo - -

- -[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) -[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) -[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia) -[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia/main.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia/actions/workflows/main.yml?query=branch%3Amain) - -> A fetch-like API for obtaining verified & trustless IPFS content on the web. - -# About - - - -`@helia/verified-fetch` provides a [fetch](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API)-like API for retrieving content from the [IPFS](https://ipfs.tech/) network. - -All content is retrieved in a [trustless manner](https://www.techopedia.com/definition/trustless), and the integrity of all bytes are verified by comparing hashes of the data. By default, CIDs are retrieved over HTTP from [trustless gateways](https://specs.ipfs.tech/http-gateways/trustless-gateway/). - -This is a marked improvement over `fetch` which offers no such protections and is vulnerable to all sorts of attacks like [Content Spoofing](https://owasp.org/www-community/attacks/Content_Spoofing), [DNS Hijacking](https://en.wikipedia.org/wiki/DNS_hijacking), etc. - -A `verifiedFetch` function is exported to get up and running quickly, and a `createVerifiedFetch` function is also available that allows customizing the underlying [Helia](https://helia.io/) node for complete control over how content is retrieved. - -Browser-cache-friendly [Response](https://developer.mozilla.org/en-US/docs/Web/API/Response) objects are returned which should be instantly familiar to web developers. - -You may use any supported resource argument to fetch content: - -- [CID](https://multiformats.github.io/js-multiformats/classes/cid.CID.html) instance -- IPFS URL -- IPNS URL - -## Example - Getting started - -```typescript -import { verifiedFetch } from '@helia/verified-fetch' - -const resp = await verifiedFetch('ipfs://bafy...') - -const json = await resp.json() -``` - -## Example - Using a CID instance to fetch JSON - -```typescript -import { verifiedFetch } from '@helia/verified-fetch' -import { CID } from 'multiformats/cid' - -const cid = CID.parse('bafyFoo') // some json file -const response = await verifiedFetch(cid) -const json = await response.json() -``` - -## Example - Using IPFS protocol to fetch an image - -```typescript -import { verifiedFetch } from '@helia/verified-fetch' - -const response = await verifiedFetch('ipfs://bafyFoo') // CID for some image file -const blob = await response.blob() -const image = document.createElement('img') -image.src = URL.createObjectURL(blob) -document.body.appendChild(image) -``` - -## Example - Using IPNS protocol to stream a big file - -```typescript -import { verifiedFetch } from '@helia/verified-fetch' - -const response = await verifiedFetch('ipns://mydomain.com/path/to/very-long-file.log') -const bigFileStreamReader = await response.body.getReader() -``` - -## Configuration - -### Custom HTTP gateways and routers - -Out of the box `@helia/verified-fetch` uses a default set of [trustless gateways](https://specs.ipfs.tech/http-gateways/trustless-gateway/) for fetching blocks and [HTTP delegated routers](https://specs.ipfs.tech/routing/http-routing-v1/) for performing routing tasks - looking up peers, resolving/publishing [IPNS](https://docs.ipfs.tech/concepts/ipns/) names, etc. - -It's possible to override these by passing `gateways` and `routers` keys to the `createVerifiedFetch` function: - -## Example - Configuring gateways and routers - -```typescript -import { createVerifiedFetch } from '@helia/verified-fetch' - -const fetch = await createVerifiedFetch({ - gateways: ['https://trustless-gateway.link'], - routers: ['http://delegated-ipfs.dev'] -}) - -const resp = await fetch('ipfs://bafy...') - -const json = await resp.json() -``` - -### Usage with customized Helia - -For full control of how `@helia/verified-fetch` fetches content from the distributed web you can pass a preconfigured Helia node to `createVerifiedFetch`. - -The [helia](https://www.npmjs.com/package/helia) module is configured with a libp2p node that is suited for decentralized applications, alternatively [@helia/http](https://www.npmjs.com/package/@helia/http) is available which uses HTTP gateways for all network operations. - -You can see variations of Helia and js-libp2p configuration options at . - -```typescript -import { trustlessGateway } from '@helia/block-brokers' -import { createHeliaHTTP } from '@helia/http' -import { delegatedHTTPRouting } from '@helia/routers' -import { createVerifiedFetch } from '@helia/verified-fetch' - -const fetch = await createVerifiedFetch( - await createHeliaHTTP({ - blockBrokers: [ - trustlessGateway({ - gateways: ['https://mygateway.example.net', 'https://trustless-gateway.link'] - }) - ], - routers: ['http://delegated-ipfs.dev'].map((routerUrl) => delegatedHTTPRouting(routerUrl)) - }) -) - -const resp = await fetch('ipfs://bafy...') - -const json = await resp.json() -``` - -### Custom content-type parsing - -By default, if the response can be parsed as JSON, `@helia/verified-fetch` sets the `Content-Type` header as `application/json`, otherwise it sets it as `application/octet-stream` - this is because the `.json()`, `.text()`, `.blob()`, and `.arrayBuffer()` methods will usually work as expected without a detailed content type. - -If you require an accurate content-type you can provide a `contentTypeParser` function as an option to `createVerifiedFetch` to handle parsing the content type. - -The function you provide will be called with the first chunk of bytes from the file and should return a string or a promise of a string. - -## Example - Customizing content-type parsing - -```typescript -import { createVerifiedFetch } from '@helia/verified-fetch' -import { fileTypeFromBuffer } from '@sgtpooki/file-type' - -const fetch = await createVerifiedFetch({ - gateways: ['https://trustless-gateway.link'], - routers: ['http://delegated-ipfs.dev'] -}, { - contentTypeParser: async (bytes) => { - // call to some magic-byte recognition library like magic-bytes, file-type, or your own custom byte recognition - const result = await fileTypeFromBuffer(bytes) - return result?.mime - } -}) -``` - -### IPLD codec handling - -IPFS supports several data formats (typically referred to as codecs) which are included in the CID. `@helia/verified-fetch` attempts to abstract away some of the details for easier consumption. - -#### DAG-PB - -[DAG-PB](https://ipld.io/docs/codecs/known/dag-pb/) is the codec we are most likely to encounter, it is what [UnixFS](https://github.com/ipfs/specs/blob/main/UNIXFS.md) uses under the hood. - -##### Using the DAG-PB codec as a Blob - -```typescript -import { verifiedFetch } from '@helia/verified-fetch' - -const res = await verifiedFetch('ipfs://Qmfoo') -const blob = await res.blob() - -console.info(blob) // Blob { size: x, type: 'application/octet-stream' } -``` - -##### Using the DAG-PB codec as an ArrayBuffer - -```typescript -import { verifiedFetch } from '@helia/verified-fetch' - -const res = await verifiedFetch('ipfs://Qmfoo') -const buf = await res.arrayBuffer() - -console.info(buf) // ArrayBuffer { [Uint8Contents]: < ... >, byteLength: x } -``` - -##### Using the DAG-PB codec as a stream - -```typescript -import { verifiedFetch } from '@helia/verified-fetch' - -const res = await verifiedFetch('ipfs://Qmfoo') -const reader = res.body?.getReader() - -while (true) { - const next = await reader.read() - - if (next?.done === true) { - break - } - - if (next?.value != null) { - console.info(next.value) // Uint8Array(x) [ ... ] - } -} -``` - -##### Content-Type - -When fetching `DAG-PB` data, the content type will be set to `application/octet-stream` unless a custom content-type parser is configured. - -#### JSON - -The JSON codec is a very simple codec, a block parseable with this codec is a JSON string encoded into a `Uint8Array`. - -##### Using the JSON codec - -```typescript -import * as json from 'multiformats/codecs/json' - -const block = new TextEncoder().encode('{ "hello": "world" }') -const obj = json.decode(block) - -console.info(obj) // { hello: 'world' } -``` - -##### Content-Type - -When the `JSON` codec is encountered, the `Content-Type` header of the response will be set to `application/json`. - -### DAG-JSON - -[DAG-JSON](https://ipld.io/docs/codecs/known/dag-json/) expands on the `JSON` codec, adding the ability to contain [CID](https://docs.ipfs.tech/concepts/content-addressing/)s which act as links to other blocks, and byte arrays. - -`CID`s and byte arrays are represented using special object structures with a single `"/"` property. - -Using `DAG-JSON` has two important caveats: - -1. Your `JSON` structure cannot contain an object with only a `"/"` property, as it will be interpreted as a special type. -2. Since `JSON` has no technical limit on number sizes, `DAG-JSON` also allows numbers larger than `Number.MAX_SAFE_INTEGER`. JavaScript requires use of `BigInt`s to represent numbers larger than this, and `JSON.parse` does not support them, so precision will be lost. - -Otherwise this codec follows the same rules as the `JSON` codec. - -##### Using the DAG-JSON codec - -```typescript -import * as dagJson from '@ipld/dag-json' - -const block = new TextEncoder().encode(`{ - "hello": "world", - "cid": { - "/": "baeaaac3imvwgy3zao5xxe3de" - }, - "buf": { - "/": { - "bytes": "AAECAwQ" - } - } -}`) - -const obj = dagJson.decode(block) - -console.info(obj) -// { -// hello: 'world', -// cid: CID(baeaaac3imvwgy3zao5xxe3de), -// buf: Uint8Array(5) [ 0, 1, 2, 3, 4 ] -// } -``` - -##### Content-Type - -When the `DAG-JSON` codec is encountered in the requested CID, the `Content-Type` header of the response will be set to `application/json`. - -`DAG-JSON` data can be parsed from the response by using the `.json()` function, which will return `CID`s/byte arrays as plain `{ "/": ... }` objects: - -```typescript -import { verifiedFetch } from '@helia/verified-fetch' -import * as dagJson from '@ipld/dag-json' - -const res = await verifiedFetch('ipfs://bafyDAGJSON') - -// either: -const obj = await res.json() -console.info(obj.cid) // { "/": "baeaaac3imvwgy3zao5xxe3de" } -console.info(obj.buf) // { "/": { "bytes": "AAECAwQ" } } -``` - -Alternatively, it can be decoded using the `@ipld/dag-json` module and the `.arrayBuffer()` method, in which case you will get `CID` objects and `Uint8Array`s: - -```typescript -import { verifiedFetch } from '@helia/verified-fetch' -import * as dagJson from '@ipld/dag-json' - -const res = await verifiedFetch('ipfs://bafyDAGJSON') - -// or: -const obj = dagJson.decode(await res.arrayBuffer()) -console.info(obj.cid) // CID(baeaaac3imvwgy3zao5xxe3de) -console.info(obj.buf) // Uint8Array(5) [ 0, 1, 2, 3, 4 ] -``` - -#### DAG-CBOR - -[DAG-CBOR](https://ipld.io/docs/codecs/known/dag-cbor/) uses the [Concise Binary Object Representation](https://cbor.io/) format for serialization instead of JSON. - -This supports more datatypes in a safer way than JSON and is smaller on the wire to boot so is usually preferable to JSON or DAG-JSON. - -##### Content-Type - -Not all data types supported by `DAG-CBOR` can be successfully turned into JSON and back into the same binary form. - -When a decoded block can be round-tripped to JSON, the `Content-Type` will be set to `application/json`. In this case the `.json()` method on the `Response` object can be used to obtain an object representation of the response. - -When it cannot, the `Content-Type` will be `application/octet-stream` - in this case the `@ipld/dag-json` module must be used to deserialize the return value from `.arrayBuffer()`. - -##### Detecting JSON-safe DAG-CBOR - -If the `Content-Type` header of the response is `application/json`, the `.json()` method may be used to access the response body in object form, otherwise the `.arrayBuffer()` method must be used to decode the raw bytes using the `@ipld/dag-cbor` module. - -```typescript -import { verifiedFetch } from '@helia/verified-fetch' -import * as dagCbor from '@ipld/dag-cbor' - -const res = await verifiedFetch('ipfs://bafyDagCborCID') -let obj - -if (res.headers.get('Content-Type') === 'application/json') { - // DAG-CBOR data can be safely decoded as JSON - obj = await res.json() -} else { - // response contains non-JSON friendly data types - obj = dagCbor.decode(await res.arrayBuffer()) -} - -console.info(obj) // ... -``` - -## The `Accept` header - -The `Accept` header can be passed to override certain response processing, or to ensure that the final `Content-Type` of the response is the one that is expected. - -If the final `Content-Type` does not match the `Accept` header, or if the content cannot be represented in the format dictated by the `Accept` header, or you have configured a custom content type parser, and that parser returns a value that isn't in the accept header, a [406: Not Acceptable](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/406) response will be returned: - -```typescript -import { verifiedFetch } from '@helia/verified-fetch' - -const res = await verifiedFetch('ipfs://bafyJPEGImageCID', { - headers: { - accept: 'image/png' - } -}) - -console.info(res.status) // 406 - the image was a JPEG but we specified PNG as the accept header -``` - -It can also be used to skip processing the data from some formats such as `DAG-CBOR` if you wish to handle decoding it yourself: - -```typescript -import { verifiedFetch } from '@helia/verified-fetch' - -const res = await verifiedFetch('ipfs://bafyDAGCBORCID', { - headers: { - accept: 'application/octet-stream' - } -}) - -console.info(res.headers.get('accept')) // application/octet-stream -const buf = await res.arrayBuffer() // raw bytes, not processed as JSON -``` - -## Comparison to fetch - -This module attempts to act as similarly to the `fetch()` API as possible. - -[The `fetch()` API](https://developer.mozilla.org/en-US/docs/Web/API/fetch) takes two parameters: - -1. A [resource](https://developer.mozilla.org/en-US/docs/Web/API/fetch#resource) -2. An [options object](https://developer.mozilla.org/en-US/docs/Web/API/fetch#options) - -### Resource argument - -This library supports the following methods of fetching web3 content from IPFS: - -1. IPFS protocol: `ipfs://` & `ipfs://` -2. IPNS protocol: `ipns://` & `ipns://` & `ipns://` -3. CID instances: An actual CID instance `CID.parse('bafy...')` - -As well as support for pathing & params for items 1 & 2 above according to [IPFS - Path Gateway Specification](https://specs.ipfs.tech/http-gateways/path-gateway) & [IPFS - Trustless Gateway Specification](https://specs.ipfs.tech/http-gateways/trustless-gateway/). Further refinement of those specifications specifically for web-based scenarios can be found in the [Web Pathing Specification IPIP](https://github.com/ipfs/specs/pull/453). - -If you pass a CID instance, it assumes you want the content for that specific CID only, and does not support pathing or params for that CID. - -### Options argument - -This library does not plan to support the exact Fetch API options object, as some of the arguments don't make sense. Instead, it will only support options necessary to meet [IPFS specs](https://specs.ipfs.tech/) related to specifying the resultant shape of desired content. - -Some of those header specifications are: - -1. -2. -3. - -Where possible, options and Helia internals will be automatically configured to the appropriate codec & content type based on the `verified-fetch` configuration and `options` argument passed. - -Known Fetch API options that will be supported: - -1. `signal` - An AbortSignal that a user can use to abort the request. -2. `redirect` - A string that specifies the redirect type. One of `follow`, `error`, or `manual`. Defaults to `follow`. Best effort to adhere to the [Fetch API redirect](https://developer.mozilla.org/en-US/docs/Web/API/fetch#redirect) parameter. -3. `headers` - An object of headers to be sent with the request. Best effort to adhere to the [Fetch API headers](https://developer.mozilla.org/en-US/docs/Web/API/fetch#headers) parameter. - - `accept` - A string that specifies the accept header. Relevant values: - - [`vnd.ipld.raw`](https://www.iana.org/assignments/media-types/application/vnd.ipld.raw). (default) - - [`vnd.ipld.car`](https://www.iana.org/assignments/media-types/application/vnd.ipld.car) - - [`vnd.ipfs.ipns-record`](https://www.iana.org/assignments/media-types/application/vnd.ipfs.ipns-record) -4. `method` - A string that specifies the HTTP method to use for the request. Defaults to `GET`. Best effort to adhere to the [Fetch API method](https://developer.mozilla.org/en-US/docs/Web/API/fetch#method) parameter. -5. `body` - An object that specifies the body of the request. Best effort to adhere to the [Fetch API body](https://developer.mozilla.org/en-US/docs/Web/API/fetch#body) parameter. -6. `cache` - Will basically act as `force-cache` for the request. Best effort to adhere to the [Fetch API cache](https://developer.mozilla.org/en-US/docs/Web/API/fetch#cache) parameter. - -Non-Fetch API options that will be supported: - -1. `onProgress` - Similar to Helia `onProgress` options, this will be a function that will be called with a progress event. Supported progress events are: - - `helia:verified-fetch:error` - An error occurred during the request. - - `helia:verified-fetch:request:start` - The request has been sent - - `helia:verified-fetch:request:complete` - The request has been sent - - `helia:verified-fetch:request:error` - An error occurred during the request. - - `helia:verified-fetch:request:abort` - The request was aborted prior to completion. - - `helia:verified-fetch:response:start` - The initial HTTP Response headers have been set, and response stream is started. - - `helia:verified-fetch:response:complete` - The response stream has completed. - - `helia:verified-fetch:response:error` - An error occurred while building the response. - -Some in-flight specs (IPIPs) that will affect the options object this library supports in the future can be seen at , a subset are: - -1. [IPIP-0412: Signaling Block Order in CARs on HTTP Gateways](https://specs.ipfs.tech/ipips/ipip-0412/) -2. [IPIP-0402: Partial CAR Support on Trustless Gateways](https://specs.ipfs.tech/ipips/ipip-0402/) -3. [IPIP-0386: Subdomain Gateway Interop with \_redirects](https://specs.ipfs.tech/ipips/ipip-0386/) -4. [IPIP-0328: JSON and CBOR Response Formats on HTTP Gateways](https://specs.ipfs.tech/ipips/ipip-0328/) -5. [IPIP-0288: TAR Response Format on HTTP Gateways](https://specs.ipfs.tech/ipips/ipip-0288/) - -### Response types - -This library's purpose is to return reasonably representable content from IPFS. In other words, fetching content is intended for leaf-node content -- such as images/videos/audio & other assets, or other IPLD content (with link) -- that can be represented by . The content type you receive back will depend upon the CID you request as well as the `Accept` header value you provide. - -All content we retrieve from the IPFS network is obtained via an AsyncIterable, and will be set as the [body of the HTTP Response](https://developer.mozilla.org/en-US/docs/Web/API/Response/Response#body) via a [`ReadableStream`](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams#consuming_a_fetch_as_a_stream) or other efficient method that avoids loading the entire response into memory or getting the entire response from the network before returning a response to the user. - -If your content doesn't have a mime-type or an [IPFS spec](https://specs.ipfs.tech), this library will not support it, but you can use the [`helia`](https://github.com/ipfs/helia) library directly for those use cases. See [Unsupported response types](#unsupported-response-types) for more information. - -#### Handling response types - -For handling responses we want to follow conventions/abstractions from Fetch API where possible: - -- For JSON, assuming you abstract any differences between dag-json/dag-cbor/json/and json-file-on-unixfs, you would call `.json()` to get a JSON object. -- For images (or other web-relevant asset) you want to add to the DOM, use `.blob()` or `.arrayBuffer()` to get the raw bytes. -- For plain text in utf-8, you would call `.text()` -- For streaming response data, use something like `response.body.getReader()` to get a [`ReadableStream`](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams#consuming_a_fetch_as_a_stream). - -#### Unsupported response types - -- Returning IPLD nodes or DAGs as JS objects is not supported, as there is no currently well-defined structure for representing this data in an [HTTP Response](https://developer.mozilla.org/en-US/docs/Web/API/Response). Instead, users should request `aplication/vnd.ipld.car` or use the [`helia`](https://github.com/ipfs/helia) library directly for this use case. -- Others? Open an issue or PR! - -### Response headers - -This library will set the [HTTP Response](https://developer.mozilla.org/en-US/docs/Web/API/Response) headers to the appropriate values for the content type according to the appropriate [IPFS Specifications](https://specs.ipfs.tech/). - -Some known header specifications: - -- -- -- - -### Possible Scenarios that could cause confusion - -#### Attempting to fetch the CID for content that does not make sense - -If you request `bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze`, which points to the root of the en.wikipedia.org mirror, a response object does not make sense. - -### Errors - -Known Errors that can be thrown: - -1. `TypeError` - If the resource argument is not a string, CID, or CID string. -2. `TypeError` - If the options argument is passed and not an object. -3. `TypeError` - If the options argument is passed and is malformed. -4. `AbortError` - If the content request is aborted due to user aborting provided AbortSignal. - -# Install - -```console -$ npm i @helia/verified-fetch -``` - -## Browser ` -``` - -# API Docs - -- - -# License - -Licensed under either of - -- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) -- MIT ([LICENSE-MIT](LICENSE-MIT) / ) - -# Contribute - -Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia/issues). - -Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. - -Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). - -Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. - -[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json deleted file mode 100644 index e71d86cc..00000000 --- a/packages/verified-fetch/package.json +++ /dev/null @@ -1,197 +0,0 @@ -{ - "name": "@helia/verified-fetch", - "version": "0.0.0", - "description": "A fetch-like API for obtaining verified & trustless IPFS content on the web.", - "license": "Apache-2.0 OR MIT", - "homepage": "https://github.com/ipfs/helia/tree/main/packages/verified-fetch#readme", - "repository": { - "type": "git", - "url": "git+https://github.com/ipfs/helia.git" - }, - "bugs": { - "url": "https://github.com/ipfs/helia/issues" - }, - "publishConfig": { - "access": "public", - "provenance": true - }, - "keywords": [ - "IPFS", - "fetch", - "helia" - ], - "type": "module", - "types": "./dist/src/index.d.ts", - "files": [ - "src", - "dist", - "!dist/test", - "!**/*.tsbuildinfo" - ], - "exports": { - ".": { - "types": "./dist/src/index.d.ts", - "import": "./dist/src/index.js" - } - }, - "eslintConfig": { - "extends": "ipfs", - "parserOptions": { - "project": true, - "sourceType": "module" - } - }, - "release": { - "branches": [ - "main" - ], - "plugins": [ - [ - "@semantic-release/commit-analyzer", - { - "preset": "conventionalcommits", - "releaseRules": [ - { - "breaking": true, - "release": "major" - }, - { - "revert": true, - "release": "patch" - }, - { - "type": "feat", - "release": "minor" - }, - { - "type": "fix", - "release": "patch" - }, - { - "type": "docs", - "release": "patch" - }, - { - "type": "test", - "release": "patch" - }, - { - "type": "deps", - "release": "patch" - }, - { - "scope": "no-release", - "release": false - } - ] - } - ], - [ - "@semantic-release/release-notes-generator", - { - "preset": "conventionalcommits", - "presetConfig": { - "types": [ - { - "type": "feat", - "section": "Features" - }, - { - "type": "fix", - "section": "Bug Fixes" - }, - { - "type": "chore", - "section": "Trivial Changes" - }, - { - "type": "docs", - "section": "Documentation" - }, - { - "type": "deps", - "section": "Dependencies" - }, - { - "type": "test", - "section": "Tests" - } - ] - } - } - ], - "@semantic-release/changelog", - "@semantic-release/npm", - "@semantic-release/github", - "@semantic-release/git" - ] - }, - "scripts": { - "clean": "aegir clean", - "lint": "aegir lint", - "dep-check": "aegir dep-check", - "build": "aegir build", - "test": "aegir test", - "test:chrome": "aegir test -t browser --cov", - "test:chrome-webworker": "aegir test -t webworker", - "test:firefox": "aegir test -t browser -- --browser firefox", - "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", - "test:node": "aegir test -t node --cov", - "test:electron-main": "aegir test -t electron-main", - "release": "aegir release" - }, - "dependencies": { - "@helia/block-brokers": "^2.0.1", - "@helia/car": "^3.0.0", - "@helia/http": "^1.0.1", - "@helia/interface": "^4.0.0", - "@helia/ipns": "^6.0.0", - "@helia/routers": "^1.0.0", - "@helia/unixfs": "^3.0.0", - "@ipld/dag-cbor": "^9.2.0", - "@ipld/dag-json": "^10.2.0", - "@ipld/dag-pb": "^4.1.0", - "@libp2p/interface": "^1.1.2", - "@libp2p/kad-dht": "^12.0.7", - "@libp2p/peer-id": "^4.0.5", - "cborg": "^4.0.9", - "hashlru": "^2.3.0", - "interface-blockstore": "^5.2.10", - "interface-datastore": "^8.2.11", - "ipfs-unixfs-exporter": "^13.5.0", - "it-map": "^3.0.5", - "it-pipe": "^3.0.1", - "it-tar": "^6.0.4", - "it-to-browser-readablestream": "^2.0.6", - "multiformats": "^13.1.0", - "progress-events": "^1.0.0", - "uint8arrays": "^5.0.2" - }, - "devDependencies": { - "@helia/car": "^3.0.0", - "@helia/dag-cbor": "^3.0.0", - "@helia/dag-json": "^3.0.0", - "@helia/json": "^3.0.0", - "@helia/utils": "^0.0.1", - "@ipld/car": "^5.2.6", - "@libp2p/logger": "^4.0.5", - "@libp2p/peer-id-factory": "^4.0.5", - "@sgtpooki/file-type": "^1.0.1", - "@types/sinon": "^17.0.3", - "aegir": "^42.2.2", - "blockstore-core": "^4.4.0", - "browser-readablestream-to-it": "^2.0.5", - "datastore-core": "^9.2.8", - "helia": "^4.0.1", - "ipfs-unixfs-importer": "^15.2.4", - "ipns": "^9.0.0", - "it-all": "^3.0.4", - "it-last": "^3.0.4", - "it-to-buffer": "^4.0.5", - "magic-bytes.js": "^1.8.0", - "p-defer": "^4.0.0", - "sinon": "^17.0.1", - "sinon-ts": "^2.0.0" - }, - "sideEffects": false -} diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts deleted file mode 100644 index ef54ce29..00000000 --- a/packages/verified-fetch/src/index.ts +++ /dev/null @@ -1,632 +0,0 @@ -/** - * @packageDocumentation - * - * `@helia/verified-fetch` provides a [fetch](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API)-like API for retrieving content from the [IPFS](https://ipfs.tech/) network. - * - * All content is retrieved in a [trustless manner](https://www.techopedia.com/definition/trustless), and the integrity of all bytes are verified by comparing hashes of the data. By default, CIDs are retrieved over HTTP from [trustless gateways](https://specs.ipfs.tech/http-gateways/trustless-gateway/). - * - * This is a marked improvement over `fetch` which offers no such protections and is vulnerable to all sorts of attacks like [Content Spoofing](https://owasp.org/www-community/attacks/Content_Spoofing), [DNS Hijacking](https://en.wikipedia.org/wiki/DNS_hijacking), etc. - * - * A `verifiedFetch` function is exported to get up and running quickly, and a `createVerifiedFetch` function is also available that allows customizing the underlying [Helia](https://helia.io/) node for complete control over how content is retrieved. - * - * Browser-cache-friendly [Response](https://developer.mozilla.org/en-US/docs/Web/API/Response) objects are returned which should be instantly familiar to web developers. - * - * You may use any supported resource argument to fetch content: - * - * - [CID](https://multiformats.github.io/js-multiformats/classes/cid.CID.html) instance - * - IPFS URL - * - IPNS URL - * - * @example Getting started - * - * ```typescript - * import { verifiedFetch } from '@helia/verified-fetch' - * - * const resp = await verifiedFetch('ipfs://bafy...') - * - * const json = await resp.json() - *``` - * - * @example Using a CID instance to fetch JSON - * - * ```typescript - * import { verifiedFetch } from '@helia/verified-fetch' - * import { CID } from 'multiformats/cid' - * - * const cid = CID.parse('bafyFoo') // some json file - * const response = await verifiedFetch(cid) - * const json = await response.json() - * ``` - * - * @example Using IPFS protocol to fetch an image - * - * ```typescript - * import { verifiedFetch } from '@helia/verified-fetch' - * - * const response = await verifiedFetch('ipfs://bafyFoo') // CID for some image file - * const blob = await response.blob() - * const image = document.createElement('img') - * image.src = URL.createObjectURL(blob) - * document.body.appendChild(image) - * ``` - * - * @example Using IPNS protocol to stream a big file - * - * ```typescript - * import { verifiedFetch } from '@helia/verified-fetch' - * - * const response = await verifiedFetch('ipns://mydomain.com/path/to/very-long-file.log') - * const bigFileStreamReader = await response.body.getReader() - * ``` - * - * ## Configuration - * - * ### Custom HTTP gateways and routers - * - * Out of the box `@helia/verified-fetch` uses a default set of [trustless gateways](https://specs.ipfs.tech/http-gateways/trustless-gateway/) for fetching blocks and [HTTP delegated routers](https://specs.ipfs.tech/routing/http-routing-v1/) for performing routing tasks - looking up peers, resolving/publishing [IPNS](https://docs.ipfs.tech/concepts/ipns/) names, etc. - * - * It's possible to override these by passing `gateways` and `routers` keys to the `createVerifiedFetch` function: - * - * @example Configuring gateways and routers - * - * ```typescript - * import { createVerifiedFetch } from '@helia/verified-fetch' - * - * const fetch = await createVerifiedFetch({ - * gateways: ['https://trustless-gateway.link'], - * routers: ['http://delegated-ipfs.dev'] - * }) - * - * const resp = await fetch('ipfs://bafy...') - * - * const json = await resp.json() - *``` - * - * ### Usage with customized Helia - * - * For full control of how `@helia/verified-fetch` fetches content from the distributed web you can pass a preconfigured Helia node to `createVerifiedFetch`. - * - * The [helia](https://www.npmjs.com/package/helia) module is configured with a libp2p node that is suited for decentralized applications, alternatively [@helia/http](https://www.npmjs.com/package/@helia/http) is available which uses HTTP gateways for all network operations. - * - * You can see variations of Helia and js-libp2p configuration options at https://helia.io/interfaces/helia.index.HeliaInit.html. - * - * ```typescript - * import { trustlessGateway } from '@helia/block-brokers' - * import { createHeliaHTTP } from '@helia/http' - * import { delegatedHTTPRouting } from '@helia/routers' - * import { createVerifiedFetch } from '@helia/verified-fetch' - * - * const fetch = await createVerifiedFetch( - * await createHeliaHTTP({ - * blockBrokers: [ - * trustlessGateway({ - * gateways: ['https://mygateway.example.net', 'https://trustless-gateway.link'] - * }) - * ], - * routers: ['http://delegated-ipfs.dev'].map((routerUrl) => delegatedHTTPRouting(routerUrl)) - * }) - * ) - * - * const resp = await fetch('ipfs://bafy...') - * - * const json = await resp.json() - * ``` - * - * ### Custom content-type parsing - * - * By default, if the response can be parsed as JSON, `@helia/verified-fetch` sets the `Content-Type` header as `application/json`, otherwise it sets it as `application/octet-stream` - this is because the `.json()`, `.text()`, `.blob()`, and `.arrayBuffer()` methods will usually work as expected without a detailed content type. - * - * If you require an accurate content-type you can provide a `contentTypeParser` function as an option to `createVerifiedFetch` to handle parsing the content type. - * - * The function you provide will be called with the first chunk of bytes from the file and should return a string or a promise of a string. - * - * @example Customizing content-type parsing - * - * ```typescript - * import { createVerifiedFetch } from '@helia/verified-fetch' - * import { fileTypeFromBuffer } from '@sgtpooki/file-type' - * - * const fetch = await createVerifiedFetch({ - * gateways: ['https://trustless-gateway.link'], - * routers: ['http://delegated-ipfs.dev'] - * }, { - * contentTypeParser: async (bytes) => { - * // call to some magic-byte recognition library like magic-bytes, file-type, or your own custom byte recognition - * const result = await fileTypeFromBuffer(bytes) - * return result?.mime - * } - * }) - * ``` - * - * ### Custom DNS resolvers - * - * If you don't want to leak DNS queries to the default resolvers, you can provide your own list of DNS resolvers to `createVerifiedFetch`. - * - * Note that you do not need to provide both a DNS-over-HTTPS and a DNS-over-JSON resolver, and you should prefer `dnsJsonOverHttps` resolvers for usage in the browser for a smaller bundle size. See https://github.com/ipfs/helia/tree/main/packages/ipns#example---using-dns-json-over-https for more information. - * - * @example Customizing DNS resolvers - * - * ```typescript - * import { createVerifiedFetch } from '@helia/verified-fetch' - * import { dnsJsonOverHttps, dnsOverHttps } from '@helia/ipns/dns-resolvers' - * - * const fetch = await createVerifiedFetch({ - * gateways: ['https://trustless-gateway.link'], - * routers: ['http://delegated-ipfs.dev'], - * dnsResolvers: [ - * dnsJsonOverHttps('https://my-dns-resolver.example.com/dns-json'), - * dnsOverHttps('https://my-dns-resolver.example.com/dns-query') - * ] - * }) - * ``` - * - * ### IPLD codec handling - * - * IPFS supports several data formats (typically referred to as codecs) which are included in the CID. `@helia/verified-fetch` attempts to abstract away some of the details for easier consumption. - * - * #### DAG-PB - * - * [DAG-PB](https://ipld.io/docs/codecs/known/dag-pb/) is the codec we are most likely to encounter, it is what [UnixFS](https://github.com/ipfs/specs/blob/main/UNIXFS.md) uses under the hood. - * - * ##### Using the DAG-PB codec as a Blob - * - * ```typescript - * import { verifiedFetch } from '@helia/verified-fetch' - * - * const res = await verifiedFetch('ipfs://Qmfoo') - * const blob = await res.blob() - * - * console.info(blob) // Blob { size: x, type: 'application/octet-stream' } - * ``` - * - * ##### Using the DAG-PB codec as an ArrayBuffer - * - * ```typescript - * import { verifiedFetch } from '@helia/verified-fetch' - * - * const res = await verifiedFetch('ipfs://Qmfoo') - * const buf = await res.arrayBuffer() - * - * console.info(buf) // ArrayBuffer { [Uint8Contents]: < ... >, byteLength: x } - * ``` - * - * ##### Using the DAG-PB codec as a stream - * - * ```typescript - * import { verifiedFetch } from '@helia/verified-fetch' - * - * const res = await verifiedFetch('ipfs://Qmfoo') - * const reader = res.body?.getReader() - * - * while (true) { - * const next = await reader.read() - * - * if (next?.done === true) { - * break - * } - * - * if (next?.value != null) { - * console.info(next.value) // Uint8Array(x) [ ... ] - * } - * } - * ``` - * - * ##### Content-Type - * - * When fetching `DAG-PB` data, the content type will be set to `application/octet-stream` unless a custom content-type parser is configured. - * - * #### JSON - * - * The JSON codec is a very simple codec, a block parseable with this codec is a JSON string encoded into a `Uint8Array`. - * - * ##### Using the JSON codec - * - * ```typescript - * import * as json from 'multiformats/codecs/json' - * - * const block = new TextEncoder().encode('{ "hello": "world" }') - * const obj = json.decode(block) - * - * console.info(obj) // { hello: 'world' } - * ``` - * - * ##### Content-Type - * - * When the `JSON` codec is encountered, the `Content-Type` header of the response will be set to `application/json`. - * - * ### DAG-JSON - * - * [DAG-JSON](https://ipld.io/docs/codecs/known/dag-json/) expands on the `JSON` codec, adding the ability to contain [CID](https://docs.ipfs.tech/concepts/content-addressing/)s which act as links to other blocks, and byte arrays. - * - * `CID`s and byte arrays are represented using special object structures with a single `"/"` property. - * - * Using `DAG-JSON` has two important caveats: - * - * 1. Your `JSON` structure cannot contain an object with only a `"/"` property, as it will be interpreted as a special type. - * 2. Since `JSON` has no technical limit on number sizes, `DAG-JSON` also allows numbers larger than `Number.MAX_SAFE_INTEGER`. JavaScript requires use of `BigInt`s to represent numbers larger than this, and `JSON.parse` does not support them, so precision will be lost. - * - * Otherwise this codec follows the same rules as the `JSON` codec. - * - * ##### Using the DAG-JSON codec - * - * ```typescript - * import * as dagJson from '@ipld/dag-json' - * - * const block = new TextEncoder().encode(`{ - * "hello": "world", - * "cid": { - * "/": "baeaaac3imvwgy3zao5xxe3de" - * }, - * "buf": { - * "/": { - * "bytes": "AAECAwQ" - * } - * } - * }`) - * - * const obj = dagJson.decode(block) - * - * console.info(obj) - * // { - * // hello: 'world', - * // cid: CID(baeaaac3imvwgy3zao5xxe3de), - * // buf: Uint8Array(5) [ 0, 1, 2, 3, 4 ] - * // } - * ``` - * - * ##### Content-Type - * - * When the `DAG-JSON` codec is encountered in the requested CID, the `Content-Type` header of the response will be set to `application/json`. - * - * `DAG-JSON` data can be parsed from the response by using the `.json()` function, which will return `CID`s/byte arrays as plain `{ "/": ... }` objects: - * - * ```typescript - * import { verifiedFetch } from '@helia/verified-fetch' - * import * as dagJson from '@ipld/dag-json' - * - * const res = await verifiedFetch('ipfs://bafyDAGJSON') - * - * // either: - * const obj = await res.json() - * console.info(obj.cid) // { "/": "baeaaac3imvwgy3zao5xxe3de" } - * console.info(obj.buf) // { "/": { "bytes": "AAECAwQ" } } - * ``` - * - * Alternatively, it can be decoded using the `@ipld/dag-json` module and the `.arrayBuffer()` method, in which case you will get `CID` objects and `Uint8Array`s: - * - *```typescript - * import { verifiedFetch } from '@helia/verified-fetch' - * import * as dagJson from '@ipld/dag-json' - * - * const res = await verifiedFetch('ipfs://bafyDAGJSON') - * - * // or: - * const obj = dagJson.decode(await res.arrayBuffer()) - * console.info(obj.cid) // CID(baeaaac3imvwgy3zao5xxe3de) - * console.info(obj.buf) // Uint8Array(5) [ 0, 1, 2, 3, 4 ] - * ``` - * - * #### DAG-CBOR - * - * [DAG-CBOR](https://ipld.io/docs/codecs/known/dag-cbor/) uses the [Concise Binary Object Representation](https://cbor.io/) format for serialization instead of JSON. - * - * This supports more datatypes in a safer way than JSON and is smaller on the wire to boot so is usually preferable to JSON or DAG-JSON. - * - * ##### Content-Type - * - * Not all data types supported by `DAG-CBOR` can be successfully turned into JSON and back into the same binary form. - * - * When a decoded block can be round-tripped to JSON, the `Content-Type` will be set to `application/json`. In this case the `.json()` method on the `Response` object can be used to obtain an object representation of the response. - * - * When it cannot, the `Content-Type` will be `application/octet-stream` - in this case the `@ipld/dag-json` module must be used to deserialize the return value from `.arrayBuffer()`. - * - * ##### Detecting JSON-safe DAG-CBOR - * - * If the `Content-Type` header of the response is `application/json`, the `.json()` method may be used to access the response body in object form, otherwise the `.arrayBuffer()` method must be used to decode the raw bytes using the `@ipld/dag-cbor` module. - * - * ```typescript - * import { verifiedFetch } from '@helia/verified-fetch' - * import * as dagCbor from '@ipld/dag-cbor' - * - * const res = await verifiedFetch('ipfs://bafyDagCborCID') - * let obj - * - * if (res.headers.get('Content-Type') === 'application/json') { - * // DAG-CBOR data can be safely decoded as JSON - * obj = await res.json() - * } else { - * // response contains non-JSON friendly data types - * obj = dagCbor.decode(await res.arrayBuffer()) - * } - * - * console.info(obj) // ... - * ``` - * - * ## The `Accept` header - * - * The `Accept` header can be passed to override certain response processing, or to ensure that the final `Content-Type` of the response is the one that is expected. - * - * If the final `Content-Type` does not match the `Accept` header, or if the content cannot be represented in the format dictated by the `Accept` header, or you have configured a custom content type parser, and that parser returns a value that isn't in the accept header, a [406: Not Acceptable](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/406) response will be returned: - * - * ```typescript - * import { verifiedFetch } from '@helia/verified-fetch' - * - * const res = await verifiedFetch('ipfs://bafyJPEGImageCID', { - * headers: { - * accept: 'image/png' - * } - * }) - * - * console.info(res.status) // 406 - the image was a JPEG but we specified PNG as the accept header - * ``` - * - * It can also be used to skip processing the data from some formats such as `DAG-CBOR` if you wish to handle decoding it yourself: - * - * ```typescript - * import { verifiedFetch } from '@helia/verified-fetch' - * - * const res = await verifiedFetch('ipfs://bafyDAGCBORCID', { - * headers: { - * accept: 'application/octet-stream' - * } - * }) - * - * console.info(res.headers.get('accept')) // application/octet-stream - * const buf = await res.arrayBuffer() // raw bytes, not processed as JSON - * ``` - * - * ## Comparison to fetch - * - * This module attempts to act as similarly to the `fetch()` API as possible. - * - * [The `fetch()` API](https://developer.mozilla.org/en-US/docs/Web/API/fetch) takes two parameters: - * - * 1. A [resource](https://developer.mozilla.org/en-US/docs/Web/API/fetch#resource) - * 2. An [options object](https://developer.mozilla.org/en-US/docs/Web/API/fetch#options) - * - * ### Resource argument - * - * This library supports the following methods of fetching web3 content from IPFS: - * - * 1. IPFS protocol: `ipfs://` & `ipfs://` - * 2. IPNS protocol: `ipns://` & `ipns://` & `ipns://` - * 3. CID instances: An actual CID instance `CID.parse('bafy...')` - * - * As well as support for pathing & params for items 1 & 2 above according to [IPFS - Path Gateway Specification](https://specs.ipfs.tech/http-gateways/path-gateway) & [IPFS - Trustless Gateway Specification](https://specs.ipfs.tech/http-gateways/trustless-gateway/). Further refinement of those specifications specifically for web-based scenarios can be found in the [Web Pathing Specification IPIP](https://github.com/ipfs/specs/pull/453). - * - * If you pass a CID instance, it assumes you want the content for that specific CID only, and does not support pathing or params for that CID. - * - * ### Options argument - * - * This library does not plan to support the exact Fetch API options object, as some of the arguments don't make sense. Instead, it will only support options necessary to meet [IPFS specs](https://specs.ipfs.tech/) related to specifying the resultant shape of desired content. - * - * Some of those header specifications are: - * - * 1. https://specs.ipfs.tech/http-gateways/path-gateway/#request-headers - * 2. https://specs.ipfs.tech/http-gateways/trustless-gateway/#request-headers - * 3. https://specs.ipfs.tech/http-gateways/subdomain-gateway/#request-headers - * - * Where possible, options and Helia internals will be automatically configured to the appropriate codec & content type based on the `verified-fetch` configuration and `options` argument passed. - * - * Known Fetch API options that will be supported: - * - * 1. `signal` - An AbortSignal that a user can use to abort the request. - * 2. `redirect` - A string that specifies the redirect type. One of `follow`, `error`, or `manual`. Defaults to `follow`. Best effort to adhere to the [Fetch API redirect](https://developer.mozilla.org/en-US/docs/Web/API/fetch#redirect) parameter. - * 3. `headers` - An object of headers to be sent with the request. Best effort to adhere to the [Fetch API headers](https://developer.mozilla.org/en-US/docs/Web/API/fetch#headers) parameter. - * - `accept` - A string that specifies the accept header. Relevant values: - * - [`vnd.ipld.raw`](https://www.iana.org/assignments/media-types/application/vnd.ipld.raw). (default) - * - [`vnd.ipld.car`](https://www.iana.org/assignments/media-types/application/vnd.ipld.car) - * - [`vnd.ipfs.ipns-record`](https://www.iana.org/assignments/media-types/application/vnd.ipfs.ipns-record) - * 4. `method` - A string that specifies the HTTP method to use for the request. Defaults to `GET`. Best effort to adhere to the [Fetch API method](https://developer.mozilla.org/en-US/docs/Web/API/fetch#method) parameter. - * 5. `body` - An object that specifies the body of the request. Best effort to adhere to the [Fetch API body](https://developer.mozilla.org/en-US/docs/Web/API/fetch#body) parameter. - * 6. `cache` - Will basically act as `force-cache` for the request. Best effort to adhere to the [Fetch API cache](https://developer.mozilla.org/en-US/docs/Web/API/fetch#cache) parameter. - * - * Non-Fetch API options that will be supported: - * - * 1. `onProgress` - Similar to Helia `onProgress` options, this will be a function that will be called with a progress event. Supported progress events are: - * - `helia:verified-fetch:error` - An error occurred during the request. - * - `helia:verified-fetch:request:start` - The request has been sent - * - `helia:verified-fetch:request:complete` - The request has been sent - * - `helia:verified-fetch:request:error` - An error occurred during the request. - * - `helia:verified-fetch:request:abort` - The request was aborted prior to completion. - * - `helia:verified-fetch:response:start` - The initial HTTP Response headers have been set, and response stream is started. - * - `helia:verified-fetch:response:complete` - The response stream has completed. - * - `helia:verified-fetch:response:error` - An error occurred while building the response. - * - * Some in-flight specs (IPIPs) that will affect the options object this library supports in the future can be seen at https://specs.ipfs.tech/ipips, a subset are: - * - * 1. [IPIP-0412: Signaling Block Order in CARs on HTTP Gateways](https://specs.ipfs.tech/ipips/ipip-0412/) - * 2. [IPIP-0402: Partial CAR Support on Trustless Gateways](https://specs.ipfs.tech/ipips/ipip-0402/) - * 3. [IPIP-0386: Subdomain Gateway Interop with _redirects](https://specs.ipfs.tech/ipips/ipip-0386/) - * 4. [IPIP-0328: JSON and CBOR Response Formats on HTTP Gateways](https://specs.ipfs.tech/ipips/ipip-0328/) - * 5. [IPIP-0288: TAR Response Format on HTTP Gateways](https://specs.ipfs.tech/ipips/ipip-0288/) - * - * ### Response types - * - * This library's purpose is to return reasonably representable content from IPFS. In other words, fetching content is intended for leaf-node content -- such as images/videos/audio & other assets, or other IPLD content (with link) -- that can be represented by https://developer.mozilla.org/en-US/docs/Web/API/Response#instance_methods. The content type you receive back will depend upon the CID you request as well as the `Accept` header value you provide. - * - * All content we retrieve from the IPFS network is obtained via an AsyncIterable, and will be set as the [body of the HTTP Response](https://developer.mozilla.org/en-US/docs/Web/API/Response/Response#body) via a [`ReadableStream`](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams#consuming_a_fetch_as_a_stream) or other efficient method that avoids loading the entire response into memory or getting the entire response from the network before returning a response to the user. - * - * If your content doesn't have a mime-type or an [IPFS spec](https://specs.ipfs.tech), this library will not support it, but you can use the [`helia`](https://github.com/ipfs/helia) library directly for those use cases. See [Unsupported response types](#unsupported-response-types) for more information. - * - * #### Handling response types - * - * For handling responses we want to follow conventions/abstractions from Fetch API where possible: - * - * - For JSON, assuming you abstract any differences between dag-json/dag-cbor/json/and json-file-on-unixfs, you would call `.json()` to get a JSON object. - * - For images (or other web-relevant asset) you want to add to the DOM, use `.blob()` or `.arrayBuffer()` to get the raw bytes. - * - For plain text in utf-8, you would call `.text()` - * - For streaming response data, use something like `response.body.getReader()` to get a [`ReadableStream`](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams#consuming_a_fetch_as_a_stream). - * - * #### Unsupported response types - * - * * Returning IPLD nodes or DAGs as JS objects is not supported, as there is no currently well-defined structure for representing this data in an [HTTP Response](https://developer.mozilla.org/en-US/docs/Web/API/Response). Instead, users should request `aplication/vnd.ipld.car` or use the [`helia`](https://github.com/ipfs/helia) library directly for this use case. - * * Others? Open an issue or PR! - * - * ### Response headers - * - * This library will set the [HTTP Response](https://developer.mozilla.org/en-US/docs/Web/API/Response) headers to the appropriate values for the content type according to the appropriate [IPFS Specifications](https://specs.ipfs.tech/). - * - * Some known header specifications: - * - * * https://specs.ipfs.tech/http-gateways/path-gateway/#response-headers - * * https://specs.ipfs.tech/http-gateways/trustless-gateway/#response-headers - * * https://specs.ipfs.tech/http-gateways/subdomain-gateway/#response-headers - * - * ### Possible Scenarios that could cause confusion - * - * #### Attempting to fetch the CID for content that does not make sense - * - * If you request `bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze`, which points to the root of the en.wikipedia.org mirror, a response object does not make sense. - * - * ### Errors - * - * Known Errors that can be thrown: - * - * 1. `TypeError` - If the resource argument is not a string, CID, or CID string. - * 2. `TypeError` - If the options argument is passed and not an object. - * 3. `TypeError` - If the options argument is passed and is malformed. - * 4. `AbortError` - If the content request is aborted due to user aborting provided AbortSignal. - */ - -import { trustlessGateway } from '@helia/block-brokers' -import { createHeliaHTTP } from '@helia/http' -import { delegatedHTTPRouting } from '@helia/routers' -import { VerifiedFetch as VerifiedFetchClass } from './verified-fetch.js' -import type { Helia } from '@helia/interface' -import type { DNSResolver, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents } from '@helia/ipns' -import type { GetEvents } from '@helia/unixfs' -import type { CID } from 'multiformats/cid' -import type { ProgressEvent, ProgressOptions } from 'progress-events' - -/** - * The types for the first argument of the `verifiedFetch` function. - */ -export type Resource = string | CID - -export interface ResourceDetail { - resource: Resource -} - -export interface CIDDetail { - cid: CID - path: string -} - -export interface CIDDetailError extends CIDDetail { - error: Error -} - -export interface VerifiedFetch { - (resource: Resource, options?: VerifiedFetchInit): Promise - start(): Promise - stop(): Promise -} - -/** - * Instead of passing a Helia instance, you can pass a list of gateways and - * routers, and a HeliaHTTP instance will be created for you. - */ -export interface CreateVerifiedFetchInit { - gateways: string[] - routers?: string[] - - /** - * In order to parse DNSLink records, we need to resolve DNS queries. You can - * pass a list of DNS resolvers that we will provide to the @helia/ipns - * instance for you. You must construct them using the `dnsJsonOverHttps` or - * `dnsOverHttps` functions exported from `@helia/ipns/dns-resolvers`. - * - * We use cloudflare and google's dnsJsonOverHttps resolvers by default. - * - * @default [dnsJsonOverHttps('https://mozilla.cloudflare-dns.com/dns-query'),dnsJsonOverHttps('https://dns.google/resolve')] - */ - dnsResolvers?: DNSResolver[] -} - -export interface CreateVerifiedFetchOptions { - /** - * A function to handle parsing content type from bytes. The function you - * provide will be passed the first set of bytes we receive from the network, - * and should return a string that will be used as the value for the - * `Content-Type` header in the response. - * - * @default undefined - */ - contentTypeParser?: ContentTypeParser -} - -/** - * A ContentTypeParser attempts to return the mime type of a given file. It - * receives the first chunk of the file data and the file name, if it is - * available. The function can be sync or async and if it returns/resolves to - * `undefined`, `application/octet-stream` will be used. - */ -export interface ContentTypeParser { - /** - * Attempt to determine a mime type, either via of the passed bytes or the - * filename if it is available. - */ - (bytes: Uint8Array, fileName?: string): Promise | string | undefined -} - -export type BubbledProgressEvents = - // unixfs - GetEvents | - // ipns - ResolveProgressEvents | ResolveDnsLinkProgressEvents | IPNSRoutingEvents - -export type VerifiedFetchProgressEvents = - ProgressEvent<'verified-fetch:request:start', CIDDetail> | - ProgressEvent<'verified-fetch:request:info', string> | - ProgressEvent<'verified-fetch:request:progress:chunk', CIDDetail> | - ProgressEvent<'verified-fetch:request:end', CIDDetail> | - ProgressEvent<'verified-fetch:request:error', CIDDetailError> - -/** - * Options for the `fetch` function returned by `createVerifiedFetch`. - * - * This interface contains all the same fields as the [options object](https://developer.mozilla.org/en-US/docs/Web/API/fetch#options) - * passed to `fetch` in browsers, plus an `onProgress` option to listen for - * progress events. - */ -export interface VerifiedFetchInit extends RequestInit, ProgressOptions { -} - -/** - * Create and return a Helia node - */ -export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchInit, options?: CreateVerifiedFetchOptions): Promise { - let dnsResolvers: DNSResolver[] | undefined - if (!isHelia(init)) { - dnsResolvers = init?.dnsResolvers - init = await createHeliaHTTP({ - blockBrokers: [ - trustlessGateway({ - gateways: init?.gateways - }) - ], - routers: (init?.routers ?? ['https://delegated-ipfs.dev']).map((routerUrl) => delegatedHTTPRouting(routerUrl)) - }) - } - - const verifiedFetchInstance = new VerifiedFetchClass({ helia: init }, { dnsResolvers, ...options }) - async function verifiedFetch (resource: Resource, options?: VerifiedFetchInit): Promise { - return verifiedFetchInstance.fetch(resource, options) - } - verifiedFetch.stop = verifiedFetchInstance.stop.bind(verifiedFetchInstance) - verifiedFetch.start = verifiedFetchInstance.start.bind(verifiedFetchInstance) - - return verifiedFetch -} - -export { verifiedFetch } from './singleton.js' - -function isHelia (obj: any): obj is Helia { - // test for the presence of known Helia properties, return a boolean value - return obj?.blockstore != null && - obj?.datastore != null && - obj?.gc != null && - obj?.stop != null && - obj?.start != null -} diff --git a/packages/verified-fetch/src/singleton.ts b/packages/verified-fetch/src/singleton.ts deleted file mode 100644 index cf877fd7..00000000 --- a/packages/verified-fetch/src/singleton.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { createVerifiedFetch } from './index.js' -import type { Resource, VerifiedFetch, VerifiedFetchInit } from './index.js' - -let impl: VerifiedFetch | undefined - -export const verifiedFetch: VerifiedFetch = async function verifiedFetch (resource: Resource, options?: VerifiedFetchInit): Promise { - if (impl == null) { - impl = await createVerifiedFetch() - } - - return impl(resource, options) -} - -verifiedFetch.start = async function () { - await impl?.start() -} - -verifiedFetch.stop = async function () { - await impl?.stop() -} diff --git a/packages/verified-fetch/src/types.ts b/packages/verified-fetch/src/types.ts deleted file mode 100644 index 4a235e1a..00000000 --- a/packages/verified-fetch/src/types.ts +++ /dev/null @@ -1 +0,0 @@ -export type RequestFormatShorthand = 'raw' | 'car' | 'tar' | 'ipns-record' | 'dag-json' | 'dag-cbor' | 'json' | 'cbor' diff --git a/packages/verified-fetch/src/utils/dag-cbor-to-safe-json.ts b/packages/verified-fetch/src/utils/dag-cbor-to-safe-json.ts deleted file mode 100644 index 0c52bdcc..00000000 --- a/packages/verified-fetch/src/utils/dag-cbor-to-safe-json.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { decode } from 'cborg' -import { encode } from 'cborg/json' -import { CID } from 'multiformats/cid' -import type { TagDecoder } from 'cborg' - -// https://github.com/ipfs/go-ipfs/issues/3570#issuecomment-273931692 -const CID_CBOR_TAG = 0x2A - -function cidDecoder (bytes: Uint8Array): CID { - if (bytes[0] !== 0) { - throw new Error('Invalid CID for CBOR tag 42; expected leading 0x00') - } - - return CID.decode(bytes.subarray(1)) // ignore leading 0x00 -} - -/** - * Take a `DAG-CBOR` encoded `Uint8Array`, deserialize it as an object and - * re-serialize it in a form that can be passed to `JSON.serialize` and then - * `JSON.parse` without losing any data. - */ -export function dagCborToSafeJSON (buf: Uint8Array): string { - const tags: TagDecoder[] = [] - tags[CID_CBOR_TAG] = cidDecoder - - const obj = decode(buf, { - allowIndefinite: false, - coerceUndefinedToNull: true, - allowNaN: false, - allowInfinity: false, - strict: true, - useMaps: false, - rejectDuplicateMapKeys: true, - tags, - - // this is different to `DAG-CBOR` - the reason we disallow BigInts is - // because we are about to re-encode to `JSON` which does not support - // BigInts. Blocks containing large numbers should be deserialized using a - // cbor decoder instead - allowBigInt: false - }) - - return new TextDecoder().decode(encode(obj)) -} diff --git a/packages/verified-fetch/src/utils/get-content-disposition-filename.ts b/packages/verified-fetch/src/utils/get-content-disposition-filename.ts deleted file mode 100644 index cee3ee1f..00000000 --- a/packages/verified-fetch/src/utils/get-content-disposition-filename.ts +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Takes a filename URL param and returns a string for use in a - * `Content-Disposition` header - */ -export function getContentDispositionFilename (filename: string): string { - const asciiOnly = replaceNonAsciiCharacters(filename) - - if (asciiOnly === filename) { - return `filename="${filename}"` - } - - return `filename="${asciiOnly}"; filename*=UTF-8''${encodeURIComponent(filename)}` -} - -function replaceNonAsciiCharacters (filename: string): string { - // eslint-disable-next-line no-control-regex - return filename.replace(/[^\x00-\x7F]/g, '_') -} diff --git a/packages/verified-fetch/src/utils/get-e-tag.ts b/packages/verified-fetch/src/utils/get-e-tag.ts deleted file mode 100644 index 74ab7303..00000000 --- a/packages/verified-fetch/src/utils/get-e-tag.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { RequestFormatShorthand } from '../types.js' -import type { CID } from 'multiformats/cid' - -interface GetETagArg { - cid: CID - reqFormat?: RequestFormatShorthand - rangeStart?: number - rangeEnd?: number - /** - * Weak Etag is used when we can't guarantee byte-for-byte-determinism (generated, or mutable content). - * Some examples: - * - IPNS requests - * - CAR streamed with blocks in non-deterministic order - * - TAR streamed with files in non-deterministic order - */ - weak?: boolean -} - -/** - * etag - * you need to wrap cid with "" - * we use strong Etags for immutable responses and weak one (prefixed with W/ ) for mutable/generated ones (ipns and generated HTML). - * block and car responses should have different etag than deserialized one, so you can add some prefix like we do in existing gateway - * - * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag - * @see https://specs.ipfs.tech/http-gateways/path-gateway/#etag-response-header - */ -export function getETag ({ cid, reqFormat, weak, rangeStart, rangeEnd }: GetETagArg): string { - const prefix = weak === true ? 'W/' : '' - let suffix = reqFormat == null ? '' : `.${reqFormat}` - if (rangeStart != null || rangeEnd != null) { - suffix += `.${rangeStart ?? '0'}-${rangeEnd ?? 'N'}` - } - - return `${prefix}"${cid.toString()}${suffix}"` -} diff --git a/packages/verified-fetch/src/utils/get-stream-from-async-iterable.ts b/packages/verified-fetch/src/utils/get-stream-from-async-iterable.ts deleted file mode 100644 index c9266e2e..00000000 --- a/packages/verified-fetch/src/utils/get-stream-from-async-iterable.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { CustomProgressEvent } from 'progress-events' -import type { VerifiedFetchInit } from '../index.js' -import type { ComponentLogger } from '@libp2p/interface' - -/** - * Converts an async iterator of Uint8Array bytes to a stream and returns the first chunk of bytes - */ -export async function getStreamFromAsyncIterable (iterator: AsyncIterable, path: string, logger: ComponentLogger, options?: Pick): Promise<{ stream: ReadableStream, firstChunk: Uint8Array }> { - const log = logger.forComponent('helia:verified-fetch:get-stream-from-async-iterable') - const reader = iterator[Symbol.asyncIterator]() - const { value: firstChunk, done } = await reader.next() - - if (done === true) { - log.error('No content found for path', path) - throw new Error('No content found') - } - - const stream = new ReadableStream({ - async start (controller) { - // the initial value is already available - options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:progress:chunk')) - controller.enqueue(firstChunk) - }, - async pull (controller) { - const { value, done } = await reader.next() - - if (done === true) { - if (value != null) { - options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:progress:chunk')) - controller.enqueue(value) - } - controller.close() - return - } - - options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:progress:chunk')) - controller.enqueue(value) - } - }) - - return { - stream, - firstChunk - } -} diff --git a/packages/verified-fetch/src/utils/get-tar-stream.ts b/packages/verified-fetch/src/utils/get-tar-stream.ts deleted file mode 100644 index eeec0b4b..00000000 --- a/packages/verified-fetch/src/utils/get-tar-stream.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { CodeError } from '@libp2p/interface' -import { exporter, recursive, type UnixFSEntry } from 'ipfs-unixfs-exporter' -import map from 'it-map' -import { pipe } from 'it-pipe' -import { pack, type TarEntryHeader, type TarImportCandidate } from 'it-tar' -import type { AbortOptions } from '@libp2p/interface' -import type { Blockstore } from 'interface-blockstore' - -const EXPORTABLE = ['file', 'raw', 'directory'] - -function toHeader (file: UnixFSEntry): Partial & { name: string } { - let mode: number | undefined - let mtime: Date | undefined - - if (file.type === 'file' || file.type === 'directory') { - mode = file.unixfs.mode - mtime = file.unixfs.mtime != null ? new Date(Number(file.unixfs.mtime.secs * 1000n)) : undefined - } - - return { - name: file.path, - mode, - mtime, - size: Number(file.size), - type: file.type === 'directory' ? 'directory' : 'file' - } -} - -function toTarImportCandidate (entry: UnixFSEntry): TarImportCandidate { - if (!EXPORTABLE.includes(entry.type)) { - throw new CodeError('Not a UnixFS node', 'ERR_NOT_UNIXFS') - } - - const candidate: TarImportCandidate = { - header: toHeader(entry) - } - - if (entry.type === 'file' || entry.type === 'raw') { - candidate.body = entry.content() - } - - return candidate -} - -export async function * tarStream (ipfsPath: string, blockstore: Blockstore, options?: AbortOptions): AsyncGenerator { - const file = await exporter(ipfsPath, blockstore, options) - - if (file.type === 'file' || file.type === 'raw') { - yield * pipe( - [toTarImportCandidate(file)], - pack() - ) - - return - } - - if (file.type === 'directory') { - yield * pipe( - recursive(ipfsPath, blockstore, options), - (source) => map(source, (entry) => toTarImportCandidate(entry)), - pack() - ) - - return - } - - throw new CodeError('Not a UnixFS node', 'ERR_NOT_UNIXFS') -} diff --git a/packages/verified-fetch/src/utils/parse-resource.ts b/packages/verified-fetch/src/utils/parse-resource.ts deleted file mode 100644 index a20e82d4..00000000 --- a/packages/verified-fetch/src/utils/parse-resource.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { CID } from 'multiformats/cid' -import { parseUrlString } from './parse-url-string.js' -import type { ParsedUrlStringResults } from './parse-url-string.js' -import type { Resource } from '../index.js' -import type { IPNS, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents } from '@helia/ipns' -import type { ComponentLogger } from '@libp2p/interface' -import type { ProgressOptions } from 'progress-events' - -export interface ParseResourceComponents { - ipns: IPNS - logger: ComponentLogger -} - -export interface ParseResourceOptions extends ProgressOptions { - -} -/** - * Handles the different use cases for the `resource` argument. - * The resource can represent an IPFS path, IPNS path, or CID. - * If the resource represents an IPNS path, we need to resolve it to a CID. - */ -export async function parseResource (resource: Resource, { ipns, logger }: ParseResourceComponents, options?: ParseResourceOptions): Promise { - if (typeof resource === 'string') { - return parseUrlString({ urlString: resource, ipns, logger }, { onProgress: options?.onProgress }) - } - - const cid = CID.asCID(resource) - - if (cid != null) { - // an actual CID - return { - cid, - protocol: 'ipfs', - path: '', - query: {} - } - } - - throw new TypeError(`Invalid resource. Cannot determine CID from resource: ${resource}`) -} diff --git a/packages/verified-fetch/src/utils/parse-url-string.ts b/packages/verified-fetch/src/utils/parse-url-string.ts deleted file mode 100644 index d869f77c..00000000 --- a/packages/verified-fetch/src/utils/parse-url-string.ts +++ /dev/null @@ -1,154 +0,0 @@ -import { peerIdFromString } from '@libp2p/peer-id' -import { CID } from 'multiformats/cid' -import { TLRU } from './tlru.js' -import type { RequestFormatShorthand } from '../types.js' -import type { IPNS, IPNSRoutingEvents, ResolveDnsLinkProgressEvents, ResolveProgressEvents, ResolveResult } from '@helia/ipns' -import type { ComponentLogger } from '@libp2p/interface' -import type { ProgressOptions } from 'progress-events' - -const ipnsCache = new TLRU(1000) - -export interface ParseUrlStringInput { - urlString: string - ipns: IPNS - logger: ComponentLogger -} -export interface ParseUrlStringOptions extends ProgressOptions { - -} - -export interface ParsedUrlQuery extends Record { - format?: RequestFormatShorthand - download?: boolean - filename?: string -} - -export interface ParsedUrlStringResults { - protocol: string - path: string - cid: CID - query: ParsedUrlQuery -} - -const URL_REGEX = /^(?ip[fn]s):\/\/(?[^/$?]+)\/?(?[^$?]*)\??(?.*)$/ - -/** - * A function that parses ipfs:// and ipns:// URLs, returning an object with easily recognizable properties. - * - * After determining the protocol successfully, we process the cidOrPeerIdOrDnsLink: - * * If it's ipfs, it parses the CID or throws an Aggregate error - * * If it's ipns, it attempts to resolve the PeerId and then the DNSLink. If both fail, an Aggregate error is thrown. - */ -export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStringInput, options?: ParseUrlStringOptions): Promise { - const log = logger.forComponent('helia:verified-fetch:parse-url-string') - const match = urlString.match(URL_REGEX) - - if (match == null || match.groups == null) { - throw new TypeError(`Invalid URL: ${urlString}, please use ipfs:// or ipns:// URLs only.`) - } - - const { protocol, cidOrPeerIdOrDnsLink, path: urlPath, queryString } = match.groups - - let cid: CID | undefined - let resolvedPath: string | undefined - const errors: Error[] = [] - - if (protocol === 'ipfs') { - try { - cid = CID.parse(cidOrPeerIdOrDnsLink) - } catch (err) { - log.error(err) - errors.push(new TypeError('Invalid CID for ipfs:// URL')) - } - } else { - let resolveResult = ipnsCache.get(cidOrPeerIdOrDnsLink) - - if (resolveResult != null) { - cid = resolveResult.cid - resolvedPath = resolveResult.path - log.trace('resolved %s to %c from cache', cidOrPeerIdOrDnsLink, cid) - } else { - // protocol is ipns - log.trace('Attempting to resolve PeerId for %s', cidOrPeerIdOrDnsLink) - let peerId = null - - try { - peerId = peerIdFromString(cidOrPeerIdOrDnsLink) - resolveResult = await ipns.resolve(peerId, { onProgress: options?.onProgress }) - cid = resolveResult?.cid - resolvedPath = resolveResult?.path - log.trace('resolved %s to %c', cidOrPeerIdOrDnsLink, cid) - ipnsCache.set(cidOrPeerIdOrDnsLink, resolveResult, 60 * 1000 * 2) - } catch (err) { - if (peerId == null) { - log.error('Could not parse PeerId string "%s"', cidOrPeerIdOrDnsLink, err) - errors.push(new TypeError(`Could not parse PeerId in ipns url "${cidOrPeerIdOrDnsLink}", ${(err as Error).message}`)) - } else { - log.error('Could not resolve PeerId %c', peerId, err) - errors.push(new TypeError(`Could not resolve PeerId "${cidOrPeerIdOrDnsLink}", ${(err as Error).message}`)) - } - } - - if (cid == null) { - log.trace('Attempting to resolve DNSLink for %s', cidOrPeerIdOrDnsLink) - - try { - resolveResult = await ipns.resolveDns(cidOrPeerIdOrDnsLink, { onProgress: options?.onProgress }) - cid = resolveResult?.cid - resolvedPath = resolveResult?.path - log.trace('resolved %s to %c', cidOrPeerIdOrDnsLink, cid) - ipnsCache.set(cidOrPeerIdOrDnsLink, resolveResult, 60 * 1000 * 2) - } catch (err) { - log.error('Could not resolve DnsLink for "%s"', cidOrPeerIdOrDnsLink, err) - errors.push(err as Error) - } - } - } - } - - if (cid == null) { - throw new AggregateError(errors, `Invalid resource. Cannot determine CID from URL "${urlString}"`) - } - - // parse query string - const query: Record = {} - - if (queryString != null && queryString.length > 0) { - const queryParts = queryString.split('&') - for (const part of queryParts) { - const [key, value] = part.split('=') - query[key] = decodeURIComponent(value) - } - - if (query.download != null) { - query.download = query.download === 'true' - } - - if (query.filename != null) { - query.filename = query.filename.toString() - } - } - - /** - * join the path from resolve result & given path. - * e.g. /ipns// that is resolved to /ipfs//, when requested as /ipns//, should be - * resolved to /ipfs/// - */ - const pathParts = [] - - if (urlPath.length > 0) { - pathParts.push(urlPath) - } - - if (resolvedPath != null && resolvedPath.length > 0) { - pathParts.push(resolvedPath) - } - const path = pathParts.join('/') - - return { - protocol, - cid, - path, - query - } -} diff --git a/packages/verified-fetch/src/utils/responses.ts b/packages/verified-fetch/src/utils/responses.ts deleted file mode 100644 index 220a1914..00000000 --- a/packages/verified-fetch/src/utils/responses.ts +++ /dev/null @@ -1,29 +0,0 @@ -export function okResponse (body?: BodyInit | null): Response { - return new Response(body, { - status: 200, - statusText: 'OK' - }) -} - -export function notSupportedResponse (body?: BodyInit | null): Response { - const response = new Response(body, { - status: 501, - statusText: 'Not Implemented' - }) - response.headers.set('X-Content-Type-Options', 'nosniff') // see https://specs.ipfs.tech/http-gateways/path-gateway/#x-content-type-options-response-header - return response -} - -export function notAcceptableResponse (body?: BodyInit | null): Response { - return new Response(body, { - status: 406, - statusText: 'Not Acceptable' - }) -} - -export function badRequestResponse (body?: BodyInit | null): Response { - return new Response(body, { - status: 400, - statusText: 'Bad Request' - }) -} diff --git a/packages/verified-fetch/src/utils/select-output-type.ts b/packages/verified-fetch/src/utils/select-output-type.ts deleted file mode 100644 index 0f7c40c9..00000000 --- a/packages/verified-fetch/src/utils/select-output-type.ts +++ /dev/null @@ -1,167 +0,0 @@ -import { code as dagCborCode } from '@ipld/dag-cbor' -import { code as dagJsonCode } from '@ipld/dag-json' -import { code as dagPbCode } from '@ipld/dag-pb' -import { code as jsonCode } from 'multiformats/codecs/json' -import { code as rawCode } from 'multiformats/codecs/raw' -import type { RequestFormatShorthand } from '../types.js' -import type { CID } from 'multiformats/cid' - -/** - * This maps supported response types for each codec supported by verified-fetch - */ -const CID_TYPE_MAP: Record = { - [dagCborCode]: [ - 'application/json', - 'application/vnd.ipld.dag-cbor', - 'application/cbor', - 'application/vnd.ipld.dag-json', - 'application/octet-stream', - 'application/vnd.ipld.raw', - 'application/vnd.ipfs.ipns-record', - 'application/vnd.ipld.car' - ], - [dagJsonCode]: [ - 'application/json', - 'application/vnd.ipld.dag-cbor', - 'application/cbor', - 'application/vnd.ipld.dag-json', - 'application/octet-stream', - 'application/vnd.ipld.raw', - 'application/vnd.ipfs.ipns-record', - 'application/vnd.ipld.car' - ], - [jsonCode]: [ - 'application/json', - 'application/vnd.ipld.dag-cbor', - 'application/cbor', - 'application/vnd.ipld.dag-json', - 'application/octet-stream', - 'application/vnd.ipld.raw', - 'application/vnd.ipfs.ipns-record', - 'application/vnd.ipld.car' - ], - [dagPbCode]: [ - 'application/octet-stream', - 'application/json', - 'application/vnd.ipld.dag-cbor', - 'application/cbor', - 'application/vnd.ipld.dag-json', - 'application/vnd.ipld.raw', - 'application/vnd.ipfs.ipns-record', - 'application/vnd.ipld.car', - 'application/x-tar' - ], - [rawCode]: [ - 'application/octet-stream', - 'application/vnd.ipld.raw', - 'application/vnd.ipfs.ipns-record', - 'application/vnd.ipld.car', - 'application/x-tar' - ] -} - -/** - * Selects an output mime-type based on the CID and a passed `Accept` header - */ -export function selectOutputType (cid: CID, accept?: string): string | undefined { - const cidMimeTypes = CID_TYPE_MAP[cid.code] - - if (accept != null) { - return chooseMimeType(accept, cidMimeTypes) - } -} - -function chooseMimeType (accept: string, validMimeTypes: string[]): string | undefined { - const requestedMimeTypes = accept - .split(',') - .map(s => { - const parts = s.trim().split(';') - - return { - mimeType: `${parts[0]}`.trim(), - weight: parseQFactor(parts[1]) - } - }) - .sort((a, b) => { - if (a.weight === b.weight) { - return 0 - } - - if (a.weight > b.weight) { - return -1 - } - - return 1 - }) - .map(s => s.mimeType) - - for (const headerFormat of requestedMimeTypes) { - for (const mimeType of validMimeTypes) { - if (headerFormat.includes(mimeType)) { - return mimeType - } - - if (headerFormat === '*/*') { - return mimeType - } - - if (headerFormat.startsWith('*/') && mimeType.split('/')[1] === headerFormat.split('/')[1]) { - return mimeType - } - - if (headerFormat.endsWith('/*') && mimeType.split('/')[0] === headerFormat.split('/')[0]) { - return mimeType - } - } - } -} - -/** - * Parses q-factor weighting from the accept header to allow letting some mime - * types take precedence over others. - * - * If the q-factor for an acceptable mime representation is omitted it defaults - * to `1`. - * - * All specified values should be in the range 0-1. - * - * @see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept#q - */ -function parseQFactor (str?: string): number { - if (str != null) { - str = str.trim() - } - - if (str == null || !str.startsWith('q=')) { - return 1 - } - - const factor = parseFloat(str.replace('q=', '')) - - if (isNaN(factor)) { - return 0 - } - - return factor -} - -const FORMAT_TO_MIME_TYPE: Record = { - raw: 'application/vnd.ipld.raw', - car: 'application/vnd.ipld.car', - 'dag-json': 'application/vnd.ipld.dag-json', - 'dag-cbor': 'application/vnd.ipld.dag-cbor', - json: 'application/json', - cbor: 'application/cbor', - 'ipns-record': 'application/vnd.ipfs.ipns-record', - tar: 'application/x-tar' -} - -/** - * Converts a `format=...` query param to a mime type as would be found in the - * `Accept` header, if a valid mapping is available - */ -export function queryFormatToAcceptHeader (format?: RequestFormatShorthand): string | undefined { - if (format != null) { - return FORMAT_TO_MIME_TYPE[format] - } -} diff --git a/packages/verified-fetch/src/utils/tlru.ts b/packages/verified-fetch/src/utils/tlru.ts deleted file mode 100644 index 0556c0e6..00000000 --- a/packages/verified-fetch/src/utils/tlru.ts +++ /dev/null @@ -1,52 +0,0 @@ -import hashlru from 'hashlru' - -/** - * Time Aware Least Recent Used Cache - * - * @see https://arxiv.org/pdf/1801.00390 - */ -export class TLRU { - private readonly lru: ReturnType - - constructor (maxSize: number) { - this.lru = hashlru(maxSize) - } - - get (key: string): T | undefined { - const value = this.lru.get(key) - - if (value != null) { - if (value.expire != null && value.expire < Date.now()) { - this.lru.remove(key) - - return undefined - } - - return value.value - } - - return undefined - } - - set (key: string, value: T, ttl: number): void { - this.lru.set(key, { value, expire: Date.now() + ttl }) - } - - has (key: string): boolean { - const value = this.get(key) - - if (value != null) { - return true - } - - return false - } - - remove (key: string): void { - this.lru.remove(key) - } - - clear (): void { - this.lru.clear() - } -} diff --git a/packages/verified-fetch/src/utils/walk-path.ts b/packages/verified-fetch/src/utils/walk-path.ts deleted file mode 100644 index 45f2066e..00000000 --- a/packages/verified-fetch/src/utils/walk-path.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { walkPath as exporterWalk, type ExporterOptions, type ReadableStorage, type UnixFSEntry } from 'ipfs-unixfs-exporter' -import type { CID } from 'multiformats/cid' - -export interface PathWalkerOptions extends ExporterOptions { - -} -export interface PathWalkerResponse { - ipfsRoots: CID[] - terminalElement: UnixFSEntry - -} - -export interface PathWalkerFn { - (blockstore: ReadableStorage, path: string, options?: PathWalkerOptions): Promise -} - -export async function walkPath (blockstore: ReadableStorage, path: string, options?: PathWalkerOptions): Promise { - const ipfsRoots: CID[] = [] - let terminalElement: UnixFSEntry | undefined - - for await (const entry of exporterWalk(path, blockstore, options)) { - ipfsRoots.push(entry.cid) - terminalElement = entry - } - - if (terminalElement == null) { - throw new Error('No terminal element found') - } - - return { - ipfsRoots, - terminalElement - } -} diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts deleted file mode 100644 index 8935ad16..00000000 --- a/packages/verified-fetch/src/verified-fetch.ts +++ /dev/null @@ -1,510 +0,0 @@ -import { car } from '@helia/car' -import { ipns as heliaIpns, type DNSResolver, type IPNS } from '@helia/ipns' -import { dnsJsonOverHttps } from '@helia/ipns/dns-resolvers' -import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs, type UnixFSStats } from '@helia/unixfs' -import * as ipldDagCbor from '@ipld/dag-cbor' -import * as ipldDagJson from '@ipld/dag-json' -import { code as dagPbCode } from '@ipld/dag-pb' -import { Record as DHTRecord } from '@libp2p/kad-dht' -import { peerIdFromString } from '@libp2p/peer-id' -import { Key } from 'interface-datastore' -import toBrowserReadableStream from 'it-to-browser-readablestream' -import { code as jsonCode } from 'multiformats/codecs/json' -import { code as rawCode } from 'multiformats/codecs/raw' -import { identity } from 'multiformats/hashes/identity' -import { CustomProgressEvent } from 'progress-events' -import { concat as uint8ArrayConcat } from 'uint8arrays/concat' -import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import { dagCborToSafeJSON } from './utils/dag-cbor-to-safe-json.js' -import { getContentDispositionFilename } from './utils/get-content-disposition-filename.js' -import { getETag } from './utils/get-e-tag.js' -import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js' -import { tarStream } from './utils/get-tar-stream.js' -import { parseResource } from './utils/parse-resource.js' -import { badRequestResponse, notAcceptableResponse, notSupportedResponse, okResponse } from './utils/responses.js' -import { selectOutputType, queryFormatToAcceptHeader } from './utils/select-output-type.js' -import { walkPath } from './utils/walk-path.js' -import type { CIDDetail, ContentTypeParser, Resource, VerifiedFetchInit as VerifiedFetchOptions } from './index.js' -import type { RequestFormatShorthand } from './types.js' -import type { Helia } from '@helia/interface' -import type { AbortOptions, Logger, PeerId } from '@libp2p/interface' -import type { UnixFSEntry } from 'ipfs-unixfs-exporter' -import type { CID } from 'multiformats/cid' - -interface VerifiedFetchComponents { - helia: Helia - ipns?: IPNS - unixfs?: HeliaUnixFs -} - -/** - * Potential future options for the VerifiedFetch constructor. - */ -interface VerifiedFetchInit { - contentTypeParser?: ContentTypeParser - dnsResolvers?: DNSResolver[] -} - -interface FetchHandlerFunctionArg { - cid: CID - path: string - options?: Omit & AbortOptions - - /** - * If present, the user has sent an accept header with this value - if the - * content cannot be represented in this format a 406 should be returned - */ - accept?: string - - /** - * The originally requested resource - */ - resource: string -} - -interface FetchHandlerFunction { - (options: FetchHandlerFunctionArg): Promise -} - -function convertOptions (options?: VerifiedFetchOptions): (Omit & AbortOptions) | undefined { - if (options == null) { - return undefined - } - - let signal: AbortSignal | undefined - if (options?.signal === null) { - signal = undefined - } - return { - ...options, - signal - } -} - -/** - * These are Accept header values that will cause content type sniffing to be - * skipped and set to these values. - */ -const RAW_HEADERS = [ - 'application/vnd.ipld.raw', - 'application/octet-stream' -] - -/** - * if the user has specified an `Accept` header, and it's in our list of - * allowable "raw" format headers, use that instead of detecting the content - * type. This avoids the user from receiving something different when they - * signal that they want to `Accept` a specific mime type. - */ -function getOverridenRawContentType (headers?: HeadersInit): string | undefined { - const acceptHeader = new Headers(headers).get('accept') ?? '' - - // e.g. "Accept: text/html, application/xhtml+xml, application/xml;q=0.9, image/webp, */*;q=0.8" - const acceptHeaders = acceptHeader.split(',') - .map(s => s.split(';')[0]) - .map(s => s.trim()) - - for (const mimeType of acceptHeaders) { - if (mimeType === '*/*') { - return - } - - if (RAW_HEADERS.includes(mimeType ?? '')) { - return mimeType - } - } -} - -export class VerifiedFetch { - private readonly helia: Helia - private readonly ipns: IPNS - private readonly unixfs: HeliaUnixFs - private readonly log: Logger - private readonly contentTypeParser: ContentTypeParser | undefined - - constructor ({ helia, ipns, unixfs }: VerifiedFetchComponents, init?: VerifiedFetchInit) { - this.helia = helia - this.log = helia.logger.forComponent('helia:verified-fetch') - this.ipns = ipns ?? heliaIpns(helia, { - resolvers: init?.dnsResolvers ?? [ - dnsJsonOverHttps('https://mozilla.cloudflare-dns.com/dns-query'), - dnsJsonOverHttps('https://dns.google/resolve') - ] - }) - this.unixfs = unixfs ?? heliaUnixFs(helia) - this.contentTypeParser = init?.contentTypeParser - this.log.trace('created VerifiedFetch instance') - } - - /** - * Accepts an `ipns://...` URL as a string and returns a `Response` containing - * a raw IPNS record. - */ - private async handleIPNSRecord ({ resource, cid, path, options }: FetchHandlerFunctionArg): Promise { - if (path !== '' || !resource.startsWith('ipns://')) { - return badRequestResponse('Invalid IPNS name') - } - - let peerId: PeerId - - try { - peerId = peerIdFromString(resource.replace('ipns://', '')) - } catch (err) { - this.log.error('could not parse peer id from IPNS url %s', resource) - - return badRequestResponse('Invalid IPNS name') - } - - // since this call happens after parseResource, we've already resolved the - // IPNS name so a local copy should be in the helia datastore, so we can - // just read it out.. - const routingKey = uint8ArrayConcat([ - uint8ArrayFromString('/ipns/'), - peerId.toBytes() - ]) - const datastoreKey = new Key('/dht/record/' + uint8ArrayToString(routingKey, 'base32'), false) - const buf = await this.helia.datastore.get(datastoreKey, options) - const record = DHTRecord.deserialize(buf) - - const response = okResponse(record.value) - response.headers.set('content-type', 'application/vnd.ipfs.ipns-record') - - return response - } - - /** - * Accepts a `CID` and returns a `Response` with a body stream that is a CAR - * of the `DAG` referenced by the `CID`. - */ - private async handleCar ({ cid, options }: FetchHandlerFunctionArg): Promise { - const c = car(this.helia) - const stream = toBrowserReadableStream(c.stream(cid, options)) - - const response = okResponse(stream) - response.headers.set('content-type', 'application/vnd.ipld.car; version=1') - - return response - } - - /** - * Accepts a UnixFS `CID` and returns a `.tar` file containing the file or - * directory structure referenced by the `CID`. - */ - private async handleTar ({ cid, path, options }: FetchHandlerFunctionArg): Promise { - if (cid.code !== dagPbCode && cid.code !== rawCode) { - return notAcceptableResponse('only UnixFS data can be returned in a TAR file') - } - - const stream = toBrowserReadableStream(tarStream(`/ipfs/${cid}/${path}`, this.helia.blockstore, options)) - - const response = okResponse(stream) - response.headers.set('content-type', 'application/x-tar') - - return response - } - - private async handleJson ({ cid, path, accept, options }: FetchHandlerFunctionArg): Promise { - this.log.trace('fetching %c/%s', cid, path) - const block = await this.helia.blockstore.get(cid, options) - let body: string | Uint8Array - - if (accept === 'application/vnd.ipld.dag-cbor' || accept === 'application/cbor') { - try { - // if vnd.ipld.dag-cbor has been specified, convert to the format - note - // that this supports more data types than regular JSON, the content-type - // response header is set so the user knows to process it differently - const obj = ipldDagJson.decode(block) - body = ipldDagCbor.encode(obj) - } catch (err) { - this.log.error('could not transform %c to application/vnd.ipld.dag-cbor', err) - return notAcceptableResponse() - } - } else { - // skip decoding - body = block - } - - const response = okResponse(body) - response.headers.set('content-type', accept ?? 'application/json') - return response - } - - private async handleDagCbor ({ cid, path, accept, options }: FetchHandlerFunctionArg): Promise { - this.log.trace('fetching %c/%s', cid, path) - - const block = await this.helia.blockstore.get(cid, options) - let body: string | Uint8Array - - if (accept === 'application/octet-stream' || accept === 'application/vnd.ipld.dag-cbor' || accept === 'application/cbor') { - // skip decoding - body = block - } else if (accept === 'application/vnd.ipld.dag-json') { - try { - // if vnd.ipld.dag-json has been specified, convert to the format - note - // that this supports more data types than regular JSON, the content-type - // response header is set so the user knows to process it differently - const obj = ipldDagCbor.decode(block) - body = ipldDagJson.encode(obj) - } catch (err) { - this.log.error('could not transform %c to application/vnd.ipld.dag-json', err) - return notAcceptableResponse() - } - } else { - try { - body = dagCborToSafeJSON(block) - } catch (err) { - if (accept === 'application/json') { - this.log('could not decode DAG-CBOR as JSON-safe, but the client sent "Accept: application/json"', err) - - return notAcceptableResponse() - } - - this.log('could not decode DAG-CBOR as JSON-safe, falling back to `application/octet-stream`', err) - body = block - } - } - - const response = okResponse(body) - - if (accept == null) { - accept = body instanceof Uint8Array ? 'application/octet-stream' : 'application/json' - } - - response.headers.set('content-type', accept) - - return response - } - - private async handleDagPb ({ cid, path, options }: FetchHandlerFunctionArg): Promise { - let terminalElement: UnixFSEntry | undefined - let ipfsRoots: CID[] | undefined - - try { - const pathDetails = await walkPath(this.helia.blockstore, `${cid.toString()}/${path}`, options) - ipfsRoots = pathDetails.ipfsRoots - terminalElement = pathDetails.terminalElement - } catch (err) { - this.log.error('Error walking path %s', path, err) - } - - let resolvedCID = terminalElement?.cid ?? cid - let stat: UnixFSStats - if (terminalElement?.type === 'directory') { - const dirCid = terminalElement.cid - - const rootFilePath = 'index.html' - try { - this.log.trace('found directory at %c/%s, looking for index.html', cid, path) - stat = await this.unixfs.stat(dirCid, { - path: rootFilePath, - signal: options?.signal, - onProgress: options?.onProgress - }) - this.log.trace('found root file at %c/%s with cid %c', dirCid, rootFilePath, stat.cid) - path = rootFilePath - resolvedCID = stat.cid - // terminalElement = stat - } catch (err: any) { - this.log('error loading path %c/%s', dirCid, rootFilePath, err) - return notSupportedResponse('Unable to find index.html for directory at given path. Support for directories with implicit root is not implemented') - } finally { - options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid: dirCid, path: rootFilePath })) - } - } - - const asyncIter = this.unixfs.cat(resolvedCID, { - signal: options?.signal, - onProgress: options?.onProgress - }) - this.log('got async iterator for %c/%s', cid, path) - - const { stream, firstChunk } = await getStreamFromAsyncIterable(asyncIter, path ?? '', this.helia.logger, { - onProgress: options?.onProgress - }) - const response = okResponse(stream) - await this.setContentType(firstChunk, path, response) - - if (ipfsRoots != null) { - response.headers.set('X-Ipfs-Roots', ipfsRoots.map(cid => cid.toV1().toString()).join(',')) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-roots-response-header - } - - return response - } - - private async handleRaw ({ cid, path, options }: FetchHandlerFunctionArg): Promise { - const result = await this.helia.blockstore.get(cid, options) - const response = okResponse(result) - - // if the user has specified an `Accept` header that corresponds to a raw - // type, honour that header, so for example they don't request - // `application/vnd.ipld.raw` but get `application/octet-stream` - const overriddenContentType = getOverridenRawContentType(options?.headers) - if (overriddenContentType != null) { - response.headers.set('content-type', overriddenContentType) - } else { - await this.setContentType(result, path, response) - } - - return response - } - - private async setContentType (bytes: Uint8Array, path: string, response: Response): Promise { - let contentType = 'application/octet-stream' - - if (this.contentTypeParser != null) { - try { - let fileName = path.split('/').pop()?.trim() - fileName = fileName === '' ? undefined : fileName - const parsed = this.contentTypeParser(bytes, fileName) - - if (isPromise(parsed)) { - const result = await parsed - - if (result != null) { - contentType = result - } - } else if (parsed != null) { - contentType = parsed - } - } catch (err) { - this.log.error('Error parsing content type', err) - } - } - - response.headers.set('content-type', contentType) - } - - /** - * If the user has not specified an Accept header or format query string arg, - * use the CID codec to choose an appropriate handler for the block data. - */ - private readonly codecHandlers: Record = { - [dagPbCode]: this.handleDagPb, - [ipldDagJson.code]: this.handleJson, - [jsonCode]: this.handleJson, - [ipldDagCbor.code]: this.handleDagCbor, - [rawCode]: this.handleRaw, - [identity.code]: this.handleRaw - } - - async fetch (resource: Resource, opts?: VerifiedFetchOptions): Promise { - this.log('fetch %s', resource) - - const options = convertOptions(opts) - - options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:start', { resource })) - - // resolve the CID/path from the requested resource - const { path, query, cid } = await parseResource(resource, { ipns: this.ipns, logger: this.helia.logger }, options) - - options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:resolve', { cid, path })) - - const requestHeaders = new Headers(options?.headers) - const incomingAcceptHeader = requestHeaders.get('accept') - - if (incomingAcceptHeader != null) { - this.log('incoming accept header "%s"', incomingAcceptHeader) - } - - const queryFormatMapping = queryFormatToAcceptHeader(query.format) - - if (query.format != null) { - this.log('incoming query format "%s", mapped to %s', query.format, queryFormatMapping) - } - - const acceptHeader = incomingAcceptHeader ?? queryFormatMapping - const accept = selectOutputType(cid, acceptHeader) - this.log('output type %s', accept) - - if (acceptHeader != null && accept == null) { - return notAcceptableResponse() - } - - let response: Response - let reqFormat: RequestFormatShorthand | undefined - - const handlerArgs = { resource: resource.toString(), cid, path, accept, options } - - if (accept === 'application/vnd.ipfs.ipns-record') { - // the user requested a raw IPNS record - reqFormat = 'ipns-record' - response = await this.handleIPNSRecord(handlerArgs) - } else if (accept === 'application/vnd.ipld.car') { - // the user requested a CAR file - reqFormat = 'car' - query.download = true - query.filename = query.filename ?? `${cid.toString()}.car` - response = await this.handleCar(handlerArgs) - } else if (accept === 'application/vnd.ipld.raw') { - // the user requested a raw block - reqFormat = 'raw' - query.download = true - query.filename = query.filename ?? `${cid.toString()}.bin` - response = await this.handleRaw(handlerArgs) - } else if (accept === 'application/x-tar') { - // the user requested a TAR file - reqFormat = 'tar' - query.download = true - query.filename = query.filename ?? `${cid.toString()}.tar` - response = await this.handleTar(handlerArgs) - } else { - // derive the handler from the CID type - const codecHandler = this.codecHandlers[cid.code] - - if (codecHandler == null) { - return notSupportedResponse(`Support for codec with code ${cid.code} is not yet implemented. Please open an issue at https://github.com/ipfs/helia/issues/new`) - } - - response = await codecHandler.call(this, handlerArgs) - } - - response.headers.set('etag', getETag({ cid, reqFormat, weak: false })) - response.headers.set('cache-control', 'public, max-age=29030400, immutable') - // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-path-response-header - response.headers.set('X-Ipfs-Path', resource.toString()) - - // set Content-Disposition header - let contentDisposition: string | undefined - - // force download if requested - if (query.download === true) { - contentDisposition = 'attachment' - } - - // override filename if requested - if (query.filename != null) { - if (contentDisposition == null) { - contentDisposition = 'inline' - } - - contentDisposition = `${contentDisposition}; ${getContentDispositionFilename(query.filename)}` - } - - if (contentDisposition != null) { - response.headers.set('Content-Disposition', contentDisposition) - } - - options?.onProgress?.(new CustomProgressEvent('verified-fetch:request:end', { cid, path })) - - return response - } - - /** - * Start the Helia instance - */ - async start (): Promise { - await this.helia.start() - } - - /** - * Shut down the Helia instance - */ - async stop (): Promise { - await this.helia.stop() - } -} - -function isPromise (p?: any): p is Promise { - return p?.then != null -} diff --git a/packages/verified-fetch/test/accept-header.spec.ts b/packages/verified-fetch/test/accept-header.spec.ts deleted file mode 100644 index a71d9882..00000000 --- a/packages/verified-fetch/test/accept-header.spec.ts +++ /dev/null @@ -1,249 +0,0 @@ -import { dagCbor } from '@helia/dag-cbor' -import { dagJson } from '@helia/dag-json' -import { ipns } from '@helia/ipns' -import * as ipldDagCbor from '@ipld/dag-cbor' -import * as ipldDagJson from '@ipld/dag-json' -import { stop } from '@libp2p/interface' -import { createEd25519PeerId } from '@libp2p/peer-id-factory' -import { expect } from 'aegir/chai' -import { marshal } from 'ipns' -import { VerifiedFetch } from '../src/verified-fetch.js' -import { createHelia } from './fixtures/create-offline-helia.js' -import type { Helia } from '@helia/interface' - -describe('accept header', () => { - let helia: Helia - let verifiedFetch: VerifiedFetch - - beforeEach(async () => { - helia = await createHelia() - verifiedFetch = new VerifiedFetch({ - helia - }) - }) - - afterEach(async () => { - await stop(helia, verifiedFetch) - }) - - it('should allow specifying application/vnd.ipld.raw accept header to skip data decoding', async () => { - // JSON-compliant CBOR - if decoded would otherwise cause `Content-Type` to - // be set to `application/json` - const obj = { - hello: 'world' - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const resp = await verifiedFetch.fetch(cid, { - headers: { - accept: 'application/vnd.ipld.raw' - } - }) - expect(resp.headers.get('content-type')).to.equal('application/vnd.ipld.raw') - - const output = ipldDagCbor.decode(new Uint8Array(await resp.arrayBuffer())) - expect(output).to.deep.equal(obj) - }) - - it('should allow specifying application/octet-stream accept header to skip data decoding', async () => { - // JSON-compliant CBOR - if decoded would otherwise cause `Content-Type` to - // be set to `application/json` - const obj = { - hello: 'world' - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const resp = await verifiedFetch.fetch(cid, { - headers: { - accept: 'application/octet-stream' - } - }) - expect(resp.headers.get('content-type')).to.equal('application/octet-stream') - - const output = ipldDagCbor.decode(new Uint8Array(await resp.arrayBuffer())) - expect(output).to.deep.equal(obj) - }) - - it('should transform DAG-CBOR to DAG-JSON', async () => { - const obj = { - hello: 'world' - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const resp = await verifiedFetch.fetch(cid, { - headers: { - accept: 'application/vnd.ipld.dag-json' - } - }) - expect(resp.headers.get('content-type')).to.equal('application/vnd.ipld.dag-json') - - const output = ipldDagJson.decode(new Uint8Array(await resp.arrayBuffer())) - expect(output).to.deep.equal(obj) - }) - - it('should transform DAG-CBOR to JSON', async () => { - const obj = { - hello: 'world' - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const resp = await verifiedFetch.fetch(cid, { - headers: { - accept: 'application/json' - } - }) - expect(resp.headers.get('content-type')).to.equal('application/json') - - const output = ipldDagJson.decode(new Uint8Array(await resp.arrayBuffer())) - expect(output).to.deep.equal(obj) - }) - - it('should transform DAG-JSON to DAG-CBOR', async () => { - const obj = { - hello: 'world' - } - const j = dagJson(helia) - const cid = await j.add(obj) - - const resp = await verifiedFetch.fetch(cid, { - headers: { - accept: 'application/vnd.ipld.dag-cbor' - } - }) - expect(resp.headers.get('content-type')).to.equal('application/vnd.ipld.dag-cbor') - - const output = ipldDagCbor.decode(new Uint8Array(await resp.arrayBuffer())) - expect(output).to.deep.equal(obj) - }) - - it('should transform DAG-JSON to CBOR', async () => { - const obj = { - hello: 'world' - } - const j = dagJson(helia) - const cid = await j.add(obj) - - const resp = await verifiedFetch.fetch(cid, { - headers: { - accept: 'application/cbor' - } - }) - expect(resp.headers.get('content-type')).to.equal('application/cbor') - - const output = ipldDagCbor.decode(new Uint8Array(await resp.arrayBuffer())) - expect(output).to.deep.equal(obj) - }) - - it('should return 406 Not Acceptable if the accept header cannot be adhered to', async () => { - const obj = { - hello: 'world' - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const resp = await verifiedFetch.fetch(cid, { - headers: { - accept: 'application/what-even-is-this' - } - }) - expect(resp.status).to.equal(406) - expect(resp.statusText).to.equal('Not Acceptable') - }) - - it('should support wildcards', async () => { - const obj = { - hello: 'world' - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const resp = await verifiedFetch.fetch(cid, { - headers: { - accept: 'application/what-even-is-this, */*, application/vnd.ipld.raw' - } - }) - expect(resp.status).to.equal(200) - expect(resp.headers.get('content-type')).to.equal('application/json') - }) - - it('should support type wildcards', async () => { - const obj = { - hello: 'world' - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const resp = await verifiedFetch.fetch(cid, { - headers: { - accept: '*/json, application/vnd.ipld.raw' - } - }) - expect(resp.status).to.equal(200) - expect(resp.headers.get('content-type')).to.equal('application/json') - }) - - it('should support subtype wildcards', async () => { - const obj = { - hello: 'world' - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const resp = await verifiedFetch.fetch(cid, { - headers: { - accept: 'application/*, application/vnd.ipld.raw' - } - }) - expect(resp.status).to.equal(200) - expect(resp.headers.get('content-type')).to.equal('application/json') - }) - - it('should support q-factor weighting', async () => { - const obj = { - hello: 'world' - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const resp = await verifiedFetch.fetch(cid, { - headers: { - // these all match, application/json would be chosen as it is first but - // application/octet-stream has a higher weighting so it should win - accept: [ - 'application/json;q=0.1', - 'application/application/vnd.ipld.raw;q=0.5', - 'application/octet-stream;q=0.8' - ].join(', ') - } - }) - expect(resp.status).to.equal(200) - expect(resp.headers.get('content-type')).to.equal('application/octet-stream') - }) - - it.skip('should support fetching IPNS records', async () => { - const peerId = await createEd25519PeerId() - const obj = { - hello: 'world' - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const i = ipns(helia) - const record = await i.publish(peerId, cid) - - const resp = await verifiedFetch.fetch(`ipns://${peerId}`, { - headers: { - accept: 'application/vnd.ipfs.ipns-record' - } - }) - expect(resp.status).to.equal(200) - expect(resp.headers.get('content-type')).to.equal('application/vnd.ipfs.ipns-record') - const buf = await resp.arrayBuffer() - - expect(buf).to.equalBytes(marshal(record)) - }) -}) diff --git a/packages/verified-fetch/test/car.spec.ts b/packages/verified-fetch/test/car.spec.ts deleted file mode 100644 index 01937dbf..00000000 --- a/packages/verified-fetch/test/car.spec.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { car } from '@helia/car' -import { dagCbor } from '@helia/dag-cbor' -import { stop } from '@libp2p/interface' -import { expect } from 'aegir/chai' -import { VerifiedFetch } from '../src/verified-fetch.js' -import { createHelia } from './fixtures/create-offline-helia.js' -import { memoryCarWriter } from './fixtures/memory-car.js' -import type { Helia } from '@helia/interface' - -describe('car files', () => { - let helia: Helia - let verifiedFetch: VerifiedFetch - - beforeEach(async () => { - helia = await createHelia() - verifiedFetch = new VerifiedFetch({ - helia - }) - }) - - afterEach(async () => { - await stop(helia, verifiedFetch) - }) - - it('should support fetching a CAR file', async () => { - const obj = { - hello: 'world' - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const ca = car(helia) - const writer = memoryCarWriter(cid) - await ca.export(cid, writer) - - const resp = await verifiedFetch.fetch(cid, { - headers: { - accept: 'application/vnd.ipld.car' - } - }) - expect(resp.status).to.equal(200) - expect(resp.headers.get('content-type')).to.equal('application/vnd.ipld.car; version=1') - expect(resp.headers.get('content-disposition')).to.equal(`attachment; filename="${cid.toString()}.car"`) - const buf = new Uint8Array(await resp.arrayBuffer()) - - expect(buf).to.equalBytes(await writer.bytes()) - }) - - it('should support specifying a filename for a CAR file', async () => { - const obj = { - hello: 'world' - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const ca = car(helia) - const writer = memoryCarWriter(cid) - await ca.export(cid, writer) - - const resp = await verifiedFetch.fetch(`ipfs://${cid}?filename=foo.bar`, { - headers: { - accept: 'application/vnd.ipld.car' - } - }) - expect(resp.status).to.equal(200) - expect(resp.headers.get('content-type')).to.equal('application/vnd.ipld.car; version=1') - expect(resp.headers.get('content-disposition')).to.equal('attachment; filename="foo.bar"') - }) -}) diff --git a/packages/verified-fetch/test/content-type-parser.spec.ts b/packages/verified-fetch/test/content-type-parser.spec.ts deleted file mode 100644 index b614529b..00000000 --- a/packages/verified-fetch/test/content-type-parser.spec.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { createHeliaHTTP } from '@helia/http' -import { unixfs } from '@helia/unixfs' -import { stop } from '@libp2p/interface' -import { fileTypeFromBuffer } from '@sgtpooki/file-type' -import { expect } from 'aegir/chai' -import { filetypemime } from 'magic-bytes.js' -import Sinon from 'sinon' -import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { createVerifiedFetch } from '../src/index.js' -import { VerifiedFetch } from '../src/verified-fetch.js' -import type { Helia } from '@helia/interface' -import type { CID } from 'multiformats/cid' - -describe('content-type-parser', () => { - let helia: Helia - let cid: CID - let verifiedFetch: VerifiedFetch - - beforeEach(async () => { - helia = await createHeliaHTTP() - const fs = unixfs(helia) - cid = await fs.addByteStream((async function * () { - yield uint8ArrayFromString('H4sICIlTHVIACw', 'base64') - })()) - }) - - afterEach(async () => { - await stop(verifiedFetch) - }) - - it('is used when passed to createVerifiedFetch', async () => { - const contentTypeParser = Sinon.stub().resolves('text/plain') - const fetch = await createVerifiedFetch(helia, { - contentTypeParser - }) - expect(fetch).to.be.ok() - const resp = await fetch(cid) - expect(resp.headers.get('content-type')).to.equal('text/plain') - await fetch.stop() - }) - - it('sets default content type if contentTypeParser is not passed', async () => { - verifiedFetch = new VerifiedFetch({ - helia - }) - const resp = await verifiedFetch.fetch(cid) - expect(resp.headers.get('content-type')).to.equal('application/octet-stream') - }) - - it('sets default content type if contentTypeParser returns undefined', async () => { - verifiedFetch = new VerifiedFetch({ - helia - }, { - contentTypeParser: () => undefined - }) - const resp = await verifiedFetch.fetch(cid) - expect(resp.headers.get('content-type')).to.equal('application/octet-stream') - }) - - it('sets default content type if contentTypeParser returns promise of undefined', async () => { - verifiedFetch = new VerifiedFetch({ - helia - }, { - contentTypeParser: async () => undefined - }) - const resp = await verifiedFetch.fetch(cid) - expect(resp.headers.get('content-type')).to.equal('application/octet-stream') - }) - - it('is passed a filename if it is available', async () => { - const fs = unixfs(helia) - const dir = await fs.addDirectory() - const index = await fs.addBytes(uint8ArrayFromString('Hello world')) - const dirCid = await fs.cp(index, dir, 'index.html') - - verifiedFetch = new VerifiedFetch({ - helia - }, { - contentTypeParser: async (data, fileName) => fileName - }) - const resp = await verifiedFetch.fetch(`ipfs://${dirCid}/index.html`) - expect(resp.headers.get('content-type')).to.equal('index.html') - }) - - it('is passed a filename from a deep traversal if it is available', async () => { - const fs = unixfs(helia) - const deepDirCid = await fs.addFile({ - path: 'foo/bar/a-file.html', - content: uint8ArrayFromString('Hello world') - }) - - verifiedFetch = new VerifiedFetch({ - helia - }, { - contentTypeParser: async (data, fileName) => fileName - }) - const resp = await verifiedFetch.fetch(`ipfs://${deepDirCid}/foo/bar/a-file.html`) - expect(resp.headers.get('content-type')).to.equal('a-file.html') - }) - - it('sets content type if contentTypeParser is passed', async () => { - verifiedFetch = new VerifiedFetch({ - helia - }, { - contentTypeParser: () => 'text/plain' - }) - const resp = await verifiedFetch.fetch(cid) - expect(resp.headers.get('content-type')).to.equal('text/plain') - }) - - it('supports @sgtpooki/file-type as a contentTypeParser', async () => { - verifiedFetch = new VerifiedFetch({ - helia - }, { - contentTypeParser: async (bytes) => { - const type = await fileTypeFromBuffer(bytes) - return type?.mime - } - }) - const resp = await verifiedFetch.fetch(cid) - expect(resp.headers.get('content-type')).to.equal('application/gzip') - }) - - it('supports magic-bytes.js as a contentTypeParser', async () => { - verifiedFetch = new VerifiedFetch({ - helia - }, { - contentTypeParser: (bytes) => { - return filetypemime(bytes)?.[0] - } - }) - const resp = await verifiedFetch.fetch(cid) - expect(resp.headers.get('content-type')).to.equal('application/gzip') - }) -}) diff --git a/packages/verified-fetch/test/custom-dns-resolvers.spec.ts b/packages/verified-fetch/test/custom-dns-resolvers.spec.ts deleted file mode 100644 index a37c5292..00000000 --- a/packages/verified-fetch/test/custom-dns-resolvers.spec.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { stop } from '@libp2p/interface' -import { expect } from 'aegir/chai' -import Sinon from 'sinon' -import { createVerifiedFetch } from '../src/index.js' -import { VerifiedFetch } from '../src/verified-fetch.js' -import { createHelia } from './fixtures/create-offline-helia.js' -import type { Helia } from '@helia/interface' - -describe('custom dns-resolvers', () => { - let helia: Helia - - beforeEach(async () => { - helia = await createHelia() - }) - - afterEach(async () => { - await stop(helia) - }) - - it('is used when passed to createVerifiedFetch', async () => { - const customDnsResolver = Sinon.stub() - - customDnsResolver.returns(Promise.resolve('/ipfs/QmVP2ip92jQuMDezVSzQBWDqWFbp9nyCHNQSiciRauPLDg')) - - const fetch = await createVerifiedFetch({ - gateways: ['http://127.0.0.1:8080'], - dnsResolvers: [customDnsResolver] - }) - // error of walking the CID/dag because we haven't actually added the block to the blockstore - await expect(fetch('ipns://some-non-cached-domain.com')).to.eventually.be.rejected.with.property('errors') - - expect(customDnsResolver.callCount).to.equal(1) - expect(customDnsResolver.getCall(0).args).to.deep.equal(['some-non-cached-domain.com', { onProgress: undefined }]) - }) - - it('is used when passed to VerifiedFetch', async () => { - const customDnsResolver = Sinon.stub() - - customDnsResolver.returns(Promise.resolve('/ipfs/QmVP2ip92jQuMDezVSzQBWDqWFbp9nyCHNQSiciRauPLDg')) - - const verifiedFetch = new VerifiedFetch({ - helia - }, { - dnsResolvers: [customDnsResolver] - }) - // error of walking the CID/dag because we haven't actually added the block to the blockstore - await expect(verifiedFetch.fetch('ipns://some-non-cached-domain2.com')).to.eventually.be.rejected.with.property('errors').that.has.lengthOf(0) - - expect(customDnsResolver.callCount).to.equal(1) - expect(customDnsResolver.getCall(0).args).to.deep.equal(['some-non-cached-domain2.com', { onProgress: undefined }]) - }) -}) diff --git a/packages/verified-fetch/test/fixtures/cids.ts b/packages/verified-fetch/test/fixtures/cids.ts deleted file mode 100644 index 03ab9f0c..00000000 --- a/packages/verified-fetch/test/fixtures/cids.ts +++ /dev/null @@ -1,18 +0,0 @@ -import * as dagCbor from '@ipld/dag-cbor' -import * as dagJson from '@ipld/dag-json' -import * as dagPb from '@ipld/dag-pb' -import { CID } from 'multiformats/cid' -import * as json from 'multiformats/codecs/json' -import * as raw from 'multiformats/codecs/raw' - -// 112 = dag-pb, 18 = sha256, 0 = CIDv0 -const mh = CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr').multihash - -export const cids: Record = { - filev0: CID.createV0(mh), - file: CID.createV1(dagPb.code, mh), - dagCbor: CID.createV1(dagCbor.code, mh), - dagJson: CID.createV1(dagJson.code, mh), - json: CID.createV1(json.code, mh), - raw: CID.createV1(raw.code, mh) -} diff --git a/packages/verified-fetch/test/fixtures/create-offline-helia.ts b/packages/verified-fetch/test/fixtures/create-offline-helia.ts deleted file mode 100644 index 2c6ab4af..00000000 --- a/packages/verified-fetch/test/fixtures/create-offline-helia.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { Helia as HeliaClass } from '@helia/utils' -import { MemoryBlockstore } from 'blockstore-core' -import { MemoryDatastore } from 'datastore-core' -import type { Helia } from '@helia/interface' - -export async function createHelia (): Promise { - const datastore = new MemoryDatastore() - const blockstore = new MemoryBlockstore() - - const helia = new HeliaClass({ - datastore, - blockstore, - blockBrokers: [], - routers: [] - }) - - await helia.start() - - return helia -} diff --git a/packages/verified-fetch/test/fixtures/memory-car.ts b/packages/verified-fetch/test/fixtures/memory-car.ts deleted file mode 100644 index ad210cb3..00000000 --- a/packages/verified-fetch/test/fixtures/memory-car.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { CarWriter } from '@ipld/car' -import toBuffer from 'it-to-buffer' -import defer from 'p-defer' -import type { CID } from 'multiformats/cid' - -export interface MemoryCar extends Pick { - bytes(): Promise -} - -export function memoryCarWriter (root: CID | CID[]): MemoryCar { - const deferred = defer() - const { writer, out } = CarWriter.create(Array.isArray(root) ? root : [root]) - - Promise.resolve() - .then(async () => { - deferred.resolve(await toBuffer(out)) - }) - .catch(err => { - deferred.reject(err) - }) - - return { - async put (block: { cid: CID, bytes: Uint8Array }): Promise { - await writer.put(block) - }, - async close (): Promise { - await writer.close() - }, - async bytes (): Promise { - return deferred.promise - } - } -} diff --git a/packages/verified-fetch/test/get-e-tag.spec.ts b/packages/verified-fetch/test/get-e-tag.spec.ts deleted file mode 100644 index a0908d58..00000000 --- a/packages/verified-fetch/test/get-e-tag.spec.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { expect } from 'aegir/chai' -import { CID } from 'multiformats/cid' -import { getETag } from '../src/utils/get-e-tag.js' - -const cidString = 'QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr' -const testCID = CID.parse(cidString) - -describe('getETag', () => { - it('CID eTag', () => { - expect(getETag({ cid: testCID, weak: true })).to.equal(`W/"${cidString}"`) - expect(getETag({ cid: testCID, weak: false })).to.equal(`"${cidString}"`) - }) - - it('should return ETag with CID and format suffix', () => { - expect(getETag({ cid: testCID, reqFormat: 'raw' })).to.equal(`"${cidString}.raw"`) - expect(getETag({ cid: testCID, reqFormat: 'json' })).to.equal(`"${cidString}.json"`) - }) - - it('should return ETag with CID and range suffix', () => { - expect(getETag({ cid: testCID, weak: true, reqFormat: 'car', rangeStart: 10, rangeEnd: 20 })).to.equal(`W/"${cidString}.car.10-20"`) - expect(getETag({ cid: testCID, weak: false, reqFormat: 'car', rangeStart: 10, rangeEnd: 20 })).to.equal(`"${cidString}.car.10-20"`) - }) - - it('should return ETag with CID, format and range suffix', () => { - expect(getETag({ cid: testCID, reqFormat: 'raw', weak: false, rangeStart: 10, rangeEnd: 20 })).to.equal(`"${cidString}.raw.10-20"`) - }) - - it('should handle undefined rangeStart and rangeEnd', () => { - expect(getETag({ cid: testCID, reqFormat: 'raw', weak: false, rangeStart: undefined, rangeEnd: undefined })).to.equal(`"${cidString}.raw"`) - expect(getETag({ cid: testCID, reqFormat: 'raw', weak: false, rangeStart: 55, rangeEnd: undefined })).to.equal(`"${cidString}.raw.55-N"`) - expect(getETag({ cid: testCID, reqFormat: 'raw', weak: false, rangeStart: undefined, rangeEnd: 77 })).to.equal(`"${cidString}.raw.0-77"`) - }) -}) diff --git a/packages/verified-fetch/test/get-stream-from-async-iterable.spec.ts b/packages/verified-fetch/test/get-stream-from-async-iterable.spec.ts deleted file mode 100644 index 1d8e6c49..00000000 --- a/packages/verified-fetch/test/get-stream-from-async-iterable.spec.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { defaultLogger } from '@libp2p/logger' -import { expect } from 'aegir/chai' -import sinon from 'sinon' -import { getStreamFromAsyncIterable } from '../src/utils/get-stream-from-async-iterable.js' - -describe('getStreamFromAsyncIterable', () => { - let onProgressSpy: sinon.SinonSpy - - beforeEach(() => { - onProgressSpy = sinon.spy() - }) - - it('should throw an error if no content is found', async () => { - const iterator = (async function * () { })() - await expect(getStreamFromAsyncIterable(iterator, 'test', defaultLogger())).to.be.rejectedWith('No content found') - }) - - it('should return the correct content type and a readable stream', async () => { - const chunks = new TextEncoder().encode('Hello, world!') - const iterator = (async function * () { yield chunks })() - const { firstChunk, stream } = await getStreamFromAsyncIterable(iterator, 'test.txt', defaultLogger(), { onProgress: onProgressSpy }) - expect(firstChunk).to.equal(chunks) - const reader = stream.getReader() - const { value } = await reader.read() - expect(onProgressSpy.callCount).to.equal(1) - expect(new TextDecoder().decode(value)).to.equal('Hello, world!') - }) - - it('should handle multiple chunks of data', async () => { - const textEncoder = new TextEncoder() - const chunks = ['Hello,', ' world!'].map((txt) => textEncoder.encode(txt)) - const iterator = (async function * () { yield chunks[0]; yield chunks[1] })() - const { firstChunk, stream } = await getStreamFromAsyncIterable(iterator, 'test.txt', defaultLogger(), { onProgress: onProgressSpy }) - expect(firstChunk).to.equal(chunks[0]) - const reader = stream.getReader() - let result = '' - let chunk - while (!(chunk = await reader.read()).done) { - result += new TextDecoder().decode(chunk.value) - } - expect(onProgressSpy.callCount).to.equal(2) - expect(result).to.equal('Hello, world!') - }) - - it('should include last value done is true', async () => { - // if done === true and there is a value - const LIMIT = 5 - let actualFirstChunk: Uint8Array - const iterator: AsyncIterable = { - [Symbol.asyncIterator] () { - let i = 0 - return { - async next () { - const done = i === LIMIT - const value = new Uint8Array([i++]) - actualFirstChunk = actualFirstChunk ?? value - return Promise.resolve({ value, done }) - } - } - } - } - const { firstChunk, stream } = await getStreamFromAsyncIterable(iterator, 'test.txt', defaultLogger(), { onProgress: onProgressSpy }) - // @ts-expect-error - actualFirstChunk is not used before set, because the await above. - expect(firstChunk).to.equal(actualFirstChunk) - const reader = stream.getReader() - const result = [] - let chunk - while (!(chunk = await reader.read()).done) { - result.push(...chunk.value) - } - expect(onProgressSpy.callCount).to.equal(6) - expect(result).to.deep.equal([...Array(LIMIT + 1).keys()]) - }) -}) diff --git a/packages/verified-fetch/test/index.spec.ts b/packages/verified-fetch/test/index.spec.ts deleted file mode 100644 index 90cd1054..00000000 --- a/packages/verified-fetch/test/index.spec.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { createHeliaHTTP } from '@helia/http' -import { expect } from 'aegir/chai' -import { createHelia } from 'helia' -import { createVerifiedFetch, verifiedFetch } from '../src/index.js' - -describe('createVerifiedFetch', () => { - it('can be constructed with a HeliaHttp instance', async () => { - const heliaHttp = await createHeliaHTTP() - const verifiedFetch = await createVerifiedFetch(heliaHttp) - - expect(verifiedFetch).to.be.ok() - await verifiedFetch.stop() - }) - - it('can be constructed with a HeliaP2P instance', async () => { - const heliaP2P = await createHelia() - const verifiedFetch = await createVerifiedFetch(heliaP2P) - - expect(verifiedFetch).to.be.ok() - await heliaP2P.stop() - await verifiedFetch.stop() - }) - - it('can be constructed with gateways', async () => { - const verifiedFetch = await createVerifiedFetch({ - gateways: ['https://127.0.0.1'] - }) - expect(verifiedFetch).to.be.ok() - await verifiedFetch.stop() - }) - - it('can be constructed with gateways & routers', async () => { - const verifiedFetch = await createVerifiedFetch({ - gateways: ['https://127.0.0.1'], - routers: ['https://127.0.0.1'] - }) - expect(verifiedFetch).to.be.ok() - await verifiedFetch.stop() - }) - - it('can be constructed with no options', async () => { - const verifiedFetch = await createVerifiedFetch() - - expect(verifiedFetch).to.be.ok() - await verifiedFetch.stop() - }) - - it('can be used as a singleton', () => { - expect(verifiedFetch).to.be.a('function') - expect(verifiedFetch.stop).to.be.a('function') - expect(verifiedFetch.start).to.be.a('function') - }) -}) diff --git a/packages/verified-fetch/test/ipns-record.spec.ts b/packages/verified-fetch/test/ipns-record.spec.ts deleted file mode 100644 index a101d7ca..00000000 --- a/packages/verified-fetch/test/ipns-record.spec.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { dagCbor } from '@helia/dag-cbor' -import { ipns } from '@helia/ipns' -import { stop } from '@libp2p/interface' -import { createEd25519PeerId } from '@libp2p/peer-id-factory' -import { expect } from 'aegir/chai' -import { marshal, unmarshal } from 'ipns' -import { VerifiedFetch } from '../src/verified-fetch.js' -import { createHelia } from './fixtures/create-offline-helia.js' -import type { Helia } from '@helia/interface' -import type { IPNS } from '@helia/ipns' - -describe('ipns records', () => { - let helia: Helia - let name: IPNS - let verifiedFetch: VerifiedFetch - - beforeEach(async () => { - helia = await createHelia() - name = ipns(helia) - verifiedFetch = new VerifiedFetch({ - helia - }) - }) - - afterEach(async () => { - await stop(helia, verifiedFetch) - }) - - it('should support fetching a raw IPNS record', async () => { - const obj = { - hello: 'world' - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const peerId = await createEd25519PeerId() - const record = await name.publish(peerId, cid) - - const resp = await verifiedFetch.fetch(`ipns://${peerId}`, { - headers: { - accept: 'application/vnd.ipfs.ipns-record' - } - }) - expect(resp.status).to.equal(200) - expect(resp.headers.get('content-type')).to.equal('application/vnd.ipfs.ipns-record') - - const buf = new Uint8Array(await resp.arrayBuffer()) - expect(marshal(record)).to.equalBytes(buf) - - const output = unmarshal(buf) - expect(output.value).to.deep.equal(`/ipfs/${cid}`) - }) - - it('should reject a request for non-IPNS url', async () => { - const resp = await verifiedFetch.fetch('ipfs://QmbxpRxwKXxnJQjnPqm1kzDJSJ8YgkLxH23mcZURwPHjGv', { - headers: { - accept: 'application/vnd.ipfs.ipns-record' - } - }) - expect(resp.status).to.equal(400) - }) - - it('should reject a request for a DNSLink url', async () => { - const resp = await verifiedFetch.fetch('ipns://ipfs.io', { - headers: { - accept: 'application/vnd.ipfs.ipns-record' - } - }) - expect(resp.status).to.equal(400) - }) - - it('should reject a request for a url with a path component', async () => { - const obj = { - hello: 'world' - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const peerId = await createEd25519PeerId() - await name.publish(peerId, cid) - - const resp = await verifiedFetch.fetch(`ipns://${peerId}/hello`, { - headers: { - accept: 'application/vnd.ipfs.ipns-record' - } - }) - expect(resp.status).to.equal(400) - }) -}) diff --git a/packages/verified-fetch/test/parse-resource.spec.ts b/packages/verified-fetch/test/parse-resource.spec.ts deleted file mode 100644 index 6e59817f..00000000 --- a/packages/verified-fetch/test/parse-resource.spec.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { defaultLogger } from '@libp2p/logger' -import { expect } from 'aegir/chai' -import { CID } from 'multiformats/cid' -import sinon from 'sinon' -import { stubInterface } from 'sinon-ts' -import { parseResource } from '../src/utils/parse-resource.js' -import type { IPNS } from '@helia/ipns' - -describe('parseResource', () => { - it('does not call @helia/ipns for CID', async () => { - const testCID = CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr') - const shouldNotBeCalled1 = sinon.stub().throws(new Error('should not be called')) - const shouldNotBeCalled2 = sinon.stub().throws(new Error('should not be called')) - const { cid, path, query } = await parseResource(testCID, { - ipns: stubInterface({ - resolveDns: shouldNotBeCalled1, - resolve: shouldNotBeCalled2 - }), - logger: defaultLogger() - }) - expect(shouldNotBeCalled1.called).to.be.false() - expect(shouldNotBeCalled2.called).to.be.false() - expect(cid.toString()).to.equal(testCID.toString()) - expect(path).to.equal('') - expect(query).to.deep.equal({}) - }) - - it('throws an error if given an invalid resource', async () => { - // @ts-expect-error - purposefully invalid input - await expect(parseResource({}, stubInterface())).to.be.rejectedWith('Invalid resource.') - }) -}) diff --git a/packages/verified-fetch/test/parse-url-string.spec.ts b/packages/verified-fetch/test/parse-url-string.spec.ts deleted file mode 100644 index 6c56f112..00000000 --- a/packages/verified-fetch/test/parse-url-string.spec.ts +++ /dev/null @@ -1,323 +0,0 @@ -import { type PeerId } from '@libp2p/interface' -import { defaultLogger } from '@libp2p/logger' -import { createEd25519PeerId } from '@libp2p/peer-id-factory' -import { expect } from 'aegir/chai' -import { CID } from 'multiformats/cid' -import { stubInterface } from 'sinon-ts' -import { parseUrlString } from '../src/utils/parse-url-string.js' -import type { IPNS } from '@helia/ipns' - -describe('parseUrlString', () => { - describe('invalid URLs', () => { - it('throws for invalid URLs', async () => { - const ipns = stubInterface({}) - try { - await parseUrlString({ - urlString: 'invalid', - ipns, - logger: defaultLogger() - }) - throw new Error('Should have thrown') - } catch (err) { - expect((err as Error).message).to.equal('Invalid URL: invalid, please use ipfs:// or ipns:// URLs only.') - } - }) - - it('throws for invalid protocols', async () => { - const ipns = stubInterface({}) - try { - await parseUrlString({ - urlString: 'http://mydomain.com', - ipns, - logger: defaultLogger() - }) - throw new Error('Should have thrown') - } catch (err) { - expect((err as Error).message).to.equal('Invalid URL: http://mydomain.com, please use ipfs:// or ipns:// URLs only.') - } - }) - - it('throws an error if resulting CID is invalid', async () => { - const ipns = stubInterface({ - // @ts-expect-error - purposefully invalid response - resolveDns: async (_: string) => { - return null - } - }) - try { - await parseUrlString({ - urlString: 'ipns://mydomain.com', - ipns, - logger: defaultLogger() - }) - throw new Error('Should have thrown') - } catch (err) { - expect((err as Error).message).to.equal('Invalid resource. Cannot determine CID from URL "ipns://mydomain.com"') - } - }) - }) - - describe('ipfs:// URLs', () => { - it('handles invalid CIDs', async () => { - const ipns = stubInterface({}) - try { - await parseUrlString({ - urlString: 'ipfs://QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4i', - ipns, - logger: defaultLogger() - }) - throw new Error('Should have thrown') - } catch (aggErr) { - expect(aggErr).to.have.property('message', 'Invalid resource. Cannot determine CID from URL "ipfs://QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4i"') - expect(aggErr).to.have.property('errors').with.lengthOf(1).that.deep.equals([ - new TypeError('Invalid CID for ipfs:// URL') - ]) - } - }) - - it('can parse a URL with CID only', async () => { - const ipns = stubInterface({}) - const result = await parseUrlString({ - urlString: 'ipfs://QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr', - ipns, - logger: defaultLogger() - }) - expect(result.protocol).to.equal('ipfs') - expect(result.cid.toString()).to.equal('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr') - expect(result.path).to.equal('') - }) - - it('can parse URL with CID+path', async () => { - const ipns = stubInterface({}) - const result = await parseUrlString({ - urlString: 'ipfs://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt', - ipns, - logger: defaultLogger() - }) - expect(result.protocol).to.equal('ipfs') - expect(result.cid.toString()).to.equal('QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm') - expect(result.path).to.equal('1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt') - }) - - it('can parse URL with CID+queryString', async () => { - const ipns = stubInterface({}) - const result = await parseUrlString({ - urlString: 'ipfs://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm?format=car', - ipns, - logger: defaultLogger() - }) - expect(result.protocol).to.equal('ipfs') - expect(result.cid.toString()).to.equal('QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm') - expect(result.path).to.equal('') - expect(result.query).to.deep.equal({ format: 'car' }) - }) - - it('can parse URL with CID+path+queryString', async () => { - const ipns = stubInterface({}) - const result = await parseUrlString({ - urlString: 'ipfs://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt?format=tar', - ipns, - logger: defaultLogger() - }) - expect(result.protocol).to.equal('ipfs') - expect(result.cid.toString()).to.equal('QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm') - expect(result.path).to.equal('1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt') - expect(result.query).to.deep.equal({ format: 'tar' }) - }) - }) - - describe('ipns:// URLs', () => { - let ipns: IPNS - - beforeEach(async () => { - ipns = stubInterface({ - resolveDns: async (dnsLink: string) => { - expect(dnsLink).to.equal('mydomain.com') - return { - cid: CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), - path: '' - } - } - }) - }) - - it('handles invalid DNSLinkDomains', async () => { - ipns = stubInterface({ - resolve: async (peerId: PeerId) => { - throw new Error('Unexpected failure from ipns resolve method') - }, - resolveDns: async (_: string) => { - return Promise.reject(new Error('Unexpected failure from dns query')) - } - }) - - try { - await parseUrlString({ urlString: 'ipns://mydomain.com', ipns, logger: defaultLogger() }) - throw new Error('Should have thrown') - } catch (aggErr) { - expect(aggErr).to.have.property('message', 'Invalid resource. Cannot determine CID from URL "ipns://mydomain.com"') - expect(aggErr).to.have.property('errors').with.lengthOf(2).that.deep.equals([ - new TypeError('Could not parse PeerId in ipns url "mydomain.com", Non-base64 character'), - new Error('Unexpected failure from dns query') - ]) - } - }) - - it('can parse a URL with DNSLinkDomain only', async () => { - const result = await parseUrlString({ - urlString: 'ipns://mydomain.com', - ipns, - logger: defaultLogger() - }) - expect(result.protocol).to.equal('ipns') - expect(result.cid.toString()).to.equal('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr') - expect(result.path).to.equal('') - }) - - it('can parse a URL with DNSLinkDomain+path', async () => { - const result = await parseUrlString({ - urlString: 'ipns://mydomain.com/some/path/to/file.txt', - ipns, - logger: defaultLogger() - }) - expect(result.protocol).to.equal('ipns') - expect(result.cid.toString()).to.equal('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr') - expect(result.path).to.equal('some/path/to/file.txt') - }) - - it('can parse a URL with DNSLinkDomain+queryString', async () => { - const result = await parseUrlString({ - urlString: 'ipns://mydomain.com?format=json', - ipns, - logger: defaultLogger() - }) - expect(result.protocol).to.equal('ipns') - expect(result.cid.toString()).to.equal('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr') - expect(result.path).to.equal('') - expect(result.query).to.deep.equal({ format: 'json' }) - }) - - it('can parse a URL with DNSLinkDomain+path+queryString', async () => { - const result = await parseUrlString({ - urlString: 'ipns://mydomain.com/some/path/to/file.txt?format=json', - ipns, - logger: defaultLogger() - }) - expect(result.protocol).to.equal('ipns') - expect(result.cid.toString()).to.equal('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr') - expect(result.path).to.equal('some/path/to/file.txt') - expect(result.query).to.deep.equal({ format: 'json' }) - }) - }) - - describe('ipns:// URLs', () => { - let ipns: IPNS - let testPeerId: PeerId - - beforeEach(async () => { - testPeerId = await createEd25519PeerId() - ipns = stubInterface({ - resolve: async (peerId: PeerId) => { - expect(peerId.toString()).to.equal(testPeerId.toString()) - return { - cid: CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), - path: '' - } - } - }) - }) - - it('handles invalid PeerIds', async () => { - ipns = stubInterface({ - resolve: async (peerId: PeerId) => { - throw new Error('Unexpected failure from ipns resolve method') - }, - resolveDns: async (_: string) => { - return Promise.reject(new Error('Unexpected failure from dns query')) - } - }) - - try { - await parseUrlString({ urlString: 'ipns://123PeerIdIsFake456', ipns, logger: defaultLogger() }) - throw new Error('Should have thrown') - } catch (aggErr) { - expect(aggErr).to.have.property('message', 'Invalid resource. Cannot determine CID from URL "ipns://123PeerIdIsFake456"') - expect(aggErr).to.have.property('errors').with.lengthOf(2).that.deep.equals([ - new TypeError('Could not parse PeerId in ipns url "123PeerIdIsFake456", Non-base58btc character'), - new Error('Unexpected failure from dns query') - ]) - } - }) - - it('handles valid PeerId resolve failures', async () => { - ipns = stubInterface({ - resolve: async (_: PeerId) => { - return Promise.reject(new Error('Unexpected failure from ipns resolve method')) - }, - resolveDns: async (_: string) => { - return Promise.reject(new Error('Unexpected failure from dns query')) - } - }) - - // await expect(parseUrlString({ urlString: `ipns://${testPeerId.toString()}`, ipns })).to.eventually.be.rejected() - // .with.property('message', `Could not resolve PeerId "${testPeerId.toString()}", Unexpected failure from ipns resolve method`) - - try { - await parseUrlString({ urlString: `ipns://${testPeerId.toString()}`, ipns, logger: defaultLogger() }) - throw new Error('Should have thrown') - } catch (aggErr) { - expect(aggErr).to.have.property('message', `Invalid resource. Cannot determine CID from URL "ipns://${testPeerId.toString()}"`) - expect(aggErr).to.have.property('errors').with.lengthOf(2).that.deep.equals([ - new TypeError(`Could not resolve PeerId "${testPeerId.toString()}", Unexpected failure from ipns resolve method`), - new Error('Unexpected failure from dns query') - ]) - } - }) - - it('can parse a URL with PeerId only', async () => { - const result = await parseUrlString({ - urlString: `ipns://${testPeerId.toString()}`, - ipns, - logger: defaultLogger() - }) - expect(result.protocol).to.equal('ipns') - expect(result.cid.toString()).to.equal('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr') - expect(result.path).to.equal('') - }) - - it('can parse a URL with PeerId+path', async () => { - const result = await parseUrlString({ - urlString: `ipns://${testPeerId.toString()}/some/path/to/file.txt`, - ipns, - logger: defaultLogger() - }) - expect(result.protocol).to.equal('ipns') - expect(result.cid.toString()).to.equal('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr') - expect(result.path).to.equal('some/path/to/file.txt') - }) - - it('can parse a URL with PeerId+queryString', async () => { - const result = await parseUrlString({ - urlString: `ipns://${testPeerId.toString()}?fomat=dag-cbor`, - ipns, - logger: defaultLogger() - }) - expect(result.protocol).to.equal('ipns') - expect(result.cid.toString()).to.equal('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr') - expect(result.path).to.equal('') - expect(result.query).to.deep.equal({ fomat: 'dag-cbor' }) - }) - - it('can parse a URL with PeerId+path+queryString', async () => { - const result = await parseUrlString({ - urlString: `ipns://${testPeerId.toString()}/some/path/to/file.txt?fomat=dag-cbor`, - ipns, - logger: defaultLogger() - }) - expect(result.protocol).to.equal('ipns') - expect(result.cid.toString()).to.equal('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr') - expect(result.path).to.equal('some/path/to/file.txt') - expect(result.query).to.deep.equal({ fomat: 'dag-cbor' }) - }) - }) -}) diff --git a/packages/verified-fetch/test/tar.spec.ts b/packages/verified-fetch/test/tar.spec.ts deleted file mode 100644 index 5b3dc091..00000000 --- a/packages/verified-fetch/test/tar.spec.ts +++ /dev/null @@ -1,149 +0,0 @@ -import { unixfs } from '@helia/unixfs' -import { stop } from '@libp2p/interface' -import { expect } from 'aegir/chai' -import browserReadableStreamToIt from 'browser-readablestream-to-it' -import all from 'it-all' -import last from 'it-last' -import { pipe } from 'it-pipe' -import { extract } from 'it-tar' -import toBuffer from 'it-to-buffer' -import { VerifiedFetch } from '../src/verified-fetch.js' -import { createHelia } from './fixtures/create-offline-helia.js' -import type { Helia } from '@helia/interface' -import type { FileCandidate } from 'ipfs-unixfs-importer' - -describe('tar files', () => { - let helia: Helia - let verifiedFetch: VerifiedFetch - - beforeEach(async () => { - helia = await createHelia() - verifiedFetch = new VerifiedFetch({ - helia - }) - }) - - afterEach(async () => { - await stop(helia, verifiedFetch) - }) - - it('should support fetching a TAR file', async () => { - const file = Uint8Array.from([0, 1, 2, 3, 4]) - const fs = unixfs(helia) - const cid = await fs.addBytes(file) - - const resp = await verifiedFetch.fetch(cid, { - headers: { - accept: 'application/x-tar' - } - }) - expect(resp.status).to.equal(200) - expect(resp.headers.get('content-type')).to.equal('application/x-tar') - expect(resp.headers.get('content-disposition')).to.equal(`attachment; filename="${cid.toString()}.tar"`) - - if (resp.body == null) { - throw new Error('Download failed') - } - - const entries = await pipe( - browserReadableStreamToIt(resp.body), - extract(), - async source => all(source) - ) - - expect(entries).to.have.lengthOf(1) - await expect(toBuffer(entries[0].body)).to.eventually.deep.equal(file) - }) - - it('should support fetching a TAR file containing a directory', async () => { - const directory: FileCandidate[] = [{ - path: 'foo.txt', - content: Uint8Array.from([0, 1, 2, 3, 4]) - }, { - path: 'bar.txt', - content: Uint8Array.from([5, 6, 7, 8, 9]) - }, { - path: 'baz/qux.txt', - content: Uint8Array.from([1, 2, 3, 4, 5]) - }] - - const fs = unixfs(helia) - const importResult = await last(fs.addAll(directory, { - wrapWithDirectory: true - })) - - if (importResult == null) { - throw new Error('Import failed') - } - - const resp = await verifiedFetch.fetch(importResult.cid, { - headers: { - accept: 'application/x-tar' - } - }) - expect(resp.status).to.equal(200) - expect(resp.headers.get('content-type')).to.equal('application/x-tar') - expect(resp.headers.get('content-disposition')).to.equal(`attachment; filename="${importResult.cid.toString()}.tar"`) - - if (resp.body == null) { - throw new Error('Download failed') - } - - const entries = await pipe( - browserReadableStreamToIt(resp.body), - extract(), - async source => all(source) - ) - - expect(entries).to.have.lengthOf(5) - expect(entries[0]).to.have.nested.property('header.name', importResult.cid.toString()) - - expect(entries[1]).to.have.nested.property('header.name', `${importResult.cid}/${directory[1].path}`) - await expect(toBuffer(entries[1].body)).to.eventually.deep.equal(directory[1].content) - - expect(entries[2]).to.have.nested.property('header.name', `${importResult.cid}/${directory[2].path?.split('/')[0]}`) - - expect(entries[3]).to.have.nested.property('header.name', `${importResult.cid}/${directory[2].path}`) - await expect(toBuffer(entries[3].body)).to.eventually.deep.equal(directory[2].content) - - expect(entries[4]).to.have.nested.property('header.name', `${importResult.cid}/${directory[0].path}`) - await expect(toBuffer(entries[4].body)).to.eventually.deep.equal(directory[0].content) - }) - - it('should support fetching a TAR file by format', async () => { - const file = Uint8Array.from([0, 1, 2, 3, 4]) - const fs = unixfs(helia) - const cid = await fs.addBytes(file) - - const resp = await verifiedFetch.fetch(`ipfs://${cid}?format=tar`) - expect(resp.status).to.equal(200) - expect(resp.headers.get('content-type')).to.equal('application/x-tar') - expect(resp.headers.get('content-disposition')).to.equal(`attachment; filename="${cid.toString()}.tar"`) - }) - - it('should support specifying a filename for a TAR file', async () => { - const file = Uint8Array.from([0, 1, 2, 3, 4]) - const fs = unixfs(helia) - const cid = await fs.addBytes(file) - - const resp = await verifiedFetch.fetch(`ipfs://${cid}?filename=foo.bar`, { - headers: { - accept: 'application/x-tar' - } - }) - expect(resp.status).to.equal(200) - expect(resp.headers.get('content-type')).to.equal('application/x-tar') - expect(resp.headers.get('content-disposition')).to.equal('attachment; filename="foo.bar"') - }) - - it('should support fetching a TAR file by format with a filename', async () => { - const file = Uint8Array.from([0, 1, 2, 3, 4]) - const fs = unixfs(helia) - const cid = await fs.addBytes(file) - - const resp = await verifiedFetch.fetch(`ipfs://${cid}?format=tar&filename=foo.bar`) - expect(resp.status).to.equal(200) - expect(resp.headers.get('content-type')).to.equal('application/x-tar') - expect(resp.headers.get('content-disposition')).to.equal('attachment; filename="foo.bar"') - }) -}) diff --git a/packages/verified-fetch/test/utils/get-content-disposition-filename.spec.ts b/packages/verified-fetch/test/utils/get-content-disposition-filename.spec.ts deleted file mode 100644 index 4fb328d0..00000000 --- a/packages/verified-fetch/test/utils/get-content-disposition-filename.spec.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { expect } from 'aegir/chai' -import { getContentDispositionFilename } from '../../src/utils/get-content-disposition-filename.js' - -describe('get-content-disposition-filename', () => { - it('should support ascii-only filenames', () => { - expect( - getContentDispositionFilename('foo.txt') - ).to.equal('filename="foo.txt"') - }) - - it('should remove non-ascii characters from filenames', () => { - expect( - getContentDispositionFilename('testтест.jpg') - ).to.equal('filename="test____.jpg"; filename*=UTF-8\'\'test%D1%82%D0%B5%D1%81%D1%82.jpg') - }) -}) diff --git a/packages/verified-fetch/test/utils/select-output-type.spec.ts b/packages/verified-fetch/test/utils/select-output-type.spec.ts deleted file mode 100644 index 244d9c87..00000000 --- a/packages/verified-fetch/test/utils/select-output-type.spec.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { expect } from 'aegir/chai' -import { selectOutputType } from '../../src/utils/select-output-type.js' -import { cids } from '../fixtures/cids.js' - -describe('select-output-type', () => { - it('should return undefined if no accept header passed', () => { - const format = selectOutputType(cids.file) - - expect(format).to.be.undefined() - }) - - it('should override query format with Accept header if available', () => { - const format = selectOutputType(cids.file, 'application/vnd.ipld.car') - - expect(format).to.equal('application/vnd.ipld.car') - }) - - it('should match accept headers with equal weighting in definition order', () => { - const format = selectOutputType(cids.file, 'application/x-tar, */*') - - expect(format).to.equal('application/x-tar') - }) - - it('should match accept headers in weighting order', () => { - const format = selectOutputType(cids.file, 'application/x-tar;q=0.1, application/octet-stream;q=0.5, text/html') - - expect(format).to.equal('application/octet-stream') - }) - - it('should support partial type wildcard', () => { - const format = selectOutputType(cids.file, '*/json') - - expect(format).to.equal('application/json') - }) - - it('should support partial subtype wildcard', () => { - const format = selectOutputType(cids.file, 'application/*') - - expect(format).to.equal('application/octet-stream') - }) -}) diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts deleted file mode 100644 index 9dc49c7e..00000000 --- a/packages/verified-fetch/test/verified-fetch.spec.ts +++ /dev/null @@ -1,554 +0,0 @@ -import { dagCbor } from '@helia/dag-cbor' -import { dagJson } from '@helia/dag-json' -import { json } from '@helia/json' -import { unixfs } from '@helia/unixfs' -import * as ipldDagCbor from '@ipld/dag-cbor' -import * as ipldDagJson from '@ipld/dag-json' -import { stop } from '@libp2p/interface' -import { defaultLogger } from '@libp2p/logger' -import { expect } from 'aegir/chai' -import last from 'it-last' -import toBuffer from 'it-to-buffer' -import { CID } from 'multiformats/cid' -import * as ipldJson from 'multiformats/codecs/json' -import * as raw from 'multiformats/codecs/raw' -import { identity } from 'multiformats/hashes/identity' -import { sha256 } from 'multiformats/hashes/sha2' -import Sinon from 'sinon' -import { stubInterface } from 'sinon-ts' -import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { VerifiedFetch } from '../src/verified-fetch.js' -import { createHelia } from './fixtures/create-offline-helia.js' -import type { Helia } from '@helia/interface' - -describe('@helia/verifed-fetch', () => { - let helia: Helia - - beforeEach(async () => { - helia = await createHelia() - }) - - afterEach(async () => { - await stop(helia) - }) - - it('starts and stops the helia node', async () => { - const helia = stubInterface({ - logger: defaultLogger() - }) - const verifiedFetch = new VerifiedFetch({ - helia - }) - - expect(helia.stop.callCount).to.equal(0) - expect(helia.start.callCount).to.equal(0) - - await verifiedFetch.start() - expect(helia.stop.callCount).to.equal(0) - expect(helia.start.callCount).to.equal(1) - - await verifiedFetch.stop() - expect(helia.stop.callCount).to.equal(1) - expect(helia.start.callCount).to.equal(1) - }) - - describe('implicit format', () => { - let verifiedFetch: VerifiedFetch - - beforeEach(async () => { - verifiedFetch = new VerifiedFetch({ - helia - }) - }) - - afterEach(async () => { - await verifiedFetch.stop() - }) - - it('should return raw data', async () => { - const finalRootFileContent = new Uint8Array([0x01, 0x02, 0x03]) - const cid = CID.createV1(raw.code, await sha256.digest(finalRootFileContent)) - await helia.blockstore.put(cid, finalRootFileContent) - - const resp = await verifiedFetch.fetch(`ipfs://${cid}`) - expect(resp).to.be.ok() - expect(resp.status).to.equal(200) - expect(resp.statusText).to.equal('OK') - const data = await resp.arrayBuffer() - expect(new Uint8Array(data)).to.equalBytes(finalRootFileContent) - }) - - it('should report progress during fetch', async () => { - const finalRootFileContent = new Uint8Array([0x01, 0x02, 0x03]) - const cid = CID.createV1(raw.code, await sha256.digest(finalRootFileContent)) - await helia.blockstore.put(cid, finalRootFileContent) - - const onProgress = Sinon.spy() - - await verifiedFetch.fetch(`ipfs://${cid}`, { - onProgress - }) - - expect(onProgress.callCount).to.equal(4) - - const onProgressEvents = onProgress.getCalls().map(call => call.args[0]) - expect(onProgressEvents[0]).to.include({ type: 'verified-fetch:request:start' }).and.to.have.property('detail').that.deep.equals({ - resource: `ipfs://${cid}` - }) - expect(onProgressEvents[1]).to.include({ type: 'verified-fetch:request:resolve' }).and.to.have.property('detail').that.deep.equals({ - cid, - path: '' - }) - expect(onProgressEvents[2]).to.include({ type: 'blocks:get:blockstore:get' }).and.to.have.property('detail').that.deep.equals(cid) - expect(onProgressEvents[3]).to.include({ type: 'verified-fetch:request:end' }).and.to.have.property('detail').that.deep.equals({ - cid, - path: '' - }) - }) - - it('should look for index files when directory is returned', async () => { - const finalRootFileContent = new Uint8Array([0x01, 0x02, 0x03]) - - const fs = unixfs(helia) - const res = await last(fs.addAll([{ - path: 'index.html', - content: finalRootFileContent - }], { - wrapWithDirectory: true - })) - - if (res == null) { - throw new Error('Import failed') - } - - const stat = await fs.stat(res.cid) - expect(stat.type).to.equal('directory') - - const resp = await verifiedFetch.fetch(res.cid) - expect(resp).to.be.ok() - expect(resp.status).to.equal(200) - expect(resp.statusText).to.equal('OK') - - const data = await resp.arrayBuffer() - expect(new Uint8Array(data)).to.equalBytes(finalRootFileContent) - }) - - it('should allow use as a stream', async () => { - const content = new Uint8Array([0x01, 0x02, 0x03]) - - const fs = unixfs(helia) - const cid = await fs.addBytes(content) - - const res = await verifiedFetch.fetch(cid) - const reader = res.body?.getReader() - const output: Uint8Array[] = [] - - while (true) { - const next = await reader?.read() - - if (next?.done === true) { - break - } - - if (next?.value != null) { - output.push(next.value) - } - } - - expect(toBuffer(output)).to.equalBytes(content) - }) - - it('should return 501 if index file is not found', async () => { - const finalRootFileContent = new Uint8Array([0x01, 0x02, 0x03]) - - const fs = unixfs(helia) - const res = await last(fs.addAll([{ - path: 'not_an_index.html', - content: finalRootFileContent - }], { - wrapWithDirectory: true - })) - - if (res == null) { - throw new Error('Import failed') - } - - const stat = await fs.stat(res.cid) - expect(stat.type).to.equal('directory') - - const resp = await verifiedFetch.fetch(res.cid) - expect(resp).to.be.ok() - expect(resp.status).to.equal(501) - expect(resp.statusText).to.equal('Not Implemented') - }) - - it('can round trip json via .json()', async () => { - const obj = { - hello: 'world' - } - const j = json(helia) - const cid = await j.add(obj) - - const resp = await verifiedFetch.fetch(cid) - expect(resp.headers.get('content-type')).to.equal('application/json') - - const output = await resp.json() - await expect(j.add(output)).to.eventually.deep.equal(cid) - }) - - it('can round trip json via .arrayBuffer()', async () => { - const obj = { - hello: 'world' - } - const j = json(helia) - const cid = await j.add(obj) - - const resp = await verifiedFetch.fetch(cid) - expect(resp.headers.get('content-type')).to.equal('application/json') - - const output = ipldJson.decode(await resp.arrayBuffer()) - await expect(j.add(output)).to.eventually.deep.equal(cid) - }) - - it('should handle dag-json block', async () => { - const obj = { - hello: 'world' - } - const j = dagJson(helia) - const cid = await j.add(obj) - - const resp = await verifiedFetch.fetch(cid) - expect(resp).to.be.ok() - expect(resp.status).to.equal(200) - expect(resp.statusText).to.equal('OK') - expect(resp.headers.get('content-type')).to.equal('application/json') - - await expect(resp.json()).to.eventually.deep.equal(obj) - }) - - it('should return dag-json data with embedded CID', async () => { - const obj = { - hello: 'world', - link: CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN') - } - const j = dagJson(helia) - const cid = await j.add(obj) - - const resp = await verifiedFetch.fetch(cid) - expect(resp.headers.get('content-type')).to.equal('application/json') - - const data = await resp.json() - expect(data).to.deep.equal({ - hello: 'world', - link: { - '/': 'QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN' - } - }) - }) - - it('should return dag-json data with embedded bytes', async () => { - const obj = { - hello: 'world', - bytes: Uint8Array.from([0, 1, 2, 3, 4]) - } - const j = dagJson(helia) - const cid = await j.add(obj) - - const resp = await verifiedFetch.fetch(cid) - expect(resp.headers.get('content-type')).to.equal('application/json') - - const data = await resp.json() - expect(data).to.deep.equal({ - hello: 'world', - bytes: { - '/': { - bytes: 'AAECAwQ' - } - } - }) - }) - - it('can round trip dag-json via .json()', async () => { - const obj = { - hello: 'world', - // n.b. cannot round-trip larger than Number.MAX_SAFE_INTEGER because - // parsing DAG-JSON as using JSON.parse loses precision - bigInt: 10n, - link: CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN') - } - const j = dagJson(helia) - const cid = await j.add(obj) - - const resp = await verifiedFetch.fetch(cid) - expect(resp.headers.get('content-type')).to.equal('application/json') - - const output = await resp.json() - await expect(j.add(output)).to.eventually.deep.equal(cid) - }) - - it('can round trip dag-json via .arrayBuffer()', async () => { - const obj = { - hello: 'world', - bigInt: BigInt(Number.MAX_SAFE_INTEGER) + 1n, - link: CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN') - } - const j = dagJson(helia) - const cid = await j.add(obj) - - const resp = await verifiedFetch.fetch(cid) - expect(resp.headers.get('content-type')).to.equal('application/json') - - const output = ipldDagJson.decode(await resp.arrayBuffer()) - await expect(j.add(output)).to.eventually.deep.equal(cid) - }) - - it('should handle JSON-compliant dag-cbor block', async () => { - const obj = { - hello: 'world' - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const resp = await verifiedFetch.fetch(cid) - expect(resp).to.be.ok() - expect(resp.status).to.equal(200) - expect(resp.statusText).to.equal('OK') - expect(resp.headers.get('content-type')).to.equal('application/json') - await expect(resp.json()).to.eventually.deep.equal(obj) - }) - - it('should return dag-cbor data with embedded CID', async () => { - const obj = { - hello: 'world', - link: CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN') - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const resp = await verifiedFetch.fetch(cid) - expect(resp.headers.get('content-type')).to.equal('application/octet-stream') - - const data = await ipldDagCbor.decode(await resp.arrayBuffer()) - expect(data).to.deep.equal(obj) - }) - - it('should return dag-cbor data with embedded bytes', async () => { - const obj = { - hello: 'world', - bytes: Uint8Array.from([0, 1, 2, 3, 4]) - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const resp = await verifiedFetch.fetch(cid) - expect(resp.headers.get('content-type')).to.equal('application/octet-stream') - - const data = await ipldDagCbor.decode(await resp.arrayBuffer()) - expect(data).to.deep.equal(obj) - }) - - it('should allow parsing dag-cbor object array buffer as dag-json', async () => { - const obj = { - hello: 'world' - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const resp = await verifiedFetch.fetch(cid) - expect(resp.headers.get('content-type')).to.equal('application/json') - - const data = ipldDagJson.decode(await resp.arrayBuffer()) - expect(data).to.deep.equal(obj) - }) - - it('should return dag-cbor with a small BigInt as application/json', async () => { - const obj = { - hello: 'world', - bigInt: 10n - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const resp = await verifiedFetch.fetch(cid) - expect(resp.headers.get('content-type')).to.equal('application/json') - - const data = await resp.json() - expect(data).to.deep.equal({ - hello: 'world', - bigInt: 10 - }) - }) - - it('should return dag-cbor with a large BigInt as application/octet-stream', async () => { - const obj = { - hello: 'world', - bigInt: BigInt(Number.MAX_SAFE_INTEGER) + 1n - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const resp = await verifiedFetch.fetch(cid) - expect(resp.headers.get('content-type')).to.equal('application/octet-stream') - - const data = ipldDagCbor.decode(await resp.arrayBuffer()) - expect(data).to.deep.equal(obj) - }) - - it('can round trip JSON-compliant dag-cbor via .json()', async () => { - const obj = { - hello: 'world' - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const resp = await verifiedFetch.fetch(cid) - expect(resp.headers.get('content-type')).to.equal('application/json') - - const output = await resp.json() - await expect(c.add(output)).to.eventually.deep.equal(cid) - }) - - // N.b. this is not possible because the incoming block is turned into JSON - // and returned as the response body, so `.arrayBuffer()` returns a string - // encoded into a Uint8Array which we can't parse as CBOR - it.skip('can round trip JSON-compliant dag-cbor via .arrayBuffer()', async () => { - const obj = { - hello: 'world' - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const resp = await verifiedFetch.fetch(cid) - expect(resp.headers.get('content-type')).to.equal('application/json') - - const output = ipldDagCbor.decode(await resp.arrayBuffer()) - await expect(c.add(output)).to.eventually.deep.equal(cid) - }) - - it('can round trip dag-cbor via .arrayBuffer()', async () => { - const obj = { - hello: 'world', - bigInt: BigInt(Number.MAX_SAFE_INTEGER) + 1n, - link: CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN') - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const resp = await verifiedFetch.fetch(cid) - expect(resp.headers.get('content-type')).to.equal('application/octet-stream') - - const output = ipldDagCbor.decode(await resp.arrayBuffer()) - await expect(c.add(output)).to.eventually.deep.equal(cid) - }) - - it('should handle json block', async () => { - const obj = { - hello: 'world' - } - const j = json(helia) - const cid = await j.add(obj) - - const resp = await verifiedFetch.fetch(cid) - expect(resp).to.be.ok() - expect(resp.status).to.equal(200) - expect(resp.statusText).to.equal('OK') - await expect(resp.json()).to.eventually.deep.equal(obj) - }) - - it('should handle identity CID', async () => { - const data = uint8ArrayFromString('hello world') - const cid = CID.createV1(identity.code, identity.digest(data)) - - const resp = await verifiedFetch.fetch(cid) - expect(resp).to.be.ok() - expect(resp.status).to.equal(200) - expect(resp.statusText).to.equal('OK') - await expect(resp.text()).to.eventually.equal('hello world') - }) - }) - - describe('accept', () => { - let helia: Helia - let verifiedFetch: VerifiedFetch - let contentTypeParser: Sinon.SinonStub - - beforeEach(async () => { - contentTypeParser = Sinon.stub() - helia = await createHelia() - verifiedFetch = new VerifiedFetch({ - helia - }, { - contentTypeParser - }) - }) - - afterEach(async () => { - await stop(helia, verifiedFetch) - }) - - it('should allow specifying an accept header', async () => { - const obj = { - hello: 'world' - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const resp = await verifiedFetch.fetch(cid, { - headers: { - accept: 'application/octet-stream' - } - }) - expect(resp.headers.get('content-type')).to.equal('application/octet-stream') - const output = ipldDagCbor.decode(new Uint8Array(await resp.arrayBuffer())) - expect(output).to.deep.equal(obj) - }) - - it('should return a 406 if the content cannot be represented by the mime type in the accept header', async () => { - const obj = { - hello: 'world', - // fails to parse as JSON - link: CID.parse('QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN') - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const resp = await verifiedFetch.fetch(cid, { - headers: { - accept: 'application/json' - } - }) - expect(resp.status).to.equal(406) - }) - - it('should return a 406 if the content type parser returns a different value to the accept header', async () => { - contentTypeParser.returns('text/plain') - - const fs = unixfs(helia) - const cid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3, 4])) - - const resp = await verifiedFetch.fetch(cid, { - headers: { - accept: 'image/jpeg' - } - }) - expect(resp.status).to.equal(406) - }) - - it('should allow specifying an accept as raw', async () => { - const obj = { - hello: 'world' - } - const c = dagCbor(helia) - const cid = await c.add(obj) - - const resp = await verifiedFetch.fetch(cid, { - headers: { - accept: 'application/vnd.ipld.raw' - } - }) - expect(resp.headers.get('content-type')).to.equal('application/vnd.ipld.raw') - const output = ipldDagCbor.decode(new Uint8Array(await resp.arrayBuffer())) - expect(output).to.deep.equal(obj) - }) - }) -}) diff --git a/packages/verified-fetch/tsconfig.json b/packages/verified-fetch/tsconfig.json deleted file mode 100644 index 6c7cd39b..00000000 --- a/packages/verified-fetch/tsconfig.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "extends": "aegir/src/config/tsconfig.aegir.json", - "compilerOptions": { - "outDir": "dist" - }, - "include": [ - "src", - "test" - ], - "references": [ - { - "path": "../block-brokers" - }, - { - "path": "../dag-cbor" - }, - { - "path": "../dag-json" - }, - { - "path": "../helia" - }, - { - "path": "../http" - }, - { - "path": "../interface" - }, - { - "path": "../ipns" - }, - { - "path": "../json" - }, - { - "path": "../routers" - }, - { - "path": "../unixfs" - } - ] -} diff --git a/packages/verified-fetch/typedoc.json b/packages/verified-fetch/typedoc.json deleted file mode 100644 index f599dc72..00000000 --- a/packages/verified-fetch/typedoc.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "entryPoints": [ - "./src/index.ts" - ] -}