diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 290ad028..0bc3b42d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,3 +6,6 @@ updates: interval: daily time: "10:00" open-pull-requests-limit: 10 + commit-message: + prefix: "deps" + prefix-development: "deps(dev)" diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index 1d6c6ebe..65d30bda 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -15,8 +15,8 @@ jobs: check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 with: node-version: lts/* - uses: ipfs/aegir/actions/cache-node-modules@master @@ -32,105 +32,98 @@ jobs: node: [16] fail-fast: true steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 with: node-version: ${{ matrix.node }} - uses: ipfs/aegir/actions/cache-node-modules@master - run: npm run --if-present test:node - - uses: codecov/codecov-action@f32b3a3741e1053eb607407145bc9619351dc93b # v2.1.0 + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 with: - directory: ./.nyc_output flags: node test-chrome: needs: check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 with: node-version: lts/* - uses: ipfs/aegir/actions/cache-node-modules@master - run: npm run --if-present test:chrome - - uses: codecov/codecov-action@f32b3a3741e1053eb607407145bc9619351dc93b # v2.1.0 + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 with: - directory: ./.nyc_output flags: chrome test-chrome-webworker: needs: check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 with: node-version: lts/* - uses: ipfs/aegir/actions/cache-node-modules@master - run: npm run --if-present test:chrome-webworker - - uses: codecov/codecov-action@f32b3a3741e1053eb607407145bc9619351dc93b # v2.1.0 + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 with: - directory: ./.nyc_output flags: chrome-webworker test-firefox: needs: check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 with: node-version: lts/* - uses: ipfs/aegir/actions/cache-node-modules@master - run: npm run --if-present test:firefox - - uses: codecov/codecov-action@f32b3a3741e1053eb607407145bc9619351dc93b # v2.1.0 + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 with: - directory: ./.nyc_output flags: firefox test-firefox-webworker: needs: check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 with: node-version: lts/* - uses: ipfs/aegir/actions/cache-node-modules@master - run: npm run --if-present test:firefox-webworker - - uses: codecov/codecov-action@f32b3a3741e1053eb607407145bc9619351dc93b # v2.1.0 + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 with: - directory: ./.nyc_output flags: firefox-webworker test-electron-main: needs: check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 with: node-version: lts/* - uses: ipfs/aegir/actions/cache-node-modules@master - run: npx xvfb-maybe npm run --if-present test:electron-main - - uses: codecov/codecov-action@f32b3a3741e1053eb607407145bc9619351dc93b # v2.1.0 + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 with: - directory: ./.nyc_output flags: electron-main test-electron-renderer: needs: check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 with: node-version: lts/* - uses: ipfs/aegir/actions/cache-node-modules@master - run: npx xvfb-maybe npm run --if-present test:electron-renderer - - uses: codecov/codecov-action@f32b3a3741e1053eb607407145bc9619351dc93b # v2.1.0 + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 with: - directory: ./.nyc_output flags: electron-renderer release: @@ -138,10 +131,10 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'push' && github.ref == 'refs/heads/master' # with #262 - 'refs/heads/${{{ github.default_branch }}}' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: actions/setup-node@v2 + - uses: actions/setup-node@v3 with: node-version: lts/* - uses: ipfs/aegir/actions/cache-node-modules@master diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 00000000..6f6d895d --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,26 @@ +name: Close and mark stale issue + +on: + schedule: + - cron: '0 0 * * *' + +jobs: + stale: + + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + + steps: + - uses: actions/stale@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: 'Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 7 days.' + close-issue-message: 'This issue was closed because it is missing author input.' + stale-issue-label: 'kind/stale' + any-of-labels: 'need/author-input' + exempt-issue-labels: 'need/triage,need/community-input,need/maintainer-input,need/maintainers-input,need/analysis,status/blocked,status/in-progress,status/ready,status/deferred,status/inactive' + days-before-issue-stale: 6 + days-before-issue-close: 7 + enable-statistics: true diff --git a/.gitignore b/.gitignore index 953d9319..da397574 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ logs *.log coverage +.coverage *.lcov # Runtime data diff --git a/README.md b/README.md index 28feb266..1fab63b7 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,38 @@ -# ipfs-unixfs +# js-ipfs-unixfs -[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) -[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) -[![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) -[![Travis CI](https://flat.badgen.net/travis/ipfs/js-ipfs-unixfs)](https://travis-ci.com/ipfs/js-ipfs-unixfs) -[![Codecov](https://codecov.io/gh/ipfs/js-ipfs-unixfs/branch/master/graph/badge.svg)](https://codecov.io/gh/ipfs/js-ipfs-unixfs) -[![Style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat-square)](https://github.com/feross/standard) +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-ipfs-unixfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-ipfs-unixfs) +[![CI](https://img.shields.io/github/workflow/status/ipfs/js-ipfs-unixfs/test%20&%20maybe%20release/master?style=flat-square)](https://github.com/ipfs/js-ipfs-unixfs/actions/workflows/js-test-and-release.yml) -> JavaScript implementation of IPFS' UnixFS (a representation of a Unix file system on top of a MerkleDAG) +> JS implementation of the IPFS UnixFS -The UnixFS spec can be found at [ipfs/specs/UNIXFS.md](https://github.com/ipfs/specs/blob/master/UNIXFS.md) - -## Lead Maintainer - -[Alex Potsides](https://github.com/achingbrain) - -## Table of Contents +## Table of contents - [Structure](#structure) +- [Lead Maintainer ](#lead-maintainer----omit-in-toc---) - [Development](#development) - [Publishing new versions](#publishing-new-versions) - [Using prerelease versions](#using-prerelease-versions) - [Contribute](#contribute) - [License](#license) +- [Contribute](#contribute-1) ## Structure -This project is broken into several modules, their purposes are: +- [`/packages/ipfs-unixfs`](./packages/ipfs-unixfs) JavaScript implementation of IPFS' unixfs (a Unix FileSystem representation on top of a MerkleDAG) +- [`/packages/ipfs-unixfs-exporter`](./packages/ipfs-unixfs-exporter) JavaScript implementation of the UnixFs exporter used by IPFS +- [`/packages/ipfs-unixfs-importer`](./packages/ipfs-unixfs-importer) JavaScript implementation of the UnixFs importer used by IPFS + +The UnixFS spec can be found at [ipfs/specs/UNIXFS.md](https://github.com/ipfs/specs/blob/master/UNIXFS.md) + +## Lead Maintainer + +[Alex Potsides](https://github.com/achingbrain) -* [`/packages/ipfs-unixfs`](./packages/ipfs-unixfs) Serialization/deserialization of UnixFS objects to protocol buffers -* [`/packages/ipfs-unixfs-importer`](./packages/ipfs-unixfs-importer) Builds DAGs from files and directories -* [`/packages/ipfs-unixfs-exporter`](./packages/ipfs-unixfs-exporter) Exports DAGs +- [`/packages/ipfs-unixfs`](./packages/ipfs-unixfs) Serialization/deserialization of UnixFS objects to protocol buffers +- [`/packages/ipfs-unixfs-importer`](./packages/ipfs-unixfs-importer) Builds DAGs from files and directories +- [`/packages/ipfs-unixfs-exporter`](./packages/ipfs-unixfs-exporter) Exports DAGs ## Development @@ -64,4 +66,19 @@ This repository falls under the IPFS [Code of Conduct](https://github.com/ipfs/c ## License -[MIT](LICENSE) +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-ipfs-unixfs/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/lerna.json b/lerna.json index aa6d0004..fe9f1a3a 100644 --- a/lerna.json +++ b/lerna.json @@ -1,12 +1,13 @@ { - "lerna": "4.0.0", - "packages": [ - "packages/*" - ], + "lerna": "5.4.0", + "useWorkspaces": true, "version": "independent", "command": { "run": { "stream": true } - } + }, + "packages": [ + "packages/*" + ] } diff --git a/package.json b/package.json index 4ad13838..ac754dd0 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "release": "lerna exec --concurrency 1 -- semantic-release -e semantic-release-monorepo" }, "dependencies": { - "lerna": "^5.0.0" + "lerna": "^6.0.1" }, "workspaces": [ "packages/*" diff --git a/packages/ipfs-unixfs-exporter/.aegir.cjs b/packages/ipfs-unixfs-exporter/.aegir.cjs deleted file mode 100644 index 7c48d0be..00000000 --- a/packages/ipfs-unixfs-exporter/.aegir.cjs +++ /dev/null @@ -1,39 +0,0 @@ -'use strict' - -/** @type {import('aegir').Options["build"]["config"]} */ -const buildConfig = { - plugins: [ - { - name: 'node built ins', - setup (build) { - build.onResolve({ filter: /^stream$/ }, () => { - return { path: require.resolve('readable-stream') } - }) - build.onResolve({ filter: /^crypto$/ }, () => { - return { path: require.resolve('crypto-browserify') } - }) - build.onResolve({ filter: /^cborg$/ }, () => { - return { path: require.resolve('cborg') } - }) - } - } - ] -} - -/** @type {import('aegir').PartialOptions} */ -module.exports = { - build: { - bundlesizeMax: '34KB', - config: buildConfig - }, - test: { - browser: { - config: { - buildConfig - } - } - }, - ts: { - copyTo: 'types' - } -} diff --git a/packages/ipfs-unixfs-exporter/.aegir.js b/packages/ipfs-unixfs-exporter/.aegir.js new file mode 100644 index 00000000..c3a96d38 --- /dev/null +++ b/packages/ipfs-unixfs-exporter/.aegir.js @@ -0,0 +1,7 @@ + +/** @type {import('aegir').PartialOptions} */ +export default { + build: { + bundlesizeMax: '31KB' + } +} diff --git a/packages/ipfs-unixfs-exporter/CHANGELOG.md b/packages/ipfs-unixfs-exporter/CHANGELOG.md index 94aede46..fe247989 100644 --- a/packages/ipfs-unixfs-exporter/CHANGELOG.md +++ b/packages/ipfs-unixfs-exporter/CHANGELOG.md @@ -1,3 +1,69 @@ +## [ipfs-unixfs-exporter-v8.0.6](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-v8.0.5...ipfs-unixfs-exporter-v8.0.6) (2022-09-21) + + +### Bug Fixes + +* prevent OOM on very deep DAGs ([#253](https://github.com/ipfs/js-ipfs-unixfs/issues/253)) ([62494bf](https://github.com/ipfs/js-ipfs-unixfs/commit/62494bf8cdea7b7ea430a6517c526798bc01b747)) + +## [ipfs-unixfs-exporter-v8.0.5](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-v8.0.4...ipfs-unixfs-exporter-v8.0.5) (2022-08-31) + + +### Bug Fixes + +* specify return type of the importer to generate correct types ([#251](https://github.com/ipfs/js-ipfs-unixfs/issues/251)) ([3343366](https://github.com/ipfs/js-ipfs-unixfs/commit/33433660a1f762d3e9b9a431cfb956136821cfb1)), closes [#214](https://github.com/ipfs/js-ipfs-unixfs/issues/214) + +## [ipfs-unixfs-exporter-v8.0.4](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-v8.0.3...ipfs-unixfs-exporter-v8.0.4) (2022-08-17) + + +### Bug Fixes + +* yield buf after reading length to make it safe to use with worker transfer ([eeeda23](https://github.com/ipfs/js-ipfs-unixfs/commit/eeeda2397ccd2ccebb1de5f34af126d697ac80f5)) + +## [ipfs-unixfs-exporter-v8.0.3](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-v8.0.2...ipfs-unixfs-exporter-v8.0.3) (2022-08-17) + + +### Bug Fixes + +* handle empty files again ([4c1b9f6](https://github.com/ipfs/js-ipfs-unixfs/commit/4c1b9f666a77ffc2bc9f5fbaa6342257052d6e62)) + +## [ipfs-unixfs-exporter-v8.0.2](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-v8.0.1...ipfs-unixfs-exporter-v8.0.2) (2022-08-17) + + +### Bug Fixes + +* parallelise loading of dag-pb links when exporting ([#249](https://github.com/ipfs/js-ipfs-unixfs/issues/249)) ([862d63b](https://github.com/ipfs/js-ipfs-unixfs/commit/862d63bc4e158187ff6fb97c680b8886e0d2001a)), closes [#237](https://github.com/ipfs/js-ipfs-unixfs/issues/237) + +## [ipfs-unixfs-exporter-v8.0.1](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-v8.0.0...ipfs-unixfs-exporter-v8.0.1) (2022-08-16) + + +### Bug Fixes + +* update types import path and deps ([#248](https://github.com/ipfs/js-ipfs-unixfs/issues/248)) ([2edd327](https://github.com/ipfs/js-ipfs-unixfs/commit/2edd327bbe25880a83a8ea00963a22e3f9fa4449)) + +## [ipfs-unixfs-exporter-v8.0.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-v7.0.11...ipfs-unixfs-exporter-v8.0.0) (2022-08-11) + + +### ⚠ BREAKING CHANGES + +* This module used to be dual published as CJS/ESM, now it is just ESM + +### Dependencies + +* update aegir to 37.x.x ([#243](https://github.com/ipfs/js-ipfs-unixfs/issues/243)) ([9fccb7c](https://github.com/ipfs/js-ipfs-unixfs/commit/9fccb7c4bf989c052ba67ab2c9280273aecfda36)) +* update hamt-sharding to 3.0.0 ([#244](https://github.com/ipfs/js-ipfs-unixfs/issues/244)) ([50a7607](https://github.com/ipfs/js-ipfs-unixfs/commit/50a7607d1876dab0b34dfe79704688f01ce1d449)) + +## [ipfs-unixfs-exporter-v7.0.11](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-v7.0.10...ipfs-unixfs-exporter-v7.0.11) (2022-05-27) + + +### Bug Fixes + +* remove typesVersions from package.json ([#219](https://github.com/ipfs/js-ipfs-unixfs/issues/219)) ([465670e](https://github.com/ipfs/js-ipfs-unixfs/commit/465670eab2e707b14b14047e2da4ede23590196e)), closes [#214](https://github.com/ipfs/js-ipfs-unixfs/issues/214) [#161](https://github.com/ipfs/js-ipfs-unixfs/issues/161) [#214](https://github.com/ipfs/js-ipfs-unixfs/issues/214) + + +### Trivial Changes + +* update dep-check command ([#221](https://github.com/ipfs/js-ipfs-unixfs/issues/221)) ([5802bd3](https://github.com/ipfs/js-ipfs-unixfs/commit/5802bd366768e1a024fad30e4190aed866f5c9ec)) + ## [ipfs-unixfs-exporter-v7.0.10](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-exporter-v7.0.9...ipfs-unixfs-exporter-v7.0.10) (2022-05-27) diff --git a/packages/ipfs-unixfs-exporter/README.md b/packages/ipfs-unixfs-exporter/README.md index 5ad2df74..a4dee931 100644 --- a/packages/ipfs-unixfs-exporter/README.md +++ b/packages/ipfs-unixfs-exporter/README.md @@ -1,42 +1,40 @@ -# ipfs-unixfs-exporter +# ipfs-unixfs-exporter + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-ipfs-unixfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-ipfs-unixfs) +[![CI](https://img.shields.io/github/workflow/status/ipfs/js-ipfs-unixfs/test%20&%20maybe%20release/master?style=flat-square)](https://github.com/ipfs/js-ipfs-unixfs/actions/workflows/js-test-and-release.yml) + +> JavaScript implementation of the UnixFs exporter used by IPFS + +## Table of contents + +- [Install](#install) +- [Lead Maintainer](#lead-maintainer) +- [Usage](#usage) + - [Example](#example) + - [API](#api) + - [`exporter(cid, blockstore, options)`](#exportercid-blockstore-options) + - [UnixFSEntry](#unixfsentry) + - [Raw entries](#raw-entries) + - [CBOR entries](#cbor-entries) + - [`entry.content({ offset, length })`](#entrycontent-offset-length-) + - [`walkPath(cid, blockstore)`](#walkpathcid-blockstore) + - [`recursive(cid, blockstore)`](#recursivecid-blockstore) +- [Contribute](#contribute) +- [License](#license) +- [Contribute](#contribute-1) -[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) -[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) -[![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) -[![Build Status](https://flat.badgen.net/travis/ipfs/js-ipfs-unixfs)](https://travis-ci.com/ipfs/js-ipfs-unixfs) -[![Codecov](https://codecov.io/gh/ipfs/js-ipfs-unixfs/branch/master/graph/badge.svg)](https://codecov.io/gh/ipfs/js-ipfs-unixfs) +## Install -> Exports UnixFS and other DAGs from IPFS +```console +$ npm i ipfs-unixfs-exporter +``` ## Lead Maintainer [Alex Potsides](https://github.com/achingbrain) -## Table of Contents - -- [ipfs-unixfs-exporter](#ipfs-unixfs-exporter) - - [Lead Maintainer](#lead-maintainer) - - [Table of Contents](#table-of-contents) - - [Install](#install) - - [Usage](#usage) - - [Example](#example) - - [API](#api) - - [`exporter(cid, blockstore, options)`](#exportercid-blockstore-options) - - [UnixFSEntry](#unixfsentry) - - [Raw entries](#raw-entries) - - [CBOR entries](#cbor-entries) - - [`entry.content({ offset, length })`](#entrycontent-offset-length-) - - [`walkPath(cid, blockstore)`](#walkpathcid-blockstore) - - [`recursive(cid, blockstore)`](#recursivecid-blockstore) - - [Contribute](#contribute) - - [License](#license) - -## Install - -``` -> npm install ipfs-unixfs-exporter -``` - ## Usage ### Example @@ -237,10 +235,6 @@ for await (const child of recursive('Qmfoo/foo/bar', blockstore)) { // entries contains all children of the `Qmfoo/foo/bar` directory and it's children ``` -[dag API]: https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DAG.md -[blockstore]: https://github.com/ipfs/js-ipfs-interfaces/tree/master/packages/interface-blockstore#readme -[UnixFS]: https://github.com/ipfs/specs/tree/master/unixfs - ## Contribute Feel free to join in. All welcome. Open an [issue](https://github.com/ipfs/js-ipfs-unixfs-exporter/issues)! @@ -251,4 +245,25 @@ This repository falls under the IPFS [Code of Conduct](https://github.com/ipfs/c ## License -[MIT](LICENSE) +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-ipfs-unixfs/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) + +[dag API]: https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DAG.md + +[blockstore]: https://github.com/ipfs/js-ipfs-interfaces/tree/master/packages/interface-blockstore#readme + +[UnixFS]: https://github.com/ipfs/specs/tree/master/unixfs diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index a9787a0b..cdf25e47 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-exporter", - "version": "7.0.6", + "version": "8.0.6", "description": "JavaScript implementation of the UnixFs exporter used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/master/packages/ipfs-unixfs-exporter#readme", @@ -19,15 +19,32 @@ "npm": ">=7.0.0" }, "type": "module", - "types": "types/src/index.d.ts", + "types": "./dist/src/index.d.ts", + "typesVersions": { + "*": { + "*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ], + "src/*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ] + } + }, "files": [ - "*", - "!**/*.tsbuildinfo", - "!**/browser-test", - "!**/node-test" + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" ], "exports": { ".": { + "types": "./dist/src/index.d.ts", "import": "./src/index.js" } }, @@ -64,15 +81,15 @@ "release": "patch" }, { - "type": "chore", + "type": "docs", "release": "patch" }, { - "type": "docs", + "type": "test", "release": "patch" }, { - "type": "test", + "type": "deps", "release": "patch" }, { @@ -102,7 +119,11 @@ }, { "type": "docs", - "section": "Trivial Changes" + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" }, { "type": "test", @@ -113,66 +134,55 @@ } ], "@semantic-release/changelog", - [ - "@semantic-release/npm", - { - "pkgRoot": "dist" - } - ], + "@semantic-release/npm", "@semantic-release/github", "@semantic-release/git" ] }, "scripts": { - "prepare": "aegir build --no-bundle && cp -R types dist", - "pretest": "aegir build --esm-tests", "test": "aegir test", "test:node": "aegir test -t node --cov", "test:chrome": "aegir test -t browser --cov", "test:firefox": "aegir test -t browser -- --browser firefox", - "build": "aegir build && cp -R types dist", - "preleaseOnly": "npx json -I -f dist/package.json -e this.types='\"src/index.d.ts\"'", - "clean": "rimraf ./dist ./types", - "lint": "aegir ts -p check && aegir lint", - "dep-check": "aegir dep-check -i @types/sinon -i nyc -i rimraf -i copy -i util -i crypto-browserify -i events -i readable-stream -i interface-blockstore", - "release": "semantic-release" + "build": "aegir build", + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check -i interface-blockstore", + "release": "aegir release" }, "dependencies": { - "@ipld/dag-cbor": "^7.0.2", - "@ipld/dag-pb": "^2.0.2", - "@multiformats/murmur3": "^1.0.3", + "@ipld/dag-cbor": "^8.0.0", + "@ipld/dag-pb": "^3.0.0", + "@multiformats/murmur3": "^2.0.0", "err-code": "^3.0.1", - "hamt-sharding": "^2.0.0", - "interface-blockstore": "^2.0.3", - "ipfs-unixfs": "^6.0.0", - "it-last": "^1.0.5", - "multiformats": "^9.4.2", - "uint8arrays": "^3.0.0" + "hamt-sharding": "^3.0.0", + "interface-blockstore": "^3.0.0", + "ipfs-unixfs": "^7.0.0", + "it-last": "^2.0.0", + "it-map": "^2.0.0", + "it-parallel": "^3.0.0", + "it-pipe": "^2.0.4", + "it-pushable": "^3.1.0", + "multiformats": "^10.0.0", + "p-queue": "^7.3.0", + "uint8arrays": "^4.0.2" }, "devDependencies": { "@types/sinon": "^10.0.0", "abort-controller": "^3.0.0", - "aegir": "^36.2.3", - "blockstore-core": "^1.0.5", - "copy": "^0.3.2", + "aegir": "^37.5.0", + "blockstore-core": "^2.0.1", "crypto-browserify": "^3.12.0", - "events": "^3.3.0", - "ipfs-unixfs-importer": "^9.0.0", - "it-all": "^1.0.5", - "it-buffer-stream": "^2.0.0", - "it-first": "^1.0.6", + "delay": "^5.0.0", + "ipfs-unixfs-importer": "^10.0.0", + "it-all": "^2.0.0", + "it-buffer-stream": "^3.0.0", + "it-first": "^2.0.0", "merge-options": "^3.0.4", "native-abort-controller": "^1.0.3", - "readable-stream": "^3.6.0", - "rimraf": "^3.0.2", - "sinon": "^14.0.0", - "util": "^0.12.3" + "sinon": "^14.0.0" }, "browser": { "fs": false - }, - "main": "src/index.js", - "publishConfig": { - "directory": "dist" } } diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.js b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.js index ce020c21..4f11cb44 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.js +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.js @@ -3,40 +3,44 @@ import validateOffsetAndLength from '../../../utils/validate-offset-and-length.j import { UnixFS } from 'ipfs-unixfs' import errCode from 'err-code' import * as dagPb from '@ipld/dag-pb' -import * as dagCbor from '@ipld/dag-cbor' import * as raw from 'multiformats/codecs/raw' +import { pushable } from 'it-pushable' +import parallel from 'it-parallel' +import { pipe } from 'it-pipe' +import map from 'it-map' +import PQueue from 'p-queue' /** * @typedef {import('../../../types').ExporterOptions} ExporterOptions * @typedef {import('interface-blockstore').Blockstore} Blockstore * @typedef {import('@ipld/dag-pb').PBNode} PBNode - * + * @typedef {import('@ipld/dag-pb').PBLink} PBLink + */ + +/** * @param {Blockstore} blockstore - * @param {PBNode} node + * @param {PBNode | Uint8Array} node + * @param {import('it-pushable').Pushable} queue + * @param {number} streamPosition * @param {number} start * @param {number} end - * @param {number} streamPosition + * @param {PQueue} walkQueue * @param {ExporterOptions} options - * @returns {AsyncIterable} + * @returns {Promise} */ -async function * emitBytes (blockstore, node, start, end, streamPosition = 0, options) { +async function walkDAG (blockstore, node, queue, streamPosition, start, end, walkQueue, options) { // a `raw` node if (node instanceof Uint8Array) { - const buf = extractDataFromBlock(node, streamPosition, start, end) - - if (buf.length) { - yield buf - } - - streamPosition += buf.length + queue.push(extractDataFromBlock(node, streamPosition, start, end)) - return streamPosition + return } if (node.Data == null) { throw errCode(new Error('no data in PBNode'), 'ERR_NOT_UNIXFS') } + /** @type {UnixFS} */ let file try { @@ -46,54 +50,78 @@ async function * emitBytes (blockstore, node, start, end, streamPosition = 0, op } // might be a unixfs `raw` node or have data on intermediate nodes - if (file.data && file.data.length) { - const buf = extractDataFromBlock(file.data, streamPosition, start, end) + if (file.data != null) { + const data = file.data + const buf = extractDataFromBlock(data, streamPosition, start, end) - if (buf.length) { - yield buf - } + queue.push(buf) - streamPosition += file.data.length + streamPosition += buf.byteLength } - let childStart = streamPosition + /** @type {Array<{ link: PBLink, blockStart: number }>} */ + const childOps = [] - // work out which child nodes contain the requested data for (let i = 0; i < node.Links.length; i++) { const childLink = node.Links[i] - const childEnd = streamPosition + file.blockSizes[i] + const childStart = streamPosition // inclusive + const childEnd = childStart + file.blockSizes[i] // exclusive if ((start >= childStart && start < childEnd) || // child has offset byte - (end > childStart && end <= childEnd) || // child has end byte + (end >= childStart && end <= childEnd) || // child has end byte (start < childStart && end > childEnd)) { // child is between offset and end bytes - const block = await blockstore.get(childLink.Hash, { - signal: options.signal + childOps.push({ + link: childLink, + blockStart: streamPosition }) - let child - switch (childLink.Hash.code) { - case dagPb.code: - child = await dagPb.decode(block) - break - case raw.code: - child = block - break - case dagCbor.code: - child = await dagCbor.decode(block) - break - default: - throw Error(`Unsupported codec: ${childLink.Hash.code}`) - } - - for await (const buf of emitBytes(blockstore, child, start, end, streamPosition, options)) { - streamPosition += buf.length - - yield buf - } } streamPosition = childEnd - childStart = childEnd + 1 + + if (streamPosition > end) { + break + } } + + await pipe( + childOps, + (source) => map(source, (op) => { + return async () => { + const block = await blockstore.get(op.link.Hash, { + signal: options.signal + }) + + return { + ...op, + block + } + } + }), + (source) => parallel(source, { + ordered: true + }), + async (source) => { + for await (const { link, block, blockStart } of source) { + /** @type {PBNode | Uint8Array} */ + let child + switch (link.Hash.code) { + case dagPb.code: + child = dagPb.decode(block) + break + case raw.code: + child = block + break + default: + queue.end(errCode(new Error(`Unsupported codec: ${link.Hash.code}`), 'ERR_NOT_UNIXFS')) + return + } + + walkQueue.add(async () => { + await walkDAG(blockstore, child, queue, blockStart, start, end, walkQueue, options) + }) + } + } + ) } /** @@ -103,7 +131,7 @@ const fileContent = (cid, node, unixfs, path, resolve, depth, blockstore) => { /** * @param {ExporterOptions} options */ - function yieldFileContent (options = {}) { + async function * yieldFileContent (options = {}) { const fileSize = unixfs.fileSize() if (fileSize === undefined) { @@ -115,10 +143,40 @@ const fileContent = (cid, node, unixfs, path, resolve, depth, blockstore) => { length } = validateOffsetAndLength(fileSize, options.offset, options.length) - const start = offset - const end = offset + length + if (length === 0) { + return + } + + // use a queue to walk the DAG instead of recursion to ensure very deep DAGs + // don't overflow the stack + const walkQueue = new PQueue({ + concurrency: 1 + }) + const queue = pushable() + + walkQueue.add(async () => { + await walkDAG(blockstore, node, queue, 0, offset, offset + length, walkQueue, options) + }) + + walkQueue.on('error', error => { + queue.end(error) + }) - return emitBytes(blockstore, node, start, end, 0, options) + let read = 0 + + for await (const buf of queue) { + if (buf == null) { + continue + } + + read += buf.byteLength + + if (read === length) { + queue.end() + } + + yield buf + } } return yieldFileContent diff --git a/packages/ipfs-unixfs-exporter/src/types.ts b/packages/ipfs-unixfs-exporter/src/types.ts index 5675ccd5..e1210b4b 100644 --- a/packages/ipfs-unixfs-exporter/src/types.ts +++ b/packages/ipfs-unixfs-exporter/src/types.ts @@ -2,6 +2,7 @@ import type { CID } from 'multiformats/cid' import type { UnixFS } from 'ipfs-unixfs' import type { PBNode } from '@ipld/dag-pb' import type { Blockstore } from 'interface-blockstore' +import type { Bucket } from 'hamt-sharding' export interface ExporterOptions { offset?: number @@ -68,3 +69,9 @@ export type UnixfsV1FileContent = AsyncIterable | Iterable | Iterable export type UnixfsV1Content = UnixfsV1FileContent | UnixfsV1DirectoryContent export interface UnixfsV1Resolver { (cid: CID, node: PBNode, unixfs: UnixFS, path: string, resolve: Resolve, depth: number, blockstore: Blockstore): (options: ExporterOptions) => UnixfsV1Content } + +export interface ShardTraversalContext { + hamtDepth: number + rootBucket: Bucket + lastBucket: Bucket +} diff --git a/packages/ipfs-unixfs-exporter/src/utils/extract-data-from-block.js b/packages/ipfs-unixfs-exporter/src/utils/extract-data-from-block.js index c5d715be..f727d92d 100644 --- a/packages/ipfs-unixfs-exporter/src/utils/extract-data-from-block.js +++ b/packages/ipfs-unixfs-exporter/src/utils/extract-data-from-block.js @@ -16,12 +16,12 @@ function extractDataFromBlock (block, blockStart, requestedStart, requestedEnd) if (requestedEnd >= blockStart && requestedEnd < blockEnd) { // If the end byte is in the current block, truncate the block to the end byte - block = block.slice(0, requestedEnd - blockStart) + block = block.subarray(0, requestedEnd - blockStart) } if (requestedStart >= blockStart && requestedStart < blockEnd) { // If the start byte is in the current block, skip to the start byte - block = block.slice(requestedStart - blockStart) + block = block.subarray(requestedStart - blockStart) } return block diff --git a/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.js b/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.js index 6945690c..d089532d 100644 --- a/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.js +++ b/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.js @@ -63,7 +63,7 @@ const toPrefix = (position) => { } /** - * @param {import('hamt-sharding').Bucket.BucketPosition} position + * @param {import('hamt-sharding').BucketPosition} position */ const toBucketPath = (position) => { let bucket = position.bucket @@ -81,10 +81,7 @@ const toBucketPath = (position) => { } /** - * @typedef {object} ShardTraversalContext - * @property {number} hamtDepth - * @property {Bucket} rootBucket - * @property {Bucket} lastBucket + * @typedef {import('../types').ShardTraversalContext} ShardTraversalContext * * @param {PBNode} node * @param {string} name diff --git a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.js b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.js index b4e6bfd9..6b2738dd 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.js +++ b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.js @@ -1,5 +1,5 @@ /* eslint-env mocha */ -import { expect } from 'aegir/utils/chai.js' +import { expect } from 'aegir/chai' import { UnixFS } from 'ipfs-unixfs' import all from 'it-all' import last from 'it-last' diff --git a/packages/ipfs-unixfs-exporter/test/exporter-subtree.spec.js b/packages/ipfs-unixfs-exporter/test/exporter-subtree.spec.js index 55119885..86ffc82d 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter-subtree.spec.js +++ b/packages/ipfs-unixfs-exporter/test/exporter-subtree.spec.js @@ -1,5 +1,5 @@ /* eslint-env mocha */ -import { expect } from 'aegir/utils/chai.js' +import { expect } from 'aegir/chai' import { importer } from 'ipfs-unixfs-importer' import all from 'it-all' import last from 'it-last' diff --git a/packages/ipfs-unixfs-exporter/test/exporter.spec.js b/packages/ipfs-unixfs-exporter/test/exporter.spec.js index 9c1de742..e54cbcd4 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter.spec.js +++ b/packages/ipfs-unixfs-exporter/test/exporter.spec.js @@ -1,6 +1,6 @@ /* eslint-env mocha */ -import { expect } from 'aegir/utils/chai.js' +import { expect } from 'aegir/chai' import { UnixFS } from 'ipfs-unixfs' import { CID } from 'multiformats/cid' import * as dagPb from '@ipld/dag-pb' @@ -20,6 +20,7 @@ import { concat as uint8ArrayConcat } from 'uint8arrays/concat' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import asAsyncIterable from './helpers/as-async-iterable.js' +import delay from 'delay' const ONE_MEG = Math.pow(1024, 2) @@ -345,6 +346,37 @@ describe('exporter', () => { expect(data).to.deep.equal(result.file.data.slice(offset, offset + length)) }) + it('exports a file in lots of blocks and a slow blockstore', async function () { + this.timeout(30 * 1000) + + const data = Uint8Array.from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]) + + const cid = await addTestFile({ + file: data, + maxChunkSize: 2 + }) + + /** @type {import('interface-blockstore').Blockstore} */ + const blockStore = { + ...block, + async get (cid, opts) { + await delay(Math.random() * 10) + + return block.get(cid, opts) + } + } + + const file = await exporter(cid, blockStore) + + if (file.type !== 'file') { + throw new Error('Unexpected type') + } + + const bytes = uint8ArrayConcat(await all(file.content())) + + expect(data).to.equalBytes(bytes) + }) + it('exports a large file > 5mb', async function () { this.timeout(30 * 1000) @@ -654,6 +686,14 @@ describe('exporter', () => { expect(data).to.deep.equal(bigFile) }) + it('reads an empty file', async () => { + const data = await addAndReadTestFile({ + file: new Uint8Array() + }) + + expect(data).to.have.property('byteLength', 0) + }) + it('returns an empty stream for dir', async () => { const imported = await first(importer([{ path: 'empty' @@ -887,7 +927,8 @@ describe('exporter', () => { ) }) - it('exports file with data on internal and leaf nodes with an offset that only fetches data from leaf nodes', async () => { + // this is not in the spec? + it.skip('exports file with data on internal and leaf nodes with an offset that only fetches data from leaf nodes', async () => { const leaf = await createAndPersistNode('raw', [0x04, 0x05, 0x06, 0x07], []) const node = await createAndPersistNode('file', [0x00, 0x01, 0x02, 0x03], [ leaf diff --git a/packages/ipfs-unixfs-exporter/test/import-export-dir-sharding.spec.js b/packages/ipfs-unixfs-exporter/test/import-export-dir-sharding.spec.js index cb7c309b..c0b5e9c8 100644 --- a/packages/ipfs-unixfs-exporter/test/import-export-dir-sharding.spec.js +++ b/packages/ipfs-unixfs-exporter/test/import-export-dir-sharding.spec.js @@ -2,7 +2,7 @@ import { importer } from 'ipfs-unixfs-importer' import { exporter } from '../src/index.js' -import { expect } from 'aegir/utils/chai.js' +import { expect } from 'aegir/chai' import all from 'it-all' import last from 'it-last' import blockApi from './helpers/block.js' diff --git a/packages/ipfs-unixfs-exporter/test/import-export-nested-dir.spec.js b/packages/ipfs-unixfs-exporter/test/import-export-nested-dir.spec.js index 746e4971..f465a3d2 100644 --- a/packages/ipfs-unixfs-exporter/test/import-export-nested-dir.spec.js +++ b/packages/ipfs-unixfs-exporter/test/import-export-nested-dir.spec.js @@ -1,5 +1,5 @@ /* eslint-env mocha */ -import { expect } from 'aegir/utils/chai.js' +import { expect } from 'aegir/chai' import all from 'it-all' import { importer } from 'ipfs-unixfs-importer' import { exporter } from '../src/index.js' diff --git a/packages/ipfs-unixfs-exporter/test/import-export.spec.js b/packages/ipfs-unixfs-exporter/test/import-export.spec.js index 12644ef8..7e03ce18 100644 --- a/packages/ipfs-unixfs-exporter/test/import-export.spec.js +++ b/packages/ipfs-unixfs-exporter/test/import-export.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ /* eslint max-nested-callbacks: ["error", 5] */ -import { expect } from 'aegir/utils/chai.js' -import loadFixture from 'aegir/utils/fixtures.js' +import { expect } from 'aegir/chai' +import loadFixture from 'aegir/fixtures' import blockApi from './helpers/block.js' import asAsyncIterable from './helpers/as-async-iterable.js' diff --git a/packages/ipfs-unixfs-exporter/test/importer.spec.js b/packages/ipfs-unixfs-exporter/test/importer.spec.js index 192557fd..66def5d5 100644 --- a/packages/ipfs-unixfs-exporter/test/importer.spec.js +++ b/packages/ipfs-unixfs-exporter/test/importer.spec.js @@ -3,11 +3,11 @@ import { importer } from 'ipfs-unixfs-importer' import { exporter, recursive } from '../src/index.js' import extend from 'merge-options' -import { expect } from 'aegir/utils/chai.js' +import { expect } from 'aegir/chai' import sinon from 'sinon' import { UnixFS, parseMtime } from 'ipfs-unixfs' import collectLeafCids from './helpers/collect-leaf-cids.js' -import loadFixture from 'aegir/utils/fixtures.js' +import loadFixture from 'aegir/fixtures' import all from 'it-all' import first from 'it-first' import blockApi from './helpers/block.js' @@ -1107,7 +1107,7 @@ describe('configuration', () => { path: 'path', content: 'content' }], block, { - /** @type {import('ipfs-unixfs-importer/src/types').DAGBuilder} */ + /** @type {import('ipfs-unixfs-importer').DAGBuilder} */ dagBuilder: async function * (source, block, opts) { // eslint-disable-line require-await yield function () { return Promise.resolve({ @@ -1118,7 +1118,7 @@ describe('configuration', () => { }) } }, - /** @type {import('ipfs-unixfs-importer/src/types').TreeBuilder} */ + /** @type {import('ipfs-unixfs-importer').TreeBuilder} */ treeBuilder: async function * (source, block, opts) { // eslint-disable-line require-await builtTree = true yield * source diff --git a/packages/ipfs-unixfs-exporter/tsconfig.json b/packages/ipfs-unixfs-exporter/tsconfig.json index c490e3c1..7665b90f 100644 --- a/packages/ipfs-unixfs-exporter/tsconfig.json +++ b/packages/ipfs-unixfs-exporter/tsconfig.json @@ -1,18 +1,13 @@ { "extends": "aegir/src/config/tsconfig.aegir.json", "compilerOptions": { - "outDir": "types", - "module": "es2020", - "importsNotUsedAsValues": "preserve" + "outDir": "dist", + "emitDeclarationOnly": true }, "include": [ "src", "test" ], - "exclude": [ - "dist", - "node_modules" - ], "references": [ { "path": "../ipfs-unixfs" diff --git a/packages/ipfs-unixfs-importer/.aegir.cjs b/packages/ipfs-unixfs-importer/.aegir.cjs deleted file mode 100644 index 6abe4ea8..00000000 --- a/packages/ipfs-unixfs-importer/.aegir.cjs +++ /dev/null @@ -1,39 +0,0 @@ -'use strict' - -/** @type {import('aegir').Options["build"]["config"]} */ -const buildConfig = { - plugins: [ - { - name: 'node built ins', - setup (build) { - build.onResolve({ filter: /^stream$/ }, () => { - return { path: require.resolve('readable-stream') } - }) - build.onResolve({ filter: /^crypto$/ }, () => { - return { path: require.resolve('crypto-browserify') } - }) - build.onResolve({ filter: /^cborg$/ }, () => { - return { path: require.resolve('cborg') } - }) - } - } - ] -} - -/** @type {import('aegir').PartialOptions} */ -module.exports = { - build: { - bundlesizeMax: '44KB', - config: buildConfig - }, - test: { - browser: { - config: { - buildConfig - } - } - }, - ts: { - copyTo: 'types' - } -} diff --git a/packages/ipfs-unixfs-importer/.aegir.js b/packages/ipfs-unixfs-importer/.aegir.js new file mode 100644 index 00000000..35115765 --- /dev/null +++ b/packages/ipfs-unixfs-importer/.aegir.js @@ -0,0 +1,7 @@ + +/** @type {import('aegir').PartialOptions} */ +export default { + build: { + bundlesizeMax: '43KB' + } +} diff --git a/packages/ipfs-unixfs-importer/CHANGELOG.md b/packages/ipfs-unixfs-importer/CHANGELOG.md index dcae67b8..3f5a5df2 100644 --- a/packages/ipfs-unixfs-importer/CHANGELOG.md +++ b/packages/ipfs-unixfs-importer/CHANGELOG.md @@ -1,3 +1,41 @@ +## [ipfs-unixfs-importer-v10.0.2](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-v10.0.1...ipfs-unixfs-importer-v10.0.2) (2022-08-31) + + +### Bug Fixes + +* specify return type of the importer to generate correct types ([#251](https://github.com/ipfs/js-ipfs-unixfs/issues/251)) ([3343366](https://github.com/ipfs/js-ipfs-unixfs/commit/33433660a1f762d3e9b9a431cfb956136821cfb1)), closes [#214](https://github.com/ipfs/js-ipfs-unixfs/issues/214) + +## [ipfs-unixfs-importer-v10.0.1](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-v10.0.0...ipfs-unixfs-importer-v10.0.1) (2022-08-16) + + +### Bug Fixes + +* update types import path and deps ([#248](https://github.com/ipfs/js-ipfs-unixfs/issues/248)) ([2edd327](https://github.com/ipfs/js-ipfs-unixfs/commit/2edd327bbe25880a83a8ea00963a22e3f9fa4449)) + +## [ipfs-unixfs-importer-v10.0.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-v9.0.10...ipfs-unixfs-importer-v10.0.0) (2022-08-11) + + +### ⚠ BREAKING CHANGES + +* This module used to be dual published as CJS/ESM, now it is just ESM + +### Dependencies + +* update aegir to 37.x.x ([#243](https://github.com/ipfs/js-ipfs-unixfs/issues/243)) ([9fccb7c](https://github.com/ipfs/js-ipfs-unixfs/commit/9fccb7c4bf989c052ba67ab2c9280273aecfda36)) +* update hamt-sharding to 3.0.0 ([#244](https://github.com/ipfs/js-ipfs-unixfs/issues/244)) ([50a7607](https://github.com/ipfs/js-ipfs-unixfs/commit/50a7607d1876dab0b34dfe79704688f01ce1d449)) + +## [ipfs-unixfs-importer-v9.0.10](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-v9.0.9...ipfs-unixfs-importer-v9.0.10) (2022-05-27) + + +### Bug Fixes + +* remove typesVersions from package.json ([#219](https://github.com/ipfs/js-ipfs-unixfs/issues/219)) ([465670e](https://github.com/ipfs/js-ipfs-unixfs/commit/465670eab2e707b14b14047e2da4ede23590196e)), closes [#214](https://github.com/ipfs/js-ipfs-unixfs/issues/214) [#161](https://github.com/ipfs/js-ipfs-unixfs/issues/161) [#214](https://github.com/ipfs/js-ipfs-unixfs/issues/214) + + +### Trivial Changes + +* update dep-check command ([#221](https://github.com/ipfs/js-ipfs-unixfs/issues/221)) ([5802bd3](https://github.com/ipfs/js-ipfs-unixfs/commit/5802bd366768e1a024fad30e4190aed866f5c9ec)) + ## [ipfs-unixfs-importer-v9.0.9](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-importer-v9.0.8...ipfs-unixfs-importer-v9.0.9) (2022-05-27) diff --git a/packages/ipfs-unixfs-importer/README.md b/packages/ipfs-unixfs-importer/README.md index 0697cf25..be2cac5b 100644 --- a/packages/ipfs-unixfs-importer/README.md +++ b/packages/ipfs-unixfs-importer/README.md @@ -1,33 +1,34 @@ # ipfs-unixfs-importer -[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) -[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) -[![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) -[![Build Status](https://flat.badgen.net/travis/ipfs/js-ipfs-unixfs)](https://travis-ci.com/ipfs/js-ipfs-unixfs) -[![Codecov](https://codecov.io/gh/ipfs/js-ipfs-unixfs/branch/master/graph/badge.svg)](https://codecov.io/gh/ipfs/js-ipfs-unixfs) +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-ipfs-unixfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-ipfs-unixfs) +[![CI](https://img.shields.io/github/workflow/status/ipfs/js-ipfs-unixfs/test%20&%20maybe%20release/master?style=flat-square)](https://github.com/ipfs/js-ipfs-unixfs/actions/workflows/js-test-and-release.yml) -> JavaScript implementation of the layout and chunking mechanisms used by IPFS to handle Files +> JavaScript implementation of the UnixFs importer used by IPFS -## Lead Maintainer - -[Alex Potsides](https://github.com/achingbrain) - -## Table of Contents +## Table of contents - [Install](#install) +- [Lead Maintainer ](#lead-maintainer----omit-in-toc---) - [Usage](#usage) - [Example](#example) - [API](#api) - - [const stream = importer(source, blockstore [, options])](#const-stream--importersource-blockstore--options) + - [const stream = importer(source, blockstore \[, options\])](#const-stream--importersource-blockstore--options) - [Overriding internals](#overriding-internals) - [Contribute](#contribute) - [License](#license) +- [Contribute](#contribute-1) ## Install +```console +$ npm i ipfs-unixfs-importer ``` -> npm install ipfs-unixfs-importer -``` + +## Lead Maintainer + +[Alex Potsides](https://github.com/achingbrain) ## Usage @@ -96,7 +97,7 @@ When run, metadata about DAGNodes in the created tree is printed until the root: import { importer } from 'ipfs-unixfs-importer' ``` -#### const stream = importer(source, blockstore [, options]) +#### const stream = importer(source, blockstore \[, options]) The `importer` function returns an async iterator takes a source async iterator that yields objects of the form: @@ -137,7 +138,7 @@ The input's file paths and directory structure will be preserved in the [`dag-pb - `progress` (function): a function that will be called with the byte length of chunks as a file is added to ipfs. - `onlyHash` (boolean, defaults to false): Only chunk and hash - do not write to disk - `hashAlg` (string): multihash hashing algorithm to use -- `cidVersion` (integer, default 0): the CID version to use when storing the data (storage keys are based on the CID, _including_ it's version) +- `cidVersion` (integer, default 0): the CID version to use when storing the data (storage keys are based on the CID, *including* it's version) - `rawLeaves` (boolean, defaults to false): When a file would span multiple DAGNodes, if this is true the leaf nodes will not be wrapped in `UnixFS` protobufs and will instead contain the raw file bytes - `leafType` (string, defaults to `'file'`) what type of UnixFS node leaves should be - can be `'file'` or `'raw'` (ignored when `rawLeaves` is `true`) - `blockWriteConcurrency` (positive integer, defaults to 10) How many blocks to hash and write to the block store concurrently. For small numbers of large files this should be high (e.g. 50). @@ -165,11 +166,6 @@ Several aspects of the importer are overridable by specifying functions as part - This function should read `{ cid, path, unixfs, node }` entries from `source` and place them in a directory structure - It should yield an object with the properties `{ cid, path, unixfs, size }` where `cid` is a `CID`, `path` is a string, `unixfs` is a UnixFS entry and `size` is a `Number`. -[blockstore]: https://github.com/ipfs/js-ipfs-interfaces/tree/master/packages/interface-blockstore#readme -[UnixFS]: https://github.com/ipfs/specs/tree/master/unixfs -[IPLD]: https://github.com/ipld/js-ipld -[CID]: https://github.com/multiformats/js-cid - ## Contribute Feel free to join in. All welcome. Open an [issue](https://github.com/ipfs/js-ipfs-unixfs-importer/issues)! @@ -180,4 +176,27 @@ This repository falls under the IPFS [Code of Conduct](https://github.com/ipfs/c ## License -[MIT](LICENSE) +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-ipfs-unixfs/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) + +[blockstore]: https://github.com/ipfs/js-ipfs-interfaces/tree/master/packages/interface-blockstore#readme + +[UnixFS]: https://github.com/ipfs/specs/tree/master/unixfs + +[IPLD]: https://github.com/ipld/js-ipld + +[CID]: https://github.com/multiformats/js-cid diff --git a/packages/ipfs-unixfs-importer/package.json b/packages/ipfs-unixfs-importer/package.json index 44e359c5..6427b0b9 100644 --- a/packages/ipfs-unixfs-importer/package.json +++ b/packages/ipfs-unixfs-importer/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs-importer", - "version": "9.0.6", + "version": "10.0.2", "description": "JavaScript implementation of the UnixFs importer used by IPFS", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/master/packages/ipfs-unixfs-importer#readme", @@ -19,15 +19,32 @@ "npm": ">=7.0.0" }, "type": "module", - "types": "types/src/index.d.ts", + "types": "./dist/src/index.d.ts", + "typesVersions": { + "*": { + "*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ], + "src/*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ] + } + }, "files": [ - "*", - "!**/*.tsbuildinfo", - "!**/browser-test", - "!**/node-test" + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" ], "exports": { ".": { + "types": "./dist/src/index.d.ts", "import": "./src/index.js" } }, @@ -64,15 +81,15 @@ "release": "patch" }, { - "type": "chore", + "type": "docs", "release": "patch" }, { - "type": "docs", + "type": "test", "release": "patch" }, { - "type": "test", + "type": "deps", "release": "patch" }, { @@ -102,7 +119,11 @@ }, { "type": "docs", - "section": "Trivial Changes" + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" }, { "type": "test", @@ -113,65 +134,47 @@ } ], "@semantic-release/changelog", - [ - "@semantic-release/npm", - { - "pkgRoot": "dist" - } - ], + "@semantic-release/npm", "@semantic-release/github", "@semantic-release/git" ] }, "scripts": { - "prepare": "aegir build --no-bundle && cp -R types dist", - "pretest": "aegir build --esm-tests", "test": "aegir test", "test:node": "aegir test -t node --cov", "test:chrome": "aegir test -t browser --cov", "test:firefox": "aegir test -t browser -- --browser firefox", - "build": "aegir build && cp -R types dist", - "preleaseOnly": "npx json -I -f dist/package.json -e this.types='\"src/index.d.ts\"'", - "clean": "rimraf ./dist ./types", - "lint": "aegir ts -p check && aegir lint", - "dep-check": "aegir dep-check -i rimraf -i copy -i util -i crypto-browserify -i events -i readable-stream -i assert -i interface-blockstore", - "release": "semantic-release" + "build": "aegir build", + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check -i interface-blockstore", + "release": "aegir release" }, "dependencies": { - "@ipld/dag-pb": "^2.0.2", - "@multiformats/murmur3": "^1.0.3", - "bl": "^5.0.0", + "@ipld/dag-pb": "^3.0.0", + "@multiformats/murmur3": "^2.0.0", "err-code": "^3.0.1", - "hamt-sharding": "^2.0.0", - "interface-blockstore": "^2.0.3", - "ipfs-unixfs": "^6.0.0", - "it-all": "^1.0.5", - "it-batch": "^1.0.8", - "it-first": "^1.0.6", - "it-parallel-batch": "^1.0.9", + "hamt-sharding": "^3.0.0", + "interface-blockstore": "^3.0.0", + "ipfs-unixfs": "^7.0.0", + "it-all": "^2.0.0", + "it-batch": "^2.0.0", + "it-first": "^2.0.0", + "it-parallel-batch": "^2.0.0", "merge-options": "^3.0.4", - "multiformats": "^9.4.2", + "multiformats": "^10.0.0", "rabin-wasm": "^0.1.4", - "uint8arrays": "^3.0.0" + "uint8arraylist": "^2.3.3", + "uint8arrays": "^4.0.2" }, "devDependencies": { - "aegir": "^36.2.3", + "aegir": "^37.5.0", "assert": "^2.0.0", - "blockstore-core": "^1.0.5", - "copy": "^0.3.2", - "crypto-browserify": "^3.12.0", - "events": "^3.3.0", - "it-buffer-stream": "^2.0.0", - "readable-stream": "^3.6.0", - "rimraf": "^3.0.2", - "util": "^0.12.3", - "wherearewe": "^1.0.0" + "blockstore-core": "^2.0.1", + "it-buffer-stream": "^3.0.0", + "wherearewe": "^2.0.1" }, "browser": { "fs": false - }, - "main": "src/index.js", - "publishConfig": { - "directory": "dist" } } diff --git a/packages/ipfs-unixfs-importer/src/chunker/fixed-size.js b/packages/ipfs-unixfs-importer/src/chunker/fixed-size.js index fe537399..874e896f 100644 --- a/packages/ipfs-unixfs-importer/src/chunker/fixed-size.js +++ b/packages/ipfs-unixfs-importer/src/chunker/fixed-size.js @@ -1,32 +1,31 @@ -// @ts-ignore -import BufferList from 'bl/BufferList.js' +import { Uint8ArrayList } from 'uint8arraylist' /** * @type {import('../types').Chunker} */ async function * fixedSizeChunker (source, options) { - let bl = new BufferList() + let list = new Uint8ArrayList() let currentLength = 0 let emitted = false const maxChunkSize = options.maxChunkSize for await (const buffer of source) { - bl.append(buffer) + list.append(buffer) currentLength += buffer.length while (currentLength >= maxChunkSize) { - yield bl.slice(0, maxChunkSize) + yield list.slice(0, maxChunkSize) emitted = true // throw away consumed bytes - if (maxChunkSize === bl.length) { - bl = new BufferList() + if (maxChunkSize === list.length) { + list = new Uint8ArrayList() currentLength = 0 } else { - const newBl = new BufferList() - newBl.append(bl.shallowSlice(maxChunkSize)) - bl = newBl + const newBl = new Uint8ArrayList() + newBl.append(list.sublist(maxChunkSize)) + list = newBl // update our offset currentLength -= maxChunkSize @@ -36,7 +35,7 @@ async function * fixedSizeChunker (source, options) { if (!emitted || currentLength) { // return any remaining bytes or an empty buffer - yield bl.slice(0, currentLength) + yield list.subarray(0, currentLength) } } diff --git a/packages/ipfs-unixfs-importer/src/chunker/rabin.js b/packages/ipfs-unixfs-importer/src/chunker/rabin.js index 539b878d..9395466c 100644 --- a/packages/ipfs-unixfs-importer/src/chunker/rabin.js +++ b/packages/ipfs-unixfs-importer/src/chunker/rabin.js @@ -1,5 +1,4 @@ -// @ts-ignore -import BufferList from 'bl/BufferList.js' +import { Uint8ArrayList } from 'uint8arraylist' // @ts-ignore import { create } from 'rabin-wasm' import errcode from 'err-code' @@ -65,7 +64,7 @@ export default rabinChunker */ async function * rabin (source, options) { const r = await create(options.bits, options.min, options.max, options.window) - const buffers = new BufferList() + const buffers = new Uint8ArrayList() for await (const chunk of source) { buffers.append(chunk) @@ -82,6 +81,6 @@ async function * rabin (source, options) { } if (buffers.length) { - yield buffers.slice(0) + yield buffers.subarray(0) } } diff --git a/packages/ipfs-unixfs-importer/src/index.js b/packages/ipfs-unixfs-importer/src/index.js index 522aefac..80044289 100644 --- a/packages/ipfs-unixfs-importer/src/index.js +++ b/packages/ipfs-unixfs-importer/src/index.js @@ -25,6 +25,7 @@ import treeBuilderFn from './tree-builder.js' * @param {AsyncIterable | Iterable | ImportCandidate} source * @param {Blockstore} blockstore * @param {UserImporterOptions} options + * @returns {AsyncGenerator} */ export async function * importer (source, blockstore, options = {}) { const opts = defaultOptions(options) diff --git a/packages/ipfs-unixfs-importer/src/types.ts b/packages/ipfs-unixfs-importer/src/types.ts index 9474a238..84779d52 100644 --- a/packages/ipfs-unixfs-importer/src/types.ts +++ b/packages/ipfs-unixfs-importer/src/types.ts @@ -1,5 +1,5 @@ import type { UnixFS, Mtime } from 'ipfs-unixfs' -import type { CID, CIDVersion } from 'multiformats/cid' +import type { CID, Version as CIDVersion } from 'multiformats/cid' import type { MultihashHasher } from 'multiformats/hashes/interface' import type { BlockCodec } from 'multiformats/codecs/interface' import type { Blockstore } from 'interface-blockstore' diff --git a/packages/ipfs-unixfs-importer/test/builder-balanced.spec.js b/packages/ipfs-unixfs-importer/test/builder-balanced.spec.js index 8403ea30..dc9d5ba6 100644 --- a/packages/ipfs-unixfs-importer/test/builder-balanced.spec.js +++ b/packages/ipfs-unixfs-importer/test/builder-balanced.spec.js @@ -1,5 +1,5 @@ /* eslint-env mocha */ -import { expect } from 'aegir/utils/chai.js' +import { expect } from 'aegir/chai' import builder from '../src/dag-builder/file/balanced.js' import { CID } from 'multiformats/cid' import defaultOptions from '../src/options.js' diff --git a/packages/ipfs-unixfs-importer/test/builder-flat.spec.js b/packages/ipfs-unixfs-importer/test/builder-flat.spec.js index 032ada55..0a95bc48 100644 --- a/packages/ipfs-unixfs-importer/test/builder-flat.spec.js +++ b/packages/ipfs-unixfs-importer/test/builder-flat.spec.js @@ -1,6 +1,6 @@ /* eslint-env mocha */ -import { expect } from 'aegir/utils/chai.js' +import { expect } from 'aegir/chai' import builder from '../src/dag-builder/file/flat.js' /** diff --git a/packages/ipfs-unixfs-importer/test/builder-only-hash.spec.js b/packages/ipfs-unixfs-importer/test/builder-only-hash.spec.js index b5dd9167..f10a78b6 100644 --- a/packages/ipfs-unixfs-importer/test/builder-only-hash.spec.js +++ b/packages/ipfs-unixfs-importer/test/builder-only-hash.spec.js @@ -1,6 +1,6 @@ /* eslint-env mocha */ -import { expect } from 'aegir/utils/chai.js' +import { expect } from 'aegir/chai' import builder from '../src/dag-builder/index.js' import all from 'it-all' import blockApi from './helpers/block.js' diff --git a/packages/ipfs-unixfs-importer/test/builder-trickle-dag.spec.js b/packages/ipfs-unixfs-importer/test/builder-trickle-dag.spec.js index f0adf6d4..1b68db12 100644 --- a/packages/ipfs-unixfs-importer/test/builder-trickle-dag.spec.js +++ b/packages/ipfs-unixfs-importer/test/builder-trickle-dag.spec.js @@ -1,6 +1,6 @@ /* eslint-env mocha */ -import { expect } from 'aegir/utils/chai.js' +import { expect } from 'aegir/chai' import builder from '../src/dag-builder/file/trickle.js' import asAsyncIterable from './helpers/as-async-iterable.js' diff --git a/packages/ipfs-unixfs-importer/test/builder.spec.js b/packages/ipfs-unixfs-importer/test/builder.spec.js index f2ea3370..f39fb794 100644 --- a/packages/ipfs-unixfs-importer/test/builder.spec.js +++ b/packages/ipfs-unixfs-importer/test/builder.spec.js @@ -1,5 +1,5 @@ /* eslint-env mocha */ -import { expect } from 'aegir/utils/chai.js' +import { expect } from 'aegir/chai' import * as mh from 'multiformats/hashes/digest' import { sha256, sha512 } from 'multiformats/hashes/sha2' import { decode } from '@ipld/dag-pb' diff --git a/packages/ipfs-unixfs-importer/test/chunker-custom.spec.js b/packages/ipfs-unixfs-importer/test/chunker-custom.spec.js index 55d6d833..96666dfd 100644 --- a/packages/ipfs-unixfs-importer/test/chunker-custom.spec.js +++ b/packages/ipfs-unixfs-importer/test/chunker-custom.spec.js @@ -1,6 +1,6 @@ /* eslint-env mocha */ import { importer } from '../src/index.js' -import { expect } from 'aegir/utils/chai.js' +import { expect } from 'aegir/chai' import * as rawCodec from 'multiformats/codecs/raw' import { sha256 } from 'multiformats/hashes/sha2' diff --git a/packages/ipfs-unixfs-importer/test/chunker-fixed-size.spec.js b/packages/ipfs-unixfs-importer/test/chunker-fixed-size.spec.js index ab446a3e..cab1bc0c 100644 --- a/packages/ipfs-unixfs-importer/test/chunker-fixed-size.spec.js +++ b/packages/ipfs-unixfs-importer/test/chunker-fixed-size.spec.js @@ -1,6 +1,6 @@ /* eslint-env mocha */ import chunker from '../src/chunker/fixed-size.js' -import { expect } from 'aegir/utils/chai.js' +import { expect } from 'aegir/chai' import all from 'it-all' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { concat as uint8ArrayConcat } from 'uint8arrays/concat' diff --git a/packages/ipfs-unixfs-importer/test/chunker-rabin.spec.js b/packages/ipfs-unixfs-importer/test/chunker-rabin.spec.js index e10e85f0..d6f779f5 100644 --- a/packages/ipfs-unixfs-importer/test/chunker-rabin.spec.js +++ b/packages/ipfs-unixfs-importer/test/chunker-rabin.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ import chunker from '../src/chunker/rabin.js' -import { expect } from 'aegir/utils/chai.js' +import { expect } from 'aegir/chai' import all from 'it-all' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { concat as uint8ArrayConcat } from 'uint8arrays/concat' diff --git a/packages/ipfs-unixfs-importer/test/hash-parity-with-go-ipfs.spec.js b/packages/ipfs-unixfs-importer/test/hash-parity-with-go-ipfs.spec.js index 3cc2c5f7..e8eb1ec2 100644 --- a/packages/ipfs-unixfs-importer/test/hash-parity-with-go-ipfs.spec.js +++ b/packages/ipfs-unixfs-importer/test/hash-parity-with-go-ipfs.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ import { importer } from '../src/index.js' -import { expect } from 'aegir/utils/chai.js' +import { expect } from 'aegir/chai' import randomByteStream from './helpers/finite-pseudorandom-byte-stream.js' import first from 'it-first' import blockApi from './helpers/block.js' diff --git a/packages/ipfs-unixfs-importer/test/utils.spec.js b/packages/ipfs-unixfs-importer/test/utils.spec.js index 3943bae6..58a9b9d3 100644 --- a/packages/ipfs-unixfs-importer/test/utils.spec.js +++ b/packages/ipfs-unixfs-importer/test/utils.spec.js @@ -1,6 +1,6 @@ /* eslint-env mocha */ -import { expect } from 'aegir/utils/chai.js' +import { expect } from 'aegir/chai' import toPathComponents from '../src/utils/to-path-components.js' describe('toPathComponents', () => { diff --git a/packages/ipfs-unixfs-importer/tsconfig.json b/packages/ipfs-unixfs-importer/tsconfig.json index c69a39ac..6655a440 100644 --- a/packages/ipfs-unixfs-importer/tsconfig.json +++ b/packages/ipfs-unixfs-importer/tsconfig.json @@ -1,18 +1,13 @@ { "extends": "aegir/src/config/tsconfig.aegir.json", "compilerOptions": { - "outDir": "types", - "module": "es2020", - "importsNotUsedAsValues": "preserve" + "outDir": "dist", + "emitDeclarationOnly": true }, "include": [ "src", "test" ], - "exclude": [ - "dist", - "node_modules" - ], "references": [ { "path": "../ipfs-unixfs" diff --git a/packages/ipfs-unixfs/.aegir.cjs b/packages/ipfs-unixfs/.aegir.js similarity index 58% rename from packages/ipfs-unixfs/.aegir.cjs rename to packages/ipfs-unixfs/.aegir.js index 6622ba0a..e2cfcdb4 100644 --- a/packages/ipfs-unixfs/.aegir.cjs +++ b/packages/ipfs-unixfs/.aegir.js @@ -1,11 +1,7 @@ -'use strict' /** @type {import('aegir').PartialOptions} */ -module.exports = { +export default { build: { bundlesizeMax: '11KB' - }, - ts: { - copyTo: 'types' } } diff --git a/packages/ipfs-unixfs/CHANGELOG.md b/packages/ipfs-unixfs/CHANGELOG.md index 84497cb9..395fe85a 100644 --- a/packages/ipfs-unixfs/CHANGELOG.md +++ b/packages/ipfs-unixfs/CHANGELOG.md @@ -1,3 +1,25 @@ +## [ipfs-unixfs-v8.0.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-v7.0.0...ipfs-unixfs-v8.0.0) (2022-10-19) + + +### ⚠ BREAKING CHANGES + +* CIDs returned are instances from `multiformats@10.x.x` and not `multiformats@9.x.x` + +### Dependencies + +* update multiformats to 10.x.x and all @ipld/* modules ([#265](https://github.com/ipfs/js-ipfs-unixfs/issues/265)) ([7a12568](https://github.com/ipfs/js-ipfs-unixfs/commit/7a1256835d85ab9e4f40dd954217d32fdb241517)) + +## [ipfs-unixfs-v7.0.0](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-v6.0.9...ipfs-unixfs-v7.0.0) (2022-08-11) + + +### ⚠ BREAKING CHANGES + +* This module used to be dual published as CJS/ESM, now it is just ESM + +### Dependencies + +* update aegir to 37.x.x ([#243](https://github.com/ipfs/js-ipfs-unixfs/issues/243)) ([9fccb7c](https://github.com/ipfs/js-ipfs-unixfs/commit/9fccb7c4bf989c052ba67ab2c9280273aecfda36)) + ## [ipfs-unixfs-v6.0.9](https://github.com/ipfs/js-ipfs-unixfs/compare/ipfs-unixfs-v6.0.8...ipfs-unixfs-v6.0.9) (2022-05-27) diff --git a/packages/ipfs-unixfs/README.md b/packages/ipfs-unixfs/README.md index 2b2b759f..65ffd966 100644 --- a/packages/ipfs-unixfs/README.md +++ b/packages/ipfs-unixfs/README.md @@ -1,23 +1,16 @@ -# ipfs-unixfs JavaScript Implementation +# ipfs-unixfs -[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) -[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) -[![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs) -[![Build Status](https://flat.badgen.net/travis/ipfs/js-ipfs-unixfs)](https://travis-ci.com/ipfs/js-ipfs-unixfs) -[![Codecov](https://codecov.io/gh/ipfs/js-ipfs-unixfs/branch/master/graph/badge.svg)](https://codecov.io/gh/ipfs/js-ipfs-unixfs) +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/js-ipfs-unixfs.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-ipfs-unixfs) +[![CI](https://img.shields.io/github/workflow/status/ipfs/js-ipfs-unixfs/test%20&%20maybe%20release/master?style=flat-square)](https://github.com/ipfs/js-ipfs-unixfs/actions/workflows/js-test-and-release.yml) -> JavaScript implementation of IPFS' UnixFS (a Unix FileSystem files representation on top of a MerkleDAG) +> JavaScript implementation of IPFS' unixfs (a Unix FileSystem representation on top of a MerkleDAG) -The UnixFS spec can be found inside the [ipfs/specs repository](http://github.com/ipfs/specs) - -## Lead Maintainer - -[Alex Potsides](https://github.com/achingbrain) - -## Table of Contents +## Table of contents - [Install](#install) - - [npm](#npm) +- [Lead Maintainer ](#lead-maintainer----omit-in-toc---) - [Use in Node.js](#use-in-nodejs) - [Use in a browser with browserify, webpack or any other bundler](#use-in-a-browser-with-browserify-webpack-or-any-other-bundler) - [Use in a browser Using a script tag](#use-in-a-browser-using-a-script-tag) @@ -26,7 +19,7 @@ The UnixFS spec can be found inside the [ipfs/specs repository](http://github.co - [Create a file composed by several blocks](#create-a-file-composed-by-several-blocks) - [Create a directory that contains several files](#create-a-directory-that-contains-several-files) - [API](#api) - - [UnixFS Data Structure](#unixfs-data-structure) + - - [UnixFS Data Structure](#unixfs-data-structure) - [create an unixfs Data element](#create-an-unixfs-data-element) - [add and remove a block size to the block size list](#add-and-remove-a-block-size-to-the-block-size-list) - [get total fileSize](#get-total-filesize) @@ -35,10 +28,19 @@ The UnixFS spec can be found inside the [ipfs/specs repository](http://github.co - [has an mtime been set?](#has-an-mtime-been-set) - [Contribute](#contribute) - [License](#license) +- [Contribute](#contribute-1) ## Install -### npm +```console +$ npm i ipfs-unixfs +``` + +The UnixFS spec can be found inside the [ipfs/specs repository](http://github.com/ipfs/specs) + +## Lead Maintainer + +[Alex Potsides](https://github.com/achingbrain) ```sh > npm i ipfs-unixfs @@ -203,4 +205,19 @@ This repository falls under the IPFS [Code of Conduct](https://github.com/ipfs/c ## License -[MIT](LICENSE) +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-ipfs-unixfs/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/ipfs-unixfs/package.json b/packages/ipfs-unixfs/package.json index 1fa42d8d..12387e46 100644 --- a/packages/ipfs-unixfs/package.json +++ b/packages/ipfs-unixfs/package.json @@ -1,6 +1,6 @@ { "name": "ipfs-unixfs", - "version": "6.0.6", + "version": "8.0.0", "description": "JavaScript implementation of IPFS' unixfs (a Unix FileSystem representation on top of a MerkleDAG)", "license": "Apache-2.0 OR MIT", "homepage": "https://github.com/ipfs/js-ipfs-unixfs/tree/master/packages/ipfs-unixfs#readme", @@ -19,15 +19,32 @@ "npm": ">=7.0.0" }, "type": "module", - "types": "types/src/index.d.ts", + "types": "./dist/src/index.d.ts", + "typesVersions": { + "*": { + "*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ], + "src/*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ] + } + }, "files": [ - "*", - "!**/*.tsbuildinfo", - "!**/browser-test", - "!**/node-test" + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" ], "exports": { ".": { + "types": "./dist/src/index.d.ts", "import": "./src/index.js" } }, @@ -67,15 +84,15 @@ "release": "patch" }, { - "type": "chore", + "type": "docs", "release": "patch" }, { - "type": "docs", + "type": "test", "release": "patch" }, { - "type": "test", + "type": "deps", "release": "patch" }, { @@ -105,7 +122,11 @@ }, { "type": "docs", - "section": "Trivial Changes" + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" }, { "type": "test", @@ -116,48 +137,35 @@ } ], "@semantic-release/changelog", - [ - "@semantic-release/npm", - { - "pkgRoot": "dist" - } - ], + "@semantic-release/npm", "@semantic-release/github", "@semantic-release/git" ] }, "scripts": { - "prepare": "aegir build --no-bundle && cp -R types dist", - "prepare:proto": "pbjs -t static-module -w es6 -r ipfs-unixfs --force-number --no-verify --no-delimited --no-create --no-beautify --no-defaults --lint eslint-disable -o src/unixfs.js ./src/unixfs.proto", - "prepare:proto-types": "pbts -o src/unixfs.d.ts src/unixfs.js", - "pretest": "aegir build --esm-tests", + "generate": "npm run generate:proto && generate:proto-types", + "generate:proto": "pbjs -t static-module -w es6 -r ipfs-unixfs --force-number --no-verify --no-delimited --no-create --no-beautify --no-defaults --lint eslint-disable -o src/unixfs.js ./src/unixfs.proto", + "generate:proto-types": "pbts -o src/unixfs.d.ts src/unixfs.js", "test": "aegir test", "test:node": "aegir test -t node --cov", "test:chrome": "aegir test -t browser --cov", "test:firefox": "aegir test -t browser -- --browser firefox", - "build": "aegir build && cp -R types dist", - "clean": "rimraf ./dist ./types", - "lint": "aegir ts -p check && aegir lint", - "dep-check": "aegir dep-check -i mkdirp -i npm-run-all -i copy -i util", - "release": "semantic-release" + "build": "aegir build", + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "release": "aegir release" }, "dependencies": { "err-code": "^3.0.1", - "protobufjs": "^6.10.2" + "protobufjs": "^7.0.0" }, "devDependencies": { - "aegir": "^36.2.3", - "copy": "^0.3.2", - "mkdirp": "^1.0.4", - "npm-run-all": "^4.1.5", - "uint8arrays": "^3.0.0", - "util": "^0.12.3" + "aegir": "^37.5.0", + "protobufjs-cli": "^1.0.0", + "uint8arrays": "^4.0.2" }, "browser": { "fs": false - }, - "main": "src/index.js", - "publishConfig": { - "directory": "dist" } } diff --git a/packages/ipfs-unixfs/test/unixfs-format.spec.js b/packages/ipfs-unixfs/test/unixfs-format.spec.js index 1dbca2c1..19e2c06b 100644 --- a/packages/ipfs-unixfs/test/unixfs-format.spec.js +++ b/packages/ipfs-unixfs/test/unixfs-format.spec.js @@ -1,9 +1,9 @@ /* eslint-env mocha */ -import { expect } from 'aegir/utils/chai.js' +import { expect } from 'aegir/chai' /** @type {(path: string) => Uint8Array} */ -import loadFixture from 'aegir/utils/fixtures.js' +import loadFixture from 'aegir/fixtures' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { UnixFS } from '../src/index.js' diff --git a/packages/ipfs-unixfs/tsconfig.json b/packages/ipfs-unixfs/tsconfig.json index 7c8a6394..e429c4a3 100644 --- a/packages/ipfs-unixfs/tsconfig.json +++ b/packages/ipfs-unixfs/tsconfig.json @@ -1,15 +1,14 @@ { "extends": "aegir/src/config/tsconfig.aegir.json", "compilerOptions": { - "outDir": "types" + "outDir": "dist", + "emitDeclarationOnly": true }, "include": [ "src", "test" ], "exclude": [ - "dist", - "node_modules", "src/unixfs.js" ] }