Skip to content

Commit

Permalink
fix!: return iterators from synchronous sources (#55)
Browse files Browse the repository at this point in the history
Crossing async boundaries is not free.  If a synchronous iterator
is passed to a function, it should not force the caller to await on
the result.

This PR updates some of the basic `it-*` modules to not create unecessary
async when they don't have to.

The return types of these functions are now derived from the input types
so if you have linting rules that forbid awaiting on non-thenables those
awaits will need to be removed.

BREAKING CHANGE: if you pass a synchronous iterator to a function it will return a synchronous generator in response
  • Loading branch information
achingbrain authored Mar 30, 2023
1 parent 9029b9a commit b6d8422
Show file tree
Hide file tree
Showing 58 changed files with 1,379 additions and 293 deletions.
20 changes: 17 additions & 3 deletions packages/it-all/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,24 @@ For when you need a one-liner to collect iterable values.
```javascript
import all from 'it-all'

// This can also be an iterator, async iterator, generator, etc
const values = [0, 1, 2, 3, 4]
// This can also be an iterator, etc
const values = function * () {
yield * [0, 1, 2, 3, 4]
}

const arr = await all(values)
const arr = all(values)

console.info(arr) // 0, 1, 2, 3, 4
```

Async sources must be awaited:

```javascript
const values = async function * () {
yield * [0, 1, 2, 3, 4]
}

const arr = await all(values())

console.info(arr) // 0, 1, 2, 3, 4
```
Expand Down
23 changes: 21 additions & 2 deletions packages/it-all/src/index.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,32 @@
function isAsyncIterable <T> (thing: any): thing is AsyncIterable<T> {
return thing[Symbol.asyncIterator] != null
}

/**
* Collects all values from an (async) iterable and returns them as an array
*/
export default async function all <T> (source: AsyncIterable<T> | Iterable<T>): Promise<T[]> {
function all <T> (source: Iterable<T>): T[]
function all <T> (source: AsyncIterable<T>): Promise<T[]>
function all <T> (source: AsyncIterable<T> | Iterable<T>): Promise<T[]> | T[] {
if (isAsyncIterable(source)) {
return (async () => {
const arr = []

for await (const entry of source) {
arr.push(entry)
}

return arr
})()
}

const arr = []

for await (const entry of source) {
for (const entry of source) {
arr.push(entry)
}

return arr
}

export default all
19 changes: 17 additions & 2 deletions packages/it-all/test/index.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,26 @@ import { expect } from 'aegir/chai'
import all from '../src/index.js'

describe('it-all', () => {
it('Should collect all entries of an async iterator as an array', async () => {
it('should collect all entries of an iterator as an array', () => {
const values = [0, 1, 2, 3, 4]

const res = await all(values)
const res = all(values)

expect(res).to.not.have.property('then')
expect(res).to.deep.equal(values)
})

it('should collect all entries of an async iterator as an array', async () => {
const values = [0, 1, 2, 3, 4]

const generator = (async function * (): AsyncGenerator<number, void, undefined> {
yield * [0, 1, 2, 3, 4]
})()

const p = all(generator)
expect(p).to.have.property('then').that.is.a('function')

const res = await p
expect(res).to.deep.equal(values)
})
})
20 changes: 18 additions & 2 deletions packages/it-batch/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,27 @@ The final batch may be smaller than the max.
import batch from 'it-batch'
import all from 'it-all'

// This can also be an iterator, async iterator, generator, etc
// This can also be an iterator, generator, etc
const values = [0, 1, 2, 3, 4]
const batchSize = 2

const result = await all(batch(values, batchSize))
const result = all(batch(values, batchSize))

console.info(result) // [0, 1], [2, 3], [4]
```

Async sources must be awaited:

```javascript
import batch from 'it-batch'
import all from 'it-all'

const values = async function * () {
yield * [0, 1, 2, 3, 4]
}
const batchSize = 2

const result = await all(batch(values(), batchSize))

console.info(result) // [0, 1], [2, 3], [4]
```
Expand Down
74 changes: 60 additions & 14 deletions packages/it-batch/src/index.ts
Original file line number Diff line number Diff line change
@@ -1,27 +1,73 @@
function isAsyncIterable <T> (thing: any): thing is AsyncIterable<T> {
return thing[Symbol.asyncIterator] != null
}

/**
* Takes an (async) iterable that emits things and returns an async iterable that
* emits those things in fixed-sized batches
*/
export default async function * batch <T> (source: AsyncIterable<T> | Iterable<T>, size: number = 1): AsyncGenerator<T[], void, undefined> {
let things: T[] = []
function batch <T> (source: Iterable<T>, size?: number): Generator<T[], void, undefined>
function batch <T> (source: AsyncIterable<T> | Iterable<T>, size?: number): AsyncGenerator<T[], void, undefined>
function batch <T> (source: AsyncIterable<T> | Iterable<T>, size: number = 1): Generator<T[], void, undefined> | AsyncGenerator<T[], void, undefined> {
size = Number(size)

if (isAsyncIterable(source)) {
return (async function * () {
let things: T[] = []

if (size < 1) {
size = 1
}

if (size !== Math.round(size)) {
throw new Error('Batch size must be an integer')
}

for await (const thing of source) {
things.push(thing)

if (size < 1) {
size = 1
while (things.length >= size) {
yield things.slice(0, size)

things = things.slice(size)
}
}

while (things.length > 0) {
yield things.slice(0, size)

things = things.slice(size)
}
}())
}

for await (const thing of source) {
things.push(thing)
return (function * () {
let things: T[] = []

while (things.length >= size) {
yield things.slice(0, size)
if (size < 1) {
size = 1
}

things = things.slice(size)
if (size !== Math.round(size)) {
throw new Error('Batch size must be an integer')
}
}

while (things.length > 0) {
yield things.slice(0, size)
for (const thing of source) {
things.push(thing)

things = things.slice(size)
}
while (things.length >= size) {
yield things.slice(0, size)

things = things.slice(size)
}
}

while (things.length > 0) {
yield things.slice(0, size)

things = things.slice(size)
}
}())
}

export default batch
39 changes: 26 additions & 13 deletions packages/it-batch/test/index.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,51 +5,64 @@ import { expect } from 'aegir/chai'
import all from 'it-all'

describe('it-batch', () => {
it('should batch up entries', async () => {
it('should batch up entries', () => {
const values = [0, 1, 2, 3, 4]
const batchSize = 2
const res = await all(batch(values, batchSize))
const gen = batch(values, batchSize)
expect(gen[Symbol.iterator]).to.be.ok()

const res = all(gen)
expect(res).to.deep.equal([[0, 1], [2, 3], [4]])
})

it('should batch up entries without batch size', async () => {
it('should batch up async iterator of entries', async () => {
const values = async function * (): AsyncGenerator<number, void, undefined> {
yield * [0, 1, 2, 3, 4]
}
const batchSize = 2
const gen = batch(values(), batchSize)
expect(gen[Symbol.asyncIterator]).to.be.ok()

const res = await all(gen)
expect(res).to.deep.equal([[0, 1], [2, 3], [4]])
})

it('should batch up entries without batch size', () => {
const values = [0, 1, 2, 3, 4]
const res = await all(batch(values))
const res = all(batch(values))

expect(res).to.deep.equal([[0], [1], [2], [3], [4]])
})

it('should batch up entries with negative batch size', async () => {
it('should batch up entries with negative batch size', () => {
const values = [0, 1, 2, 3, 4]
const batchSize = -1
const res = await all(batch(values, batchSize))
const res = all(batch(values, batchSize))

expect(res).to.deep.equal([[0], [1], [2], [3], [4]])
})

it('should batch up entries with zero batch size', async () => {
it('should batch up entries with zero batch size', () => {
const values = [0, 1, 2, 3, 4]
const batchSize = 0
const res = await all(batch(values, batchSize))
const res = all(batch(values, batchSize))

expect(res).to.deep.equal([[0], [1], [2], [3], [4]])
})

it('should batch up entries with string batch size', async () => {
it('should batch up entries with string batch size', () => {
const values = [0, 1, 2, 3, 4]
const batchSize = '2'
// @ts-expect-error batchSize type is incorrect
const res = await all(batch(values, batchSize))
const res = all(batch(values, batchSize))

expect(res).to.deep.equal([[0, 1], [2, 3], [4]])
})

it('should batch up entries with non-integer batch size', async () => {
it('should throw when batching up entries with non-integer batch size', () => {
const values = [0, 1, 2, 3, 4]
const batchSize = 2.5
const res = await all(batch(values, batchSize))

expect(res).to.deep.equal([[0, 1], [2, 3], [4]])
expect(() => all(batch(values, batchSize))).to.throw('Batch size must be an integer')
})
})
22 changes: 21 additions & 1 deletion packages/it-batched-bytes/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ The final batch may be smaller than the max.
import batch from 'it-batched-bytes'
import all from 'it-all'

// This can also be an iterator, async iterator, generator, etc
// This can also be an iterator, generator, etc
const values = [
Uint8Array.from([0]),
Uint8Array.from([1]),
Expand All @@ -45,6 +45,26 @@ const values = [
]
const batchSize = 2

const result = all(batch(values, { size: batchSize }))

console.info(result) // [0, 1], [2, 3], [4]
```

Async sources must be awaited:

```javascript
import batch from 'it-batched-bytes'
import all from 'it-all'

const values = async function * () {
yield Uint8Array.from([0])
yield Uint8Array.from([1])
yield Uint8Array.from([2])
yield Uint8Array.from([3])
yield Uint8Array.from([4])
}
const batchSize = 2

const result = await all(batch(values, { size: batchSize }))

console.info(result) // [0, 1], [2, 3], [4]
Expand Down
1 change: 0 additions & 1 deletion packages/it-batched-bytes/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,6 @@
"release": "aegir release"
},
"dependencies": {
"it-stream-types": "^1.0.4",
"p-defer": "^4.0.0",
"uint8arraylist": "^2.4.1"
},
Expand Down
Loading

0 comments on commit b6d8422

Please sign in to comment.