From 966e17ef5a00c041be88d9b2fd60cfdd83dffac7 Mon Sep 17 00:00:00 2001 From: uhyo Date: Fri, 22 Sep 2023 13:47:07 +0900 Subject: [PATCH] feat(volume): implement readv and writev (#946) * fix(volume): readBase throws error when given length is larger than buffer size * test(volume): add tests for readSync * feat(volume): add readv and readvSync * feat(volume): add readv and writev * feat(volume): add readv and writev to FileHandle * fix: add methods to the list * fix: support -1 in all read and write methods --- src/__tests__/promises.test.ts | 50 +++++++++- src/__tests__/volume.test.ts | 73 ++++++++++++++ src/__tests__/volume/readSync.test.ts | 33 ++++++- src/node/FileHandle.ts | 18 ++++ src/node/lists/fsCallbackApiList.ts | 2 + src/node/lists/fsSynchronousApiList.ts | 3 +- src/node/types/misc.ts | 12 +++ src/node/util.ts | 3 + src/volume.ts | 127 ++++++++++++++++++++++--- 9 files changed, 304 insertions(+), 17 deletions(-) diff --git a/src/__tests__/promises.test.ts b/src/__tests__/promises.test.ts index e49002799..ffe82ccb2 100644 --- a/src/__tests__/promises.test.ts +++ b/src/__tests__/promises.test.ts @@ -103,8 +103,8 @@ describe('Promises API', () => { }); it('Read data from an existing file', async () => { const fileHandle = await promises.open('/foo', 'r+'); - const buff = Buffer.from('foo'); - const { bytesRead, buffer } = await fileHandle.read(buff, 0, 42, 0); + const buff = Buffer.from('foofoo'); + const { bytesRead, buffer } = await fileHandle.read(buff, 0, 6, 0); expect(bytesRead).toEqual(3); expect(buffer).toBe(buff); await fileHandle.close(); @@ -115,6 +115,30 @@ describe('Promises API', () => { return expect(fileHandle.read(Buffer.from('foo'), 0, 42, 0)).rejects.toBeInstanceOf(Error); }); }); + describe('readv(buffers, position)', () => { + const vol = new Volume(); + const { promises } = vol; + vol.fromJSON({ + '/foo': 'Hello, world!', + }); + it('Read data from an existing file', async () => { + const fileHandle = await promises.open('/foo', 'r+'); + const buf1 = Buffer.alloc(5); + const buf2 = Buffer.alloc(5); + const { bytesRead, buffers } = await fileHandle.readv([buf1, buf2], 0); + expect(bytesRead).toEqual(10); + expect(buffers).toEqual([buf1, buf2]); + expect(buf1.toString()).toEqual('Hello'); + expect(buf2.toString()).toEqual(', wor'); + await fileHandle.close(); + }); + it('Reject when the file handle was closed', async () => { + const fileHandle = await promises.open('/foo', 'r+'); + await fileHandle.close(); + return expect(fileHandle.readv([Buffer.alloc(10)], 0)).rejects.toBeInstanceOf(Error); + }); + }); + describe('readFile([options])', () => { const vol = new Volume(); const { promises } = vol; @@ -243,6 +267,28 @@ describe('Promises API', () => { return expect(fileHandle.write(Buffer.from('foo'))).rejects.toBeInstanceOf(Error); }); }); + describe('writev(buffers[, position])', () => { + const vol = new Volume(); + const { promises } = vol; + vol.fromJSON({ + '/foo': 'Hello, world!', + }); + it('Write data to an existing file', async () => { + const fileHandle = await promises.open('/foo', 'w'); + const buf1 = Buffer.from('foo'); + const buf2 = Buffer.from('bar'); + const { bytesWritten, buffers } = await fileHandle.writev([buf1, buf2], 0); + expect(vol.readFileSync('/foo').toString()).toEqual('foobar'); + expect(bytesWritten).toEqual(6); + expect(buffers).toEqual([buf1, buf2]); + await fileHandle.close(); + }); + it('Reject when the file handle was closed', async () => { + const fileHandle = await promises.open('/foo', 'w'); + await fileHandle.close(); + return expect(fileHandle.writev([Buffer.from('foo')], 0)).rejects.toBeInstanceOf(Error); + }); + }); describe('writeFile(data[, options])', () => { const vol = new Volume(); const { promises } = vol; diff --git a/src/__tests__/volume.test.ts b/src/__tests__/volume.test.ts index 7053715c7..6cd139bc0 100644 --- a/src/__tests__/volume.test.ts +++ b/src/__tests__/volume.test.ts @@ -501,6 +501,63 @@ describe('volume', () => { expect(fn).toThrowError('EPERM'); }); }); + describe('.readv(fd, buffers, position, callback)', () => { + it('Simple read', done => { + const vol = new Volume(); + vol.writeFileSync('/test.txt', 'hello'); + const fd = vol.openSync('/test.txt', 'r'); + + const buf1 = Buffer.alloc(2); + const buf2 = Buffer.alloc(2); + const buf3 = Buffer.alloc(2); + vol.readv(fd, [buf1, buf2, buf3], 0, (err, bytesRead, buffers) => { + expect(err).toBe(null); + expect(bytesRead).toBe(5); + expect(buffers).toEqual([buf1, buf2, buf3]); + expect(buf1.toString()).toBe('he'); + expect(buf2.toString()).toBe('ll'); + expect(buf3.toString()).toBe('o\0'); + done(); + }); + }); + it('Read from position', done => { + const vol = new Volume(); + vol.writeFileSync('/test.txt', 'hello'); + const fd = vol.openSync('/test.txt', 'r'); + + const buf1 = Buffer.alloc(2); + const buf2 = Buffer.alloc(2); + const buf3 = Buffer.alloc(2, 0); + vol.readv(fd, [buf1, buf2, buf3], 1, (err, bytesRead, buffers) => { + expect(err).toBe(null); + expect(bytesRead).toBe(4); + expect(buffers).toEqual([buf1, buf2, buf3]); + expect(buf1.toString()).toBe('el'); + expect(buf2.toString()).toBe('lo'); + expect(buf3.toString()).toBe('\0\0'); + done(); + }); + }); + it('Read from current position', done => { + const vol = new Volume(); + vol.writeFileSync('/test.txt', 'hello, world!'); + const fd = vol.openSync('/test.txt', 'r'); + vol.readSync(fd, Buffer.alloc(3), 0, 3, null); + + const buf1 = Buffer.alloc(2); + const buf2 = Buffer.alloc(2); + const buf3 = Buffer.alloc(2); + vol.readv(fd, [buf1, buf2, buf3], (err, bytesRead, buffers) => { + expect(err).toBe(null); + expect(bytesRead).toBe(6); + expect(buffers).toEqual([buf1, buf2, buf3]); + expect(buf1.toString()).toBe('lo'); + expect(buf2.toString()).toBe(', '); + expect(buf3.toString()).toBe('wo'); + done(); + }); + }); + }); describe('.readFileSync(path[, options])', () => { const vol = new Volume(); const data = 'trololo'; @@ -624,6 +681,22 @@ describe('volume', () => { }); }); }); + describe('.writev(fd, buffers, position, callback)', () => { + it('Simple write to a file descriptor', done => { + const vol = new Volume(); + const fd = vol.openSync('/test.txt', 'w+'); + const data1 = 'Hello'; + const data2 = ', '; + const data3 = 'world!'; + vol.writev(fd, [Buffer.from(data1), Buffer.from(data2), Buffer.from(data3)], 0, (err, bytes) => { + expect(err).toBe(null); + expect(bytes).toBe(data1.length + data2.length + data3.length); + vol.closeSync(fd); + expect(vol.readFileSync('/test.txt', 'utf8')).toBe(data1 + data2 + data3); + done(); + }); + }); + }); describe('.writeFile(path, data[, options], callback)', () => { const vol = new Volume(); const data = 'asdfasidofjasdf'; diff --git a/src/__tests__/volume/readSync.test.ts b/src/__tests__/volume/readSync.test.ts index 80b13a271..2cacc5b4b 100644 --- a/src/__tests__/volume/readSync.test.ts +++ b/src/__tests__/volume/readSync.test.ts @@ -8,8 +8,35 @@ describe('.readSync(fd, buffer, offset, length, position)', () => { expect(bytes).toBe(3); expect(buf.equals(Buffer.from('345'))).toBe(true); }); - xit('Read more than buffer space', () => {}); - xit('Read over file boundary', () => {}); - xit('Read multiple times, caret position should adjust', () => {}); + it('Attempt to read more than buffer space should throw ERR_OUT_OF_RANGE', () => { + const vol = create({ '/test.txt': '01234567' }); + const buf = Buffer.alloc(3, 0); + const fn = () => vol.readSync(vol.openSync('/test.txt', 'r'), buf, 0, 10, 3); + expect(fn).toThrow('ERR_OUT_OF_RANGE'); + }); + it('Read over file boundary', () => { + const vol = create({ '/test.txt': '01234567' }); + const buf = Buffer.alloc(3, 0); + const bytes = vol.readSync(vol.openSync('/test.txt', 'r'), buf, 0, 3, 6); + expect(bytes).toBe(2); + expect(buf.equals(Buffer.from('67\0'))).toBe(true); + }); + it('Read multiple times, caret position should adjust', () => { + const vol = create({ '/test.txt': '01234567' }); + const buf = Buffer.alloc(3, 0); + const fd = vol.openSync('/test.txt', 'r'); + let bytes = vol.readSync(fd, buf, 0, 3, null); + expect(bytes).toBe(3); + expect(buf.equals(Buffer.from('012'))).toBe(true); + bytes = vol.readSync(fd, buf, 0, 3, null); + expect(bytes).toBe(3); + expect(buf.equals(Buffer.from('345'))).toBe(true); + bytes = vol.readSync(fd, buf, 0, 3, null); + expect(bytes).toBe(2); + expect(buf.equals(Buffer.from('675'))).toBe(true); + bytes = vol.readSync(fd, buf, 0, 3, null); + expect(bytes).toBe(0); + expect(buf.equals(Buffer.from('675'))).toBe(true); + }); xit('Negative tests', () => {}); }); diff --git a/src/node/FileHandle.ts b/src/node/FileHandle.ts index 81467d6bd..b6e40dfbe 100644 --- a/src/node/FileHandle.ts +++ b/src/node/FileHandle.ts @@ -37,6 +37,10 @@ export class FileHandle implements IFileHandle { return promisify(this.fs, 'read', bytesRead => ({ bytesRead, buffer }))(this.fd, buffer, offset, length, position); } + readv(buffers: ArrayBufferView[], position?: number | null | undefined): Promise { + return promisify(this.fs, 'readv', bytesRead => ({ bytesRead, buffers }))(this.fd, buffers, position); + } + readFile(options?: opts.IReadFileOptions | string): Promise { return promisify(this.fs, 'readFile')(this.fd, options); } @@ -72,6 +76,10 @@ export class FileHandle implements IFileHandle { ); } + writev(buffers: ArrayBufferView[], position?: number | null | undefined): Promise { + return promisify(this.fs, 'writev', bytesWritten => ({ bytesWritten, buffers }))(this.fd, buffers, position); + } + writeFile(data: TData, options?: opts.IWriteFileOptions): Promise { return promisify(this.fs, 'writeFile')(this.fd, data, options); } @@ -86,3 +94,13 @@ export interface TFileHandleWriteResult { bytesWritten: number; buffer: Buffer | Uint8Array; } + +export interface TFileHandleReadvResult { + bytesRead: number; + buffers: ArrayBufferView[]; +} + +export interface TFileHandleWritevResult { + bytesWritten: number; + buffers: ArrayBufferView[]; +} diff --git a/src/node/lists/fsCallbackApiList.ts b/src/node/lists/fsCallbackApiList.ts index 38cd546bd..9d3773ac7 100644 --- a/src/node/lists/fsCallbackApiList.ts +++ b/src/node/lists/fsCallbackApiList.ts @@ -25,6 +25,7 @@ export const fsCallbackApiList: Array = [ 'mkdtemp', 'open', 'read', + 'readv', 'readdir', 'readFile', 'readlink', @@ -41,5 +42,6 @@ export const fsCallbackApiList: Array = [ 'watch', 'watchFile', 'write', + 'writev', 'writeFile', ]; diff --git a/src/node/lists/fsSynchronousApiList.ts b/src/node/lists/fsSynchronousApiList.ts index b2b4110fe..a02489bb4 100644 --- a/src/node/lists/fsSynchronousApiList.ts +++ b/src/node/lists/fsSynchronousApiList.ts @@ -26,6 +26,7 @@ export const fsSynchronousApiList: Array = [ 'readFileSync', 'readlinkSync', 'readSync', + 'readvSync', 'realpathSync', 'renameSync', 'rmdirSync', @@ -37,9 +38,9 @@ export const fsSynchronousApiList: Array = [ 'utimesSync', 'writeFileSync', 'writeSync', + 'writevSync', // 'cpSync', // 'lutimesSync', // 'statfsSync', - // 'writevSync', ]; diff --git a/src/node/types/misc.ts b/src/node/types/misc.ts index 5a44b9e81..8769ee694 100644 --- a/src/node/types/misc.ts +++ b/src/node/types/misc.ts @@ -130,6 +130,7 @@ export interface IFileHandle { close(): Promise; datasync(): Promise; read(buffer: Buffer | Uint8Array, offset: number, length: number, position: number): Promise; + readv(buffers: ArrayBufferView[], position?: number | null): Promise; readFile(options?: IReadFileOptions | string): Promise; stat(options?: IStatOptions): Promise; truncate(len?: number): Promise; @@ -140,6 +141,7 @@ export interface IFileHandle { length?: number, position?: number, ): Promise; + writev(buffers: ArrayBufferView[], position?: number | null): Promise; writeFile(data: TData, options?: IWriteFileOptions): Promise; } @@ -155,4 +157,14 @@ export interface TFileHandleWriteResult { buffer: Buffer | Uint8Array; } +export interface TFileHandleReadvResult { + bytesRead: number; + buffers: ArrayBufferView[]; +} + +export interface TFileHandleWritevResult { + bytesWritten: number; + buffers: ArrayBufferView[]; +} + export type AssertCallback = T extends () => void ? T : never; diff --git a/src/node/util.ts b/src/node/util.ts index 0fb341794..59c24d805 100644 --- a/src/node/util.ts +++ b/src/node/util.ts @@ -96,6 +96,7 @@ const EISDIR = 'EISDIR'; const ENOTEMPTY = 'ENOTEMPTY'; const ENOSYS = 'ENOSYS'; const ERR_FS_EISDIR = 'ERR_FS_EISDIR'; +const ERR_OUT_OF_RANGE = 'ERR_OUT_OF_RANGE'; function formatError(errorCode: string, func = '', path = '', path2 = '') { let pathFormatted = ''; @@ -129,6 +130,8 @@ function formatError(errorCode: string, func = '', path = '', path2 = '') { return `ENOSYS: function not implemented, ${func}${pathFormatted}`; case ERR_FS_EISDIR: return `[ERR_FS_EISDIR]: Path is a directory: ${func} returned EISDIR (is a directory) ${path}`; + case ERR_OUT_OF_RANGE: + return `[ERR_OUT_OF_RANGE]: value out of range, ${func}${pathFormatted}`; default: return `${errorCode}: error occurred, ${func}${pathFormatted}`; } diff --git a/src/volume.ts b/src/volume.ts index e77ef17a4..433776598 100644 --- a/src/volume.ts +++ b/src/volume.ts @@ -14,7 +14,7 @@ import { FileHandle } from './node/FileHandle'; import * as util from 'util'; import * as misc from './node/types/misc'; import * as opts from './node/types/options'; -import { FsCallbackApi } from './node/types/FsCallbackApi'; +import { FsCallbackApi, WritevCallback } from './node/types/FsCallbackApi'; import { FsPromises } from './node/FsPromises'; import { ToTreeOptions, toTreeSync } from './print'; import { ERRSTR, FLAGS, MODE } from './node/constants'; @@ -57,6 +57,7 @@ import { } from './node/util'; import type { PathLike, symlink } from 'fs'; import type { FsPromisesApi, FsSynchronousApi } from './node/types'; +import { fsSynchronousApiList } from './node/lists/fsSynchronousApiList'; const resolveCrossPlatform = pathModule.resolve; const { @@ -108,6 +109,7 @@ const EISDIR = 'EISDIR'; const ENOTEMPTY = 'ENOTEMPTY'; const ENOSYS = 'ENOSYS'; const ERR_FS_EISDIR = 'ERR_FS_EISDIR'; +const ERR_OUT_OF_RANGE = 'ERR_OUT_OF_RANGE'; // ---------------------------------------- Flags @@ -230,7 +232,7 @@ const notImplemented: (...args: any[]) => any = () => { /** * `Volume` represents a file system. */ -export class Volume implements FsCallbackApi { +export class Volume implements FsCallbackApi, FsSynchronousApi { static fromJSON(json: DirectoryJSON, cwd?: string): Volume { const vol = new Volume(); vol.fromJSON(json, cwd); @@ -761,13 +763,21 @@ export class Volume implements FsCallbackApi { buffer: Buffer | ArrayBufferView | DataView, offset: number, length: number, - position: number, + position: number | null, ): number { + if (buffer.byteLength < length) { + throw createError(ERR_OUT_OF_RANGE, 'read', undefined, undefined, RangeError); + } const file = this.getFileByFdOrThrow(fd); if (file.node.isSymlink()) { throw createError(EPERM, 'read', file.link.getPath()); } - return file.read(buffer, Number(offset), Number(length), position); + return file.read( + buffer, + Number(offset), + Number(length), + position === -1 || typeof position !== 'number' ? undefined : position, + ); } readSync( @@ -775,7 +785,7 @@ export class Volume implements FsCallbackApi { buffer: Buffer | ArrayBufferView | DataView, offset: number, length: number, - position: number, + position: number | null, ): number { validateFd(fd); return this.readBase(fd, buffer, offset, length, position); @@ -786,7 +796,7 @@ export class Volume implements FsCallbackApi { buffer: Buffer | ArrayBufferView | DataView, offset: number, length: number, - position: number, + position: number | null, callback: (err?: Error | null, bytesRead?: number, buffer?: Buffer | ArrayBufferView | DataView) => void, ) { validateCallback(callback); @@ -808,6 +818,60 @@ export class Volume implements FsCallbackApi { }); } + private readvBase(fd: number, buffers: ArrayBufferView[], position: number | null): number { + const file = this.getFileByFdOrThrow(fd); + let p = position ?? undefined; + if (p === -1) { + p = undefined; + } + let bytesRead = 0; + for (const buffer of buffers) { + const bytes = file.read(buffer, 0, buffer.byteLength, p); + p = undefined; + bytesRead += bytes; + if (bytes < buffer.byteLength) break; + } + return bytesRead; + } + + readv(fd: number, buffers: ArrayBufferView[], callback: misc.TCallback2): void; + readv( + fd: number, + buffers: ArrayBufferView[], + position: number | null, + callback: misc.TCallback2, + ): void; + readv( + fd: number, + buffers: ArrayBufferView[], + a: number | null | misc.TCallback2, + b?: misc.TCallback2, + ): void { + let position: number | null = a as number | null; + let callback: misc.TCallback2 = b as misc.TCallback2; + + if (typeof a === 'function') { + position = null; + callback = a; + } + + validateCallback(callback); + + setImmediate(() => { + try { + const bytes = this.readvBase(fd, buffers, position); + callback(null, bytes, buffers); + } catch (err) { + callback(err); + } + }); + } + + readvSync(fd: number, buffers: ArrayBufferView[], position: number | null): number { + validateFd(fd); + return this.readvBase(fd, buffers, position); + } + private readFileBase(id: TFileId, flagsNum: number, encoding: BufferEncoding): Buffer | string { let result: Buffer | string; @@ -859,7 +923,7 @@ export class Volume implements FsCallbackApi { if (file.node.isSymlink()) { throw createError(EBADF, 'write', file.link.getPath()); } - return file.write(buf, offset, length, position); + return file.write(buf, offset, length, position === -1 || typeof position !== 'number' ? undefined : position); } writeSync( @@ -917,6 +981,51 @@ export class Volume implements FsCallbackApi { }); } + private writevBase(fd: number, buffers: ArrayBufferView[], position: number | null): number { + const file = this.getFileByFdOrThrow(fd); + let p = position ?? undefined; + if (p === -1) { + p = undefined; + } + let bytesWritten = 0; + for (const buffer of buffers) { + const nodeBuf = Buffer.from(buffer.buffer, buffer.byteOffset, buffer.byteLength); + const bytes = file.write(nodeBuf, 0, nodeBuf.byteLength, p); + p = undefined; + bytesWritten += bytes; + if (bytes < nodeBuf.byteLength) break; + } + return bytesWritten; + } + + writev(fd: number, buffers: ArrayBufferView[], callback: WritevCallback): void; + writev(fd: number, buffers: ArrayBufferView[], position: number | null, callback: WritevCallback): void; + writev(fd: number, buffers: ArrayBufferView[], a: number | null | WritevCallback, b?: WritevCallback): void { + let position: number | null = a as number | null; + let callback: WritevCallback = b as WritevCallback; + + if (typeof a === 'function') { + position = null; + callback = a; + } + + validateCallback(callback); + + setImmediate(() => { + try { + const bytes = this.writevBase(fd, buffers, position); + callback(null, bytes, buffers); + } catch (err) { + callback(err); + } + }); + } + + writevSync(fd: number, buffers: ArrayBufferView[], position: number | null): number { + validateFd(fd); + return this.writevBase(fd, buffers, position); + } + private writeFileBase(id: TFileId, buf: Buffer, flagsNum: number, modeNum: number) { // console.log('writeFileBase', id, buf, flagsNum, modeNum); // const node = this.getNodeByIdOrCreate(id, flagsNum, modeNum); @@ -1889,15 +1998,11 @@ export class Volume implements FsCallbackApi { public cpSync: FsSynchronousApi['cpSync'] = notImplemented; public lutimesSync: FsSynchronousApi['lutimesSync'] = notImplemented; public statfsSync: FsSynchronousApi['statfsSync'] = notImplemented; - public writevSync: FsSynchronousApi['writevSync'] = notImplemented; - public readvSync: FsSynchronousApi['readvSync'] = notImplemented; public opendirSync: FsSynchronousApi['opendirSync'] = notImplemented; public cp: FsCallbackApi['cp'] = notImplemented; public lutimes: FsCallbackApi['lutimes'] = notImplemented; public statfs: FsCallbackApi['statfs'] = notImplemented; - public writev: FsCallbackApi['writev'] = notImplemented; - public readv: FsCallbackApi['readv'] = notImplemented; public openAsBlob: FsCallbackApi['openAsBlob'] = notImplemented; public opendir: FsCallbackApi['opendir'] = notImplemented; }