Generic filehandle remoteFile refactor

package.json

@@ -53,8 +53,8 @@
     "@types/jest": "^24.0.13",
     "@types/node": "^12.0.4",
     "@types/range-parser": "^1.2.3",
-    "@typescript-eslint/eslint-plugin": "^1.10.2",
-    "@typescript-eslint/parser": "^1.10.2",
+    "@typescript-eslint/eslint-plugin": "^2.3.3",
+    "@typescript-eslint/parser": "^2.3.3",
     "babel-eslint": "^10.0.1",
     "babel-preset-typescript": "^7.0.0-alpha.19",
     "cross-fetch": "^3.0.3",

src/blobFile.ts

@@ -1,7 +1,4 @@
-import { GenericFilehandle, FilehandleOptions } from './filehandle'
-interface Stats {
-  size: number
-}
+import { GenericFilehandle, FilehandleOptions, Stats } from './filehandle'
 
 // Using this you can "await" the file like a normal promise
 // https://blog.shovonhasan.com/using-promises-with-filereader/

src/filehandle.ts

@@ -1,3 +1,14 @@
+export type Fetcher = (input: RequestInfo, init?: RequestInit) => Promise<PolyfilledResponse>
+
+/**
+ * a fetch response object that might have some additional properties
+ * that come from the underlying fetch implementation, such as the
+ * `buffer` method on node-fetch responses.
+ */
+export interface PolyfilledResponse extends Response {
+  buffer?: Function | void
+}
+
 export interface FilehandleOptions {
   /**
    * optional AbortSignal object for aborting the request
@@ -11,7 +22,12 @@ export interface FilehandleOptions {
    * global fetch. if there is no global fetch, and a fetch function is not provided,
    * throws an error.
    */
-  fetch?: Function
+  fetch?: Fetcher
+}
+
+export interface Stats {
+  size: number
+  [key: string]: any
 }
 
 export interface GenericFilehandle {
@@ -23,4 +39,5 @@ export interface GenericFilehandle {
     opts?: FilehandleOptions,
   ): Promise<{ bytesRead: number; buffer: Buffer }>
   readFile(options?: FilehandleOptions | string): Promise<Buffer | string>
+  stat(): Promise<Stats>
 }

src/localFile.ts

@@ -1,6 +1,7 @@
 import { promisify } from 'es6-promisify'
 import { GenericFilehandle, FilehandleOptions } from './filehandle'
-declare var __webpack_require__: any // eslint-disable-line @typescript-eslint/camelcase
+// eslint-disable-next-line @typescript-eslint/camelcase,no-var
+declare var __webpack_require__: any
 
 // don't load fs native module if running in webpacked code
 const fs = typeof __webpack_require__ !== 'function' ? require('fs') : null // eslint-disable-line @typescript-eslint/camelcase
@@ -28,9 +29,9 @@ export default class LocalFile implements GenericFilehandle {
 
   public async read(
     buffer: Buffer,
-    offset: number = 0,
+    offset = 0,
     length: number,
-    position: number = 0,
+    position = 0,
   ): Promise<{ bytesRead: number; buffer: Buffer }> {
     const fetchLength = Math.min(buffer.length - offset, length)
     const ret = await fsRead(await this.getFd(), buffer, offset, fetchLength, position)

src/remoteFile.ts

@@ -1,26 +1,14 @@
 import uri2path from 'file-uri-to-path'
-import { GenericFilehandle, FilehandleOptions } from './filehandle'
+import { GenericFilehandle, FilehandleOptions, Stats, Fetcher, PolyfilledResponse } from './filehandle'
 import { LocalFile } from '.'
 
 const myGlobal = typeof window !== 'undefined' ? window : typeof self !== 'undefined' ? self : { fetch: undefined }
 
-/**
- * a fetch response object that might have some additional properties
- * that come from the underlying fetch implementation, such as the
- * `buffer` method on node-fetch responses.
- */
-interface PolyfilledResponse extends Response {
-  buffer: Function | void
-}
-
-interface Stats {
-  size: number
-}
 export default class RemoteFile implements GenericFilehandle {
-  private url: string
-  private _stat?: Stats
-  private fetch: Function
-  private baseOverrides: any = {}
+  protected url: string
+  protected _stat?: Stats
+  protected fetch?: Fetcher
+  protected baseOverrides: any = {}
 
   private async getBufferFromResponse(response: PolyfilledResponse): Promise<Buffer> {
     if (typeof response.buffer === 'function') {
@@ -44,18 +32,49 @@ export default class RemoteFile implements GenericFilehandle {
       this.read = localFile.read.bind(localFile)
       this.readFile = localFile.readFile.bind(localFile)
       this.stat = localFile.stat.bind(localFile)
-      this.fetch = (): void => {}
       return
     }
 
     const fetch = opts.fetch || (myGlobal.fetch && myGlobal.fetch.bind(myGlobal))
     if (!fetch) {
       throw new TypeError(`no fetch function supplied, and none found in global environment`)
     }
+    this.fetch = fetch
+
     if (opts.overrides) {
       this.baseOverrides = opts.overrides
     }
-    this.fetch = fetch
+  }
+
+  public async getFetch(opts: FilehandleOptions): Promise<PolyfilledResponse> {
+    if (!this.fetch) throw new Error('a fetch function must be available unless using a file:// url')
+    const { headers = {}, signal, overrides = {} } = opts
+    const requestOptions = {
+      headers,
+      method: 'GET',
+      redirect: 'follow',
+      mode: 'cors',
+      signal,
+      ...this.baseOverrides,
+      ...overrides,
+    }
+    const response = await this.fetch(this.url, requestOptions)
+    if (!this._stat) {
+      // try to parse out the size of the remote file
+      if (requestOptions.headers && requestOptions.headers.range) {
+        const contentRange = response.headers.get('content-range')
+        const sizeMatch = /\/(\d+)$/.exec(contentRange || '')
+        if (sizeMatch && sizeMatch[1]) this._stat = { size: parseInt(sizeMatch[1], 10) }
+      } else {
+        const contentLength = response.headers.get('content-length')
+        if (contentLength) this._stat = { size: parseInt(contentLength, 10) }
+      }
+    }
+    return response
+  }
+
+  public async headFetch(): Promise<PolyfilledResponse> {
+    return this.getFetch({ overrides: { method: 'HEAD' } })
   }
 
   public async read(
@@ -65,32 +84,18 @@ export default class RemoteFile implements GenericFilehandle {
     position = 0,
     opts: FilehandleOptions = {},
   ): Promise<{ bytesRead: number; buffer: Buffer }> {
-    const { headers = {}, signal, overrides = {} } = opts
+    opts.headers = opts.headers || {}
     if (length < Infinity) {
-      headers.range = `bytes=${position}-${position + length}`
+      opts.headers.range = `bytes=${position}-${position + length}`
     } else if (length === Infinity && position !== 0) {
-      headers.range = `bytes=${position}-`
+      opts.headers.range = `bytes=${position}-`
     }
 
-    const response = await this.fetch(this.url, {
-      headers,
-      method: 'GET',
-      redirect: 'follow',
-      mode: 'cors',
-      signal,
-      ...this.baseOverrides,
-      ...overrides,
-    })
-
+    const response = await this.getFetch(opts)
     if ((response.status === 200 && position === 0) || response.status === 206) {
       const responseData = await this.getBufferFromResponse(response)
       const bytesCopied = responseData.copy(buffer, offset, 0, Math.min(length, responseData.length))
 
-      // try to parse out the size of the remote file
-      const res = response.headers.get('content-range')
-      const sizeMatch = /\/(\d+)$/.exec(res || '')
-      if (sizeMatch && sizeMatch[1]) this._stat = { size: parseInt(sizeMatch[1], 10) }
-
       return { bytesRead: bytesCopied, buffer }
     }
 
@@ -109,16 +114,7 @@ export default class RemoteFile implements GenericFilehandle {
       opts = options
       delete opts.encoding
     }
-    const { headers = {}, signal, overrides = {} } = opts
-    const response = await this.fetch(this.url, {
-      headers,
-      method: 'GET',
-      redirect: 'follow',
-      mode: 'cors',
-      signal,
-      ...this.baseOverrides,
-      ...overrides,
-    })
+    const response = await this.getFetch(opts)
     if (response.status !== 200) {
       throw Object.assign(new Error(`HTTP ${response.status} fetching ${this.url}`), {
         status: response.status,
@@ -130,11 +126,9 @@ export default class RemoteFile implements GenericFilehandle {
   }
 
   public async stat(): Promise<Stats> {
-    if (!this._stat) {
-      const buf = Buffer.allocUnsafe(10)
-      await this.read(buf, 0, 10, 0)
-      if (!this._stat) throw new Error(`unable to determine size of file at ${this.url}`)
-    }
+    if (!this._stat) await this.headFetch()
+    if (!this._stat) await this.read(Buffer.allocUnsafe(10), 0, 10, 0)
+    if (!this._stat) throw new Error(`unable to determine size of file at ${this.url}`)
     return this._stat
   }
 }

test/remoteFile.test.ts

@@ -1,11 +1,9 @@
 /* eslint-disable @typescript-eslint/explicit-function-return-type */
 
-//@ts-ignore TODO the @types/fetch-mock is confusing, it does have a default export
 import fetchMock from 'fetch-mock'
 import { LocalFile, RemoteFile } from '../src/'
 import tenaciousFetch from 'tenacious-fetch'
 
-//@ts-ignore TODO the @types/range-parser is confusing, it does have a default export
 import rangeParser from 'range-parser'
 fetchMock.config.sendAsJson = false
 
@@ -14,7 +12,6 @@ const getFile = (url: string) => new LocalFile(require.resolve(url.replace('http
 const readBuffer = async (url: string, args: any) => {
   const file = getFile(url)
   const range = rangeParser(10000, args.headers.range)
-  // @ts-ignore
   const { start, end } = range[0]
   const len = end - start
   let buf = Buffer.alloc(len)
@@ -29,6 +26,17 @@ const readBuffer = async (url: string, args: any) => {
 }
 
 const readFile = async (url: string) => {
+  const file = getFile(url)
+  const ret = await file.readFile()
+  return {
+    status: 200,
+    body: ret,
+    headers: { 'content-length': ret.length },
+  }
+}
+
+const readFileNoContentLength = async (url: string, args: any) => {
+  if (args && args.headers && args.headers.range) return readBuffer(url, args)
   const file = getFile(url)
   const ret = await file.readFile()
   return {
@@ -76,7 +84,8 @@ describe('remote file tests', () => {
     const f = new RemoteFile('http://fakehost/test.txt', {
       async fetch(url, opts) {
         const res = await mockedFetch(url, opts)
-        //@ts-ignore
+        // eslint-disable-next-line @typescript-eslint/ban-ts-ignore
+        // @ts-ignore
         res.buffer = 0 // obscure the buffer method to test our arraybuffer parse
         return res
       },
@@ -143,7 +152,8 @@ describe('remote file tests', () => {
     const f = new RemoteFile('http://fakehost/test.txt', {
       async fetch(url, opts) {
         const res = await mockedFetch(url, opts)
-        //@ts-ignore
+        // eslint-disable-next-line @typescript-eslint/ban-ts-ignore
+        // @ts-ignore
         res.buffer = undefined // obscure the buffer method to test our arraybuffer parse
         res.arrayBuffer = undefined // also obscure arrayBuffer
         return res
@@ -162,8 +172,14 @@ describe('remote file tests', () => {
     expect(buf.toString()[0]).toBe('\0')
     expect(res.bytesRead).toEqual(0)
   })
-  it('stat', async () => {
-    fetchMock.mock('http://fakehost/test.txt', readBuffer)
+  it('stat using content-length', async () => {
+    fetchMock.mock('http://fakehost/test.txt', readFile)
+    const f = new RemoteFile('http://fakehost/test.txt')
+    const stat = await f.stat()
+    expect(stat.size).toEqual(8)
+  })
+  it('stat using content-range', async () => {
+    fetchMock.mock('http://fakehost/test.txt', readFileNoContentLength)
     const f = new RemoteFile('http://fakehost/test.txt')
     const stat = await f.stat()
     expect(stat.size).toEqual(8)