feat(request/http): add progress monitor to node http library

feat(request/http): add progress monitor to node http library

manual_tests/upload_new.ts

@@ -22,7 +22,7 @@
 import { S3Uploader } from './../src/lib/api/upload/uploaders/s3';
 import { getFile } from '../src/lib/api/upload';
 
-const createFile = (size = 443200) => Buffer.alloc(size);
+const createFile = (size = 10 * 1024 * 1024) => Buffer.alloc(size).fill('a');
 // Sentry.init({ dsn: 'DSN' });
 
 // const fs = new Client(process.env.API_KEY);
@@ -55,22 +55,22 @@ const createFile = (size = 443200) => Buffer.alloc(size);
 
 (async () => {
   const file = await getFile(createFile());
-  const file1 = await getFile(createFile());
+  // const file1 = await getFile(createFile());
   file.name = 'test.txt';
-
+  console.log('Uploadinf file', file.size);
   // const token = new FsCancelToken();
 
   const u = new S3Uploader({});
   u.setUrl('https://upload.filestackapi.com');
   u.setApikey(process.env.API_KEY);
-  u.setIntegrityCheck(false);
+  // u.setIntegrityCheck(false);
   u.addFile(file);
-  u.addFile(file1);
+  // u.addFile(file1);
 
-  setTimeout(() => {
-    console.log('abort call');
-    u.abort();
-  }, 3000);
+  // setTimeout(() => {
+  //   console.log('abort call');
+  //   u.abort();
+  // }, 3000);
 
   const res = await u.execute().catch((e) => {
     console.log('ERROR', e);

package.json

@@ -32,6 +32,8 @@
     "prepare": "npm run build",
     "publish:beta": "npm run build && node scripts/publish.js --beta",
     "publish": "npm run docs:publish && node scripts/publish.js --current --latest",
+    "cache:clean": "node scripts/cache.js --current --latest",
+    "cache:clean:beta": "node scripts/cache.js --beta",
     "examples": "npm run build && node ./examples/examples.js"
   },
   "engines": {

scripts/publish.js

@@ -41,7 +41,6 @@ const pushOneFileToS3 = (basePath, to) => {
         console.error('Upload ERROR:', err);
         reject(err);
       } else {
-        console.log(`Uploaded: ${path}`);
         resolve(`File: ${file} has been uploaded to: ${to.bucket}/${uploadKey}`);
       }
     });

src/lib/api/store.ts

@@ -70,6 +70,8 @@ export const storeURL = (
     options.cancelToken = cancelToken;
   }
 
+  // @todo change to post and add tags
+
   return FsRequest.get(baseURL.toString(), options).then((res) => {
     if (res.data && res.data.handle) {
       return { ...res.data, mimetype: res.data.type };

src/lib/api/upload/file.ts

@@ -16,6 +16,10 @@
  */
 import { md5, sanitizeName, SanitizeOptions } from './../../utils';
 
+export interface Tags {
+  [key: string]: string;
+}
+
 export interface FileInstance {
   name: string;
   type: string;
@@ -68,6 +72,8 @@ export class File {
 
   public workflows: any[];
 
+  public tags: Tags;
+
   constructor(private readonly _file: FileInstance, private readonly _sanitizeOptions?: SanitizeOptions) {
     this._file.name = sanitizeName(this._file.name, this._sanitizeOptions);
   }
@@ -254,6 +260,7 @@ export class File {
       size: this.size,
       url: this.url,
       handle: this.handle,
+      tags: this.tags,
     };
   }
 }

src/lib/api/upload/types.ts

@@ -16,6 +16,7 @@
  */
 
 import { StoreBaseParams } from './../../filelink';
+import { Tags } from './file';
 import { SanitizeOptions } from './../../utils/index';
 
 export interface UploadOptions {
@@ -72,6 +73,14 @@ export interface UploadOptions {
    * On slower devices it can boost upload performance (disable counting md5 from file parts)
    */
   disableIntegrityCheck?: boolean;
+
+  /**
+   * Define tags to be passed to webhook
+   *
+   * @type {Tags}
+   * @memberof UploadOptions
+   */
+  tags?: Tags;
 }
 
 export type StoreUploadOptions = StoreBaseParams & {

src/lib/api/upload/upload.ts

@@ -23,7 +23,7 @@ import { SanitizeOptions } from './../../utils';
 
 import { UploadOptions, StoreUploadOptions } from '../upload/types';
 import { getFile, InputFile } from './file_tools';
-import { FileState } from './file';
+import { FileState, Tags } from './file';
 import { UploadMode } from './uploaders/abstract';
 import { getValidator, UploadParamsSchema, StoreParamsSchema } from './../../../schema';
 
@@ -49,9 +49,32 @@ const normalizeProgress = (current, last) => {
  * @class Upload
  */
 export class Upload extends EventEmitter {
+
+  /**
+   * Uploader instance
+   *
+   * @private
+   * @type {S3Uploader}
+   * @memberof Upload
+   */
   private uploader: S3Uploader;
 
-  private overwriteFileName;
+  /**
+   * Should we overwrite file name
+   *
+   * @private
+   * @memberof Upload
+   */
+  private overrideFileName;
+
+  /**
+   * Upload tags
+   *
+   * @private
+   * @type {Tags}
+   * @memberof Upload
+   */
+  private tags: Tags = {};
 
   private lastProgress: ProgressEvent = {
     totalBytes: 0,
@@ -79,7 +102,7 @@ export class Upload extends EventEmitter {
     }
 
     if (storeOptions.filename) {
-      this.overwriteFileName = storeOptions.filename;
+      this.overrideFileName = storeOptions.filename;
       delete this.storeOptions.filename;
     }
 
@@ -115,6 +138,10 @@ export class Upload extends EventEmitter {
       this.uploader.setUploadMode(options.intelligent === 'fallback' ? UploadMode.FALLBACK : UploadMode.INTELLIGENT);
     }
 
+    if (options.tags) {
+      this.tags = options.tags;
+    }
+
     this.uploader.on('error', (e) => this.emit('error', e));
     this.uploader.on('progress', this.handleProgress.bind(this));
   }
@@ -168,6 +195,16 @@ export class Upload extends EventEmitter {
     this.uploader.setSecurity(security);
   }
 
+  /**
+   * Set upload tags
+   *
+   * @param {Tags} tags
+   * @memberof Upload
+   */
+  public setTags(tags: Tags) {
+    this.tags = tags;
+  }
+
   /**
    * Upload single file
    *
@@ -178,7 +215,7 @@ export class Upload extends EventEmitter {
   async upload(input: InputFile): Promise<any> {
 
     const f = await getFile(input, this.sanitizerOptions);
-    f.customName = this.overwriteFileName;
+    f.customName = this.overrideFileName;
     this.uploader.addFile(f);
 
     this.startProgressInterval();
@@ -209,7 +246,7 @@ export class Upload extends EventEmitter {
       }
 
       const f = await getFile(input[i], this.sanitizerOptions);
-      f.customName = this.overwriteFileName;
+      f.customName = this.overrideFileName;
       this.uploader.addFile(f);
     }
 

src/lib/api/upload/uploaders/abstract.ts

@@ -17,7 +17,7 @@
 import Debug from 'debug';
 import * as EventEmitter from 'eventemitter3';
 
-import { File } from './../file';
+import { File, Tags } from './../file';
 import { StoreUploadOptions } from './../types';
 import { Security } from './../../../client';
 import { FsRetryConfig } from './../../../request';
@@ -69,6 +69,8 @@ export abstract class UploaderAbstract extends EventEmitter {
   protected retryConfig: FsRetryConfig;
   protected integrityCheck: boolean = true;
 
+  protected tags: Tags = null;
+
   constructor(protected readonly storeOptions: StoreUploadOptions, protected readonly concurrency: number = 3) {
     super();
   }
@@ -89,13 +91,20 @@ export abstract class UploaderAbstract extends EventEmitter {
   }
 
   public setRetryConfig(cfg: FsRetryConfig) {
+    debug(`Set retry config to ${cfg}`);
     this.retryConfig = cfg;
   }
 
   public setUrl(url: string): void {
+    debug(`Set upload url to ${url}`);
     this.url = url;
   }
 
+  public setTags(tags: Tags) {
+    debug(`Set tags to %O`, tags);
+    this.tags = tags;
+  }
+
   /**
    * Set state of checking file integrity
    * @param state

src/lib/api/upload/uploaders/s3.ts

@@ -645,6 +645,7 @@ export class S3Uploader extends UploaderAbstract {
         filename: payload.file.name,
         mimetype: payload.file.type,
         size: payload.file.size,
+        tags: this.tags && this.tags.length ? this.tags : undefined,
         parts: parts.length ? parts : undefined,
       },
       {
@@ -675,6 +676,7 @@ export class S3Uploader extends UploaderAbstract {
         file.url = res.data.url;
         file.container = res.data.container;
         file.key = res.data.key;
+        file.tags = res.data.tags;
         file.workflows = res.data.workflows;
         file.status = res.data.status;
 

src/lib/picker.ts

@@ -19,6 +19,7 @@ import { loadModule, FILESTACK_MODULES } from '@filestack/loader';
 import { FilestackError, FilestackErrorType } from './../filestack_error';
 import { Client } from './client';
 import { FSProgressEvent, UploadOptions, WorkflowConfig } from './api/upload/types';
+import { Tags } from './api/upload/file';
 import { getValidator, PickerParamsSchema } from './../schema';
 
 export interface PickerInstance {
@@ -662,6 +663,8 @@ export interface PickerOptions {
    * It can override global objects like console, error etc. Defaults to `true`.
    */
   useSentryBreadcrumbs?: boolean;
+
+  tags?: Tags;
 }
 
 export interface PickerCropOptions {