Make lz4 module optional (#246)

* Make `lz4` module optional Signed-off-by: Levko Kravets <levko.ne@gmail.com> * Fix lint errors Signed-off-by: Levko Kravets <levko.ne@gmail.com> --------- Signed-off-by: Levko Kravets <levko.ne@gmail.com>
databricks · Apr 19, 2024 · 1fd9e1e · 1fd9e1e
1 parent 38db1b0
commit 1fd9e1e
Show file tree

Hide file tree

Showing 8 changed files with 70 additions and 18 deletions.
diff --git a/.eslintrc b/.eslintrc
@@ -15,7 +15,14 @@
         "@typescript-eslint/no-throw-literal": "off",
         "no-restricted-syntax": "off",
         "no-case-declarations": "off",
-        "max-classes-per-file": "off"
+        "max-classes-per-file": "off",
+        "import/no-extraneous-dependencies": [
+          "error",
+          {
+            "devDependencies": true,
+            "optionalDependencies": true
+          }
+        ]
       }
     }
   ]

diff --git a/lib/DBSQLSession.ts b/lib/DBSQLSession.ts
@@ -29,7 +29,7 @@ import IOperation from './contracts/IOperation';
 import DBSQLOperation from './DBSQLOperation';
 import Status from './dto/Status';
 import InfoValue from './dto/InfoValue';
-import { definedOrError } from './utils';
+import { definedOrError, LZ4 } from './utils';
 import CloseableCollection from './utils/CloseableCollection';
 import { LogLevel } from './contracts/IDBSQLLogger';
 import HiveDriverError from './errors/HiveDriverError';
@@ -190,7 +190,7 @@ export default class DBSQLSession implements IDBSQLSession {
       ...getArrowOptions(clientConfig),
       canDownloadResult: options.useCloudFetch ?? clientConfig.useCloudFetch,
       parameters: getQueryParameters(this.sessionHandle, options.namedParameters, options.ordinalParameters),
-      canDecompressLZ4Result: clientConfig.useLZ4Compression,
+      canDecompressLZ4Result: clientConfig.useLZ4Compression && Boolean(LZ4),
     });
     const response = await this.handleResponse(operationPromise);
     const operation = this.createOperation(response);

diff --git a/lib/result/ArrowResultHandler.ts b/lib/result/ArrowResultHandler.ts
@@ -1,8 +1,9 @@
-import LZ4 from 'lz4';
 import { TGetResultSetMetadataResp, TRowSet } from '../../thrift/TCLIService_types';
+import HiveDriverError from '../errors/HiveDriverError';
 import IClientContext from '../contracts/IClientContext';
 import IResultsProvider, { ResultsProviderFetchNextOptions } from './IResultsProvider';
 import { ArrowBatch, hiveSchemaToArrowSchema } from './utils';
+import { LZ4 } from '../utils';
 
 export default class ArrowResultHandler implements IResultsProvider<ArrowBatch> {
   protected readonly context: IClientContext;
@@ -24,6 +25,10 @@ export default class ArrowResultHandler implements IResultsProvider<ArrowBatch>
     // so it's possible to infer Arrow schema from Hive schema ignoring `useArrowNativeTypes` option
     this.arrowSchema = arrowSchema ?? hiveSchemaToArrowSchema(schema);
     this.isLZ4Compressed = lz4Compressed ?? false;
+
+    if (this.isLZ4Compressed && !LZ4) {
+      throw new HiveDriverError('Cannot handle LZ4 compressed result: module `lz4` not installed');
+    }
   }
 
   public async hasMore() {
@@ -47,7 +52,7 @@ export default class ArrowResultHandler implements IResultsProvider<ArrowBatch>
     let totalRowCount = 0;
     rowSet?.arrowBatches?.forEach(({ batch, rowCount }) => {
       if (batch) {
-        batches.push(this.isLZ4Compressed ? LZ4.decode(batch) : batch);
+        batches.push(this.isLZ4Compressed ? LZ4!.decode(batch) : batch);
         totalRowCount += rowCount.toNumber(true);
       }
     });

diff --git a/lib/result/CloudFetchResultHandler.ts b/lib/result/CloudFetchResultHandler.ts
@@ -1,9 +1,10 @@
-import LZ4 from 'lz4';
 import fetch, { RequestInfo, RequestInit, Request } from 'node-fetch';
 import { TGetResultSetMetadataResp, TRowSet, TSparkArrowResultLink } from '../../thrift/TCLIService_types';
+import HiveDriverError from '../errors/HiveDriverError';
 import IClientContext from '../contracts/IClientContext';
 import IResultsProvider, { ResultsProviderFetchNextOptions } from './IResultsProvider';
 import { ArrowBatch } from './utils';
+import { LZ4 } from '../utils';
 
 export default class CloudFetchResultHandler implements IResultsProvider<ArrowBatch> {
   protected readonly context: IClientContext;
@@ -24,6 +25,10 @@ export default class CloudFetchResultHandler implements IResultsProvider<ArrowBa
     this.context = context;
     this.source = source;
     this.isLZ4Compressed = lz4Compressed ?? false;
+
+    if (this.isLZ4Compressed && !LZ4) {
+      throw new HiveDriverError('Cannot handle LZ4 compressed result: module `lz4` not installed');
+    }
   }
 
   public async hasMore() {
@@ -58,7 +63,7 @@ export default class CloudFetchResultHandler implements IResultsProvider<ArrowBa
     }
 
     if (this.isLZ4Compressed) {
-      batch.batches = batch.batches.map((buffer) => LZ4.decode(buffer));
+      batch.batches = batch.batches.map((buffer) => LZ4!.decode(buffer));
     }
     return batch;
   }

diff --git a/lib/utils/index.ts b/lib/utils/index.ts
@@ -1,5 +1,6 @@
 import definedOrError from './definedOrError';
 import buildUserAgentString from './buildUserAgentString';
 import formatProgress, { ProgressUpdateTransformer } from './formatProgress';
+import LZ4 from './lz4';
 
-export { definedOrError, buildUserAgentString, formatProgress, ProgressUpdateTransformer };
+export { definedOrError, buildUserAgentString, formatProgress, ProgressUpdateTransformer, LZ4 };
diff --git a/lib/utils/lz4.ts b/lib/utils/lz4.ts
@@ -0,0 +1,16 @@
+import type LZ4Namespace from 'lz4';
+
+type LZ4Module = typeof LZ4Namespace;
+
+function tryLoadLZ4Module(): LZ4Module | undefined {
+  try {
+    return require('lz4'); // eslint-disable-line global-require
+  } catch (err) {
+    const isModuleNotFoundError = err instanceof Error && 'code' in err && err.code === 'MODULE_NOT_FOUND';
+    if (!isModuleNotFoundError) {
+      throw err;
+    }
+  }
+}
+
+export default tryLoadLZ4Module();
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -74,7 +74,6 @@
   "dependencies": {
     "apache-arrow": "^13.0.0",
     "commander": "^9.3.0",
-    "lz4": "^0.6.5",
     "node-fetch": "^2.6.12",
     "node-int64": "^0.4.0",
     "open": "^8.4.2",
@@ -83,5 +82,8 @@
     "thrift": "^0.16.0",
     "uuid": "^9.0.0",
     "winston": "^3.8.2"
+  },
+  "optionalDependencies": {
+    "lz4": "^0.6.5"
   }
 }