chore(schema-compiler)!: Remove old blocking model compilation and make TRANSPILATION_WORKER_THREADS new default

.github/workflows/push.yml

@@ -63,11 +63,8 @@ jobs:
         node-version: [22.x]
         # Don't forget to update build-native-release
         python-version: [3.11]
-        transpile-worker-threads: [false, true]
       fail-fast: false
 
-    env:
-      CUBEJS_TRANSPILATION_WORKER_THREADS: ${{ matrix.transpile-worker-threads }}
     steps:
       - id: get-tag-out
         run: echo "$OUT"
@@ -121,16 +118,16 @@ jobs:
       - name: Lerna test
         run: yarn lerna run --concurrency 1 --stream --no-prefix unit
       - name: Fix lcov paths
-        if: (matrix.node-version == '22.x' && matrix.transpile-worker-threads == true)
+        if: (matrix.node-version == '22.x')
         run: |
           ./.github/actions/codecov-fix.sh
       - name: Combine all fixed LCOV files
-        if: (matrix.node-version == '22.x' && matrix.transpile-worker-threads == true)
+        if: (matrix.node-version == '22.x')
         run: |
           echo "" > ./combined-unit.lcov
           find ./packages -type f -name lcov.fixed.info -exec cat {} + >> ./combined-unit.lcov || true
       - name: Upload coverage artifact
-        if: (matrix.node-version == '22.x' && matrix.transpile-worker-threads == true)
+        if: (matrix.node-version == '22.x')
         uses: actions/upload-artifact@v4
         with:
           name: coverage-unit

docs/pages/product/configuration/reference/environment-variables.mdx

@@ -1326,21 +1326,6 @@ learn more.
 | --------------- | ---------------------- | --------------------- |
 | A valid number  | 86400                  | 86400                 |
 
-## `CUBEJS_TRANSPILATION_WORKER_THREADS`
-
-If `true`, optimizes data model compilation by running critical parts of the
-code in worker threads.
-
-| Possible Values | Default in Development | Default in Production |
-| --------------- | ---------------------- | --------------------- |
-| `true`, `false` | `false`                | `false`               |
-
-<ReferenceBox>
-
-See [this issue](https://github.com/cube-js/cube/issues/9285) for details.
-
-</ReferenceBox>
-
 ## `CUBEJS_WEB_SOCKETS`
 
 If `true`, then use WebSocket for data fetching.

packages/cubejs-backend-shared/src/env.ts

@@ -223,16 +223,12 @@ const variables: Record<string, (...args: any) => any> = {
   nativeOrchestrator: () => get('CUBEJS_TESSERACT_ORCHESTRATOR')
     .default('true')
     .asBoolStrict(),
-  transpilationWorkerThreads: () => get('CUBEJS_TRANSPILATION_WORKER_THREADS')
-    .default('false')
-    .asBoolStrict(),
   allowNonStrictDateRangeMatching: () => get('CUBEJS_PRE_AGGREGATIONS_ALLOW_NON_STRICT_DATE_RANGE_MATCH')
     .default('true')
     .asBoolStrict(),
   transpilationWorkerThreadsCount: () => get('CUBEJS_TRANSPILATION_WORKER_THREADS_COUNT')
     .default('0')
     .asInt(),
-  // This one takes precedence over CUBEJS_TRANSPILATION_WORKER_THREADS
   transpilationNative: () => get('CUBEJS_TRANSPILATION_NATIVE')
     .default('false')
     .asBoolStrict(),

packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js

@@ -4,9 +4,6 @@ import fs from 'fs';
 import os from 'os';
 import path from 'path';
 import syntaxCheck from 'syntax-error';
-import { parse } from '@babel/parser';
-import babelGenerator from '@babel/generator';
-import babelTraverse from '@babel/traverse';
 import R from 'ramda';
 import workerpool from 'workerpool';
 
@@ -114,12 +111,11 @@ export class DataSchemaCompiler {
     const errorsReport = new ErrorReporter(null, [], this.errorReport);
     this.errorsReport = errorsReport;
 
-    const transpilationWorkerThreads = getEnv('transpilationWorkerThreads');
     const transpilationNative = getEnv('transpilationNative');
     const transpilationNativeThreadsCount = getThreadsCount();
     const { compilerId } = this;
 
-    if (!transpilationNative && transpilationWorkerThreads) {
+    if (!transpilationNative) {
       const wc = getEnv('transpilationWorkerThreadsCount');
       this.workerPool = workerpool.pool(
         path.join(__dirname, 'transpilers/transpiler_worker'),
@@ -132,32 +128,27 @@ export class DataSchemaCompiler {
      * @returns {Promise<*>}
      */
     const transpile = async (stage) => {
-      let cubeNames;
-      let cubeSymbols;
-      let transpilerNames;
       let results;
 
-      if (transpilationNative || transpilationWorkerThreads) {
-        cubeNames = Object.keys(this.cubeDictionary.byId);
-        // We need only cubes and all its member names for transpiling.
-        // Cubes doesn't change during transpiling, but are changed during compilation phase,
-        // so we can prepare them once for every phase.
-        // Communication between main and worker threads uses
-        // The structured clone algorithm (@see https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Structured_clone_algorithm)
-        // which doesn't allow passing any function objects, so we need to sanitize the symbols.
-        // Communication with native backend also involves deserialization.
-        cubeSymbols = Object.fromEntries(
-          Object.entries(this.cubeSymbols.symbols)
-            .map(
-              ([key, value]) => [key, Object.fromEntries(
-                Object.keys(value).map((k) => [k, true]),
-              )],
-            ),
-        );
+      const cubeNames = Object.keys(this.cubeDictionary.byId);
+      // We need only cubes and all its member names for transpiling.
+      // Cubes doesn't change during transpiling, but are changed during compilation phase,
+      // so we can prepare them once for every phase.
+      // Communication between main and worker threads uses
+      // The structured clone algorithm (@see https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Structured_clone_algorithm)
+      // which doesn't allow passing any function objects, so we need to sanitize the symbols.
+      // Communication with native backend also involves deserialization.
+      const cubeSymbols = Object.fromEntries(
+        Object.entries(this.cubeSymbols.symbols)
+          .map(
+            ([key, value]) => [key, Object.fromEntries(
+              Object.keys(value).map((k) => [k, true]),
+            )],
+          ),
+      );
 
-        // Transpilers are the same for all files within phase.
-        transpilerNames = this.transpilers.map(t => t.constructor.name);
-      }
+      // Transpilers are the same for all files within phase.
+      const transpilerNames = this.transpilers.map(t => t.constructor.name);
 
       if (transpilationNative) {
         // Warming up swc compiler cache
@@ -192,10 +183,8 @@ export class DataSchemaCompiler {
         }
 
         results = (await Promise.all([...nonJsFilesTasks, ...JsFilesTasks])).flat();
-      } else if (transpilationWorkerThreads) {
-        results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames })));
       } else {
-        results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, {})));
+        results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames })));
       }
 
       return results.filter(f => !!f);
@@ -225,7 +214,7 @@ export class DataSchemaCompiler {
             errorsReport,
             { cubeNames: [], cubeSymbols: {}, transpilerNames: [], contextSymbols: {}, compilerId: this.compilerId, stage: 0 }
           );
-        } else if (transpilationWorkerThreads && this.workerPool) {
+        } else if (this.workerPool) {
           this.workerPool.terminate();
         }
 
@@ -334,7 +323,7 @@ export class DataSchemaCompiler {
         errorsReport.exitFile();
 
         return { ...file, content: res[0].code };
-      } else if (getEnv('transpilationWorkerThreads')) {
+      } else {
         const data = {
           fileName: file.fileName,
           content: file.content,
@@ -348,24 +337,6 @@ export class DataSchemaCompiler {
         errorsReport.addWarnings(res.warnings);
 
         return { ...file, content: res.content };
-      } else {
-        const ast = parse(
-          file.content,
-          {
-            sourceFilename: file.fileName,
-            sourceType: 'module',
-            plugins: ['objectRestSpread'],
-          },
-        );
-
-        errorsReport.inFile(file);
-        this.transpilers.forEach((t) => {
-          babelTraverse(ast, t.traverseObject(errorsReport));
-        });
-        errorsReport.exitFile();
-
-        const content = babelGenerator(ast, {}, file.content).code;
-        return { ...file, content };
       }
     } catch (e) {
       if (e.toString().indexOf('SyntaxError') !== -1) {

packages/cubejs-schema-compiler/test/integration/postgres/dataschema-compiler.test.ts

@@ -186,7 +186,7 @@ describe('DataSchemaCompiler', () => {
         })
       `;
 
-      it('Should compile 200 schemas in less than 2500ms * 10', async () => {
+      it('Should compile 200 schemas in less than 10000ms * 10', async () => {
         const repeats = 200;
 
         const compilerWith = prepareJsCompiler(schema, { allowJsDuplicatePropsInSchema: false });
@@ -198,7 +198,7 @@ describe('DataSchemaCompiler', () => {
         const end = new Date().getTime();
         const time = end - start;
 
-        expect(time).toBeLessThan(2500 * 10);
+        expect(time).toBeLessThan(10000 * 10);
       });
     });
   });