fix(schema-compiler): Use join tree for pre-agg matching

packages/cubejs-schema-compiler/src/adapter/BaseDimension.ts

@@ -142,6 +142,11 @@ export class BaseDimension {
     if (this.expression) {
       return `expr:${this.expressionName}`;
     }
-    return this.query.cubeEvaluator.pathFromArray(this.path() as string[]);
+    const path = this.path();
+    if (path === null) {
+      // Sanity check, this should not actually happen because we checked this.expression earlier
+      throw new Error('Unexpected null path');
+    }
+    return this.query.cubeEvaluator.pathFromArray(path);
   }
 }

packages/cubejs-schema-compiler/src/adapter/BaseFilter.ts

@@ -1,6 +1,6 @@
 import inlection from 'inflection';
 import moment from 'moment-timezone';
-import { contains, join, map } from 'ramda';
+import { includes, join, map } from 'ramda';
 import { FROM_PARTITION_RANGE, TO_PARTITION_RANGE } from '@cubejs-backend/shared';
 
 import { BaseDimension } from './BaseDimension';
@@ -134,7 +134,7 @@ export class BaseFilter extends BaseDimension {
   }
 
   public isDateOperator(): boolean {
-    return contains(this.camelizeOperator, DATE_OPERATORS);
+    return includes(this.camelizeOperator, DATE_OPERATORS);
   }
 
   public valuesArray() {

packages/cubejs-schema-compiler/src/adapter/BaseMeasure.ts

@@ -320,17 +320,22 @@ export class BaseMeasure {
     return this.measureDefinition().sql;
   }
 
-  public path() {
+  public path(): Array<string> | null {
     if (this.expression) {
       return null;
     }
     return this.query.cubeEvaluator.parsePath('measures', this.measure);
   }
 
-  public expressionPath() {
+  public expressionPath(): string {
     if (this.expression) {
-      return `expr:${this.expression.expressionName}`;
+      return `expr:${this.expressionName}`;
     }
-    return this.query.cubeEvaluator.pathFromArray(this.path() as string[]);
+    const path = this.path();
+    if (path === null) {
+      // Sanity check, this should not actually happen because we checked this.expression earlier
+      throw new Error('Unexpected null path');
+    }
+    return this.query.cubeEvaluator.pathFromArray(path);
   }
 }

packages/cubejs-schema-compiler/src/adapter/BaseQuery.js

@@ -327,6 +327,9 @@ export class BaseQuery {
     }).filter(R.identity).map(this.newTimeDimension.bind(this));
     this.allFilters = this.timeDimensions.concat(this.segments).concat(this.filters);
     /**
+     * For now this might come only from SQL API, it might be some queries that uses measures and filters to
+     * get the dimensions that are then used as join conditions to get the final results.
+     * As consequence - if there are such sub query joins - pre-aggregations can't be used.
      * @type {Array<{sql: string, on: {expression: Function}, joinType: 'LEFT' | 'INNER', alias: string}>}
      */
     this.customSubQueryJoins = this.options.subqueryJoins ?? [];
@@ -360,6 +363,17 @@ export class BaseQuery {
     try {
       // TODO allJoinHints should contain join hints form pre-agg
       this.join = this.joinGraph.buildJoin(this.allJoinHints);
+      /**
+       * @type {Record<string, string[]>}
+       */
+      const queryJoinGraph = {};
+      for (const { originalFrom, originalTo } of (this.join?.joins || [])) {
+        if (!queryJoinGraph[originalFrom]) {
+          queryJoinGraph[originalFrom] = [];
+        }
+        queryJoinGraph[originalFrom].push(originalTo);
+      }
+      this.joinGraphPaths = queryJoinGraph || {};
     } catch (e) {
       if (this.useNativeSqlPlanner) {
         // Tesseract doesn't require join to be prebuilt and there's a case where single join can't be built for multi-fact query
@@ -415,7 +429,7 @@ export class BaseQuery {
 
   /**
    *
-   * @returns {Array<Array<string>>}
+   * @returns {Array<string | Array<string>>}
    */
   get allJoinHints() {
     if (!this.collectedJoinHints) {
@@ -703,7 +717,9 @@ export class BaseQuery {
     if (this.from) {
       return this.simpleQuery();
     }
-    if (!this.options.preAggregationQuery) {
+    const hasMemberExpressions = this.allMembersConcat(false).some(m => m.isMemberExpression);
+
+    if (!this.options.preAggregationQuery && !this.customSubQueryJoins.length && !hasMemberExpressions) {
       preAggForQuery =
         this.preAggregations.findPreAggregationForQuery();
       if (this.options.disableExternalPreAggregations && preAggForQuery?.preAggregation.external) {
@@ -715,8 +731,6 @@ export class BaseQuery {
         multipliedMeasures,
         regularMeasures,
         cumulativeMeasures,
-        withQueries,
-        multiStageMembers,
       } = this.fullKeyQueryAggregateMeasures();
 
       if (cumulativeMeasures.length === 0) {
@@ -780,7 +794,7 @@ export class BaseQuery {
   externalPreAggregationQuery() {
     if (!this.options.preAggregationQuery && !this.options.disableExternalPreAggregations && this.externalQueryClass) {
       const preAggregationForQuery = this.preAggregations.findPreAggregationForQuery();
-      if (preAggregationForQuery && preAggregationForQuery.preAggregation.external) {
+      if (preAggregationForQuery?.preAggregation.external) {
         return true;
       }
       const preAggregationsDescription = this.preAggregations.preAggregationsDescription();
@@ -2133,6 +2147,12 @@ export class BaseQuery {
     ));
   }
 
+  /**
+   *
+   * @param {string} cube
+   * @param {boolean} [isLeftJoinCondition]
+   * @returns {[string, string, string?]}
+   */
   rewriteInlineCubeSql(cube, isLeftJoinCondition) {
     const sql = this.cubeSql(cube);
     const cubeAlias = this.cubeAlias(cube);
@@ -2240,6 +2260,11 @@ export class BaseQuery {
     return this.filtersWithoutSubQueriesValue;
   }
 
+  /**
+   *
+   * @param {string} dimension
+   * @returns {{ prefix: string, subQuery: this, cubeName: string }}
+   */
   subQueryDescription(dimension) {
     const symbol = this.cubeEvaluator.dimensionByPath(dimension);
     const [cubeName, name] = this.cubeEvaluator.parsePath('dimensions', dimension);
@@ -2284,6 +2309,12 @@ export class BaseQuery {
     return { prefix, subQuery, cubeName };
   }
 
+  /**
+   *
+   * @param {string} cubeName
+   * @param {string} name
+   * @returns {string}
+   */
   subQueryName(cubeName, name) {
     return `${cubeName}_${name}_subquery`;
   }
@@ -2644,6 +2675,11 @@ export class BaseQuery {
     );
   }
 
+  /**
+   *
+   * @param {() => void} fn
+   * @returns {Array<string>}
+   */
   collectSubQueryDimensionsFor(fn) {
     const context = { subQueryDimensions: [] };
     this.evaluateSymbolSqlWithContext(
@@ -3206,6 +3242,11 @@ export class BaseQuery {
     return strings.join(' || ');
   }
 
+  /**
+   *
+   * @param {string} cubeName
+   * @returns {Array<string>}
+   */
   primaryKeyNames(cubeName) {
     const primaryKeys = this.cubeEvaluator.primaryKeys[cubeName];
     if (!primaryKeys || !primaryKeys.length) {
@@ -3712,8 +3753,8 @@ export class BaseQuery {
 
   /**
    *
-   * @param options
-   * @returns {BaseQuery}
+   * @param {unknown} options
+   * @returns {this}
    */
   newSubQuery(options) {
     const QueryClass = this.constructor;
@@ -4859,7 +4900,10 @@ export class BaseQuery {
    */
   backAliasMembers(members) {
     const query = this;
-    return Object.fromEntries(members.flatMap(
+
+    const buildJoinPath = this.buildJoinPathFn();
+
+    const aliases = Object.fromEntries(members.flatMap(
       member => {
         const collectedMembers = query.evaluateSymbolSqlWithContext(
           () => query.collectFrom([member], query.collectMemberNamesFor.bind(query), 'collectMemberNamesFor'),
@@ -4877,5 +4921,88 @@ export class BaseQuery {
           .map(d => [query.cubeEvaluator.byPathAnyType(d).aliasMember, memberPath]);
       }
     ));
+
+    /**
+     * @type {Record<string, string>}
+     */
+    const res = {};
+    for (const [original, alias] of Object.entries(aliases)) {
+      const [cube, field] = original.split('.');
+      const path = buildJoinPath(cube);
+
+      const [aliasCube, aliasField] = alias.split('.');
+      const aliasPath = aliasCube !== cube ? buildJoinPath(aliasCube) : path;
+
+      if (path) {
+        res[`${path}.${field}`] = aliasPath ? `${aliasPath}.${aliasField}` : alias;
+      }
+
+      // Aliases might come from proxied members, in such cases
+      // we need to map them to originals too
+      if (aliasPath) {
+        res[original] = `${aliasPath}.${aliasField}`;
+      }
+    }
+
+    return res;
+  }
+
+  buildJoinPathFn() {
+    const query = this;
+    const { root } = this.join || {};
+
+    return (target) => {
+      const visited = new Set();
+      const path = [];
+
+      /**
+       * @param {string} node
+       * @returns {boolean}
+       */
+      function dfs(node) {
+        if (node === target) {
+          path.push(node);
+          return true;
+        }
+
+        if (visited.has(node)) return false;
+        visited.add(node);
+
+        const neighbors = query.joinGraphPaths[node] || [];
+        for (const neighbor of neighbors) {
+          if (dfs(neighbor)) {
+            path.unshift(node);
+            return true;
+          }
+        }
+
+        return false;
+      }
+
+      return dfs(root) ? path.join('.') : null;
+    };
+  }
+
+  /**
+   * Returns a function that constructs the full member path
+   * based on the query's join structure.
+   * @returns {(function(member: string): (string))}
+   */
+  resolveFullMemberPathFn() {
+    const { root: queryJoinRoot } = this.join || {};
+
+    const buildJoinPath = this.buildJoinPathFn();
+
+    return (member) => {
+      const [cube, field] = member.split('.');
+      if (!cube || !field) return member;
+
+      if (cube === queryJoinRoot.root) {
+        return member;
+      }
+
+      const path = buildJoinPath(cube);
+      return path ? `${path}.${field}` : member;
+    };
   }
 }

packages/cubejs-schema-compiler/src/adapter/BaseSegment.ts

@@ -81,17 +81,22 @@ export class BaseSegment {
     return this.segmentDefinition().sql;
   }
 
-  public path() {
+  public path(): Array<string> | null {
     if (this.expression) {
       return null;
     }
     return this.query.cubeEvaluator.parsePath('segments', this.segment);
   }
 
-  public expressionPath() {
+  public expressionPath(): string {
     if (this.expression) {
-      return `expr:${this.expression.expressionName}`;
+      return `expr:${this.expressionName}`;
     }
-    return this.query.cubeEvaluator.pathFromArray(this.path() as string[]);
+    const path = this.path();
+    if (path === null) {
+      // Sanity check, this should not actually happen because we checked this.expression earlier
+      throw new Error('Unexpected null path');
+    }
+    return this.query.cubeEvaluator.pathFromArray(path);
   }
 }