feat(schema-compiler,api-gateway): Nested folders support

packages/cubejs-api-gateway/openspec.yml

@@ -173,7 +173,9 @@ components:
         members:
           type: array
           items:
-            type: "string"
+            oneOf:
+              - type: string
+              - $ref: "#/components/schemas/V1CubeMetaFolder"
     V1CubeMetaHierarchy:
       type: "object"
       required:

packages/cubejs-client-core/src/types.ts

@@ -414,7 +414,7 @@ export type CubeMember = TCubeMeasure | TCubeDimension | TCubeSegment;
 
 export type TCubeFolder = {
   name: string;
-  members: string[];
+  members: (string | TCubeFolder)[];
 };
 
 export type TCubeHierarchy = {

packages/cubejs-playground/src/QueryBuilderV2/components/SidePanelCubeItem.tsx

@@ -147,7 +147,9 @@ export function SidePanelCubeItem(props: CubeListItemProps) {
   const [openHierarchies, setOpenHierarchies] = useState<string[]>([]);
 
   const folderMembers = folders.reduce((acc, folder) => {
-    return acc.concat(folder.members);
+    // FIXME: Temporary marked folder members as strings
+    // It should be aware of recursive folders structure
+    return acc.concat(folder.members as string[]);
   }, [] as string[]);
   const hierarchyMembers = hierarchies.reduce((acc, hierarchy) => {
     return acc.concat(hierarchy.levels);

packages/cubejs-playground/src/QueryBuilderV2/hooks/query-builder.ts

@@ -1286,30 +1286,32 @@ export function useQueryBuilder(props: UseQueryBuilderProps) {
 
           const folderStats = stats.folders[folderName];
 
+          // FIXME: Temporary marked folder members as strings
+          // It should be aware of recursive folders structure
           folder.members.forEach((memberName) => {
-            if (stats.dimensions.includes(memberName)) {
-              if (!folderStats.dimensions.includes(memberName)) {
-                folderStats.dimensions.push(memberName);
+            if (stats.dimensions.includes(memberName as string)) {
+              if (!folderStats.dimensions.includes(memberName as string)) {
+                folderStats.dimensions.push(memberName as string);
               }
-            } else if (stats.measures.includes(memberName)) {
-              if (!folderStats.measures.includes(memberName)) {
-                folderStats.measures.push(memberName);
+            } else if (stats.measures.includes(memberName as string)) {
+              if (!folderStats.measures.includes(memberName as string)) {
+                folderStats.measures.push(memberName as string);
               }
-            } else if (stats.segments.includes(memberName)) {
-              if (!folderStats.segments.includes(memberName)) {
-                folderStats.segments.push(memberName);
+            } else if (stats.segments.includes(memberName as string)) {
+              if (!folderStats.segments.includes(memberName as string)) {
+                folderStats.segments.push(memberName as string);
               }
-            } else if (stats.hierarchies[memberName]) {
+            } else if (stats.hierarchies[memberName as string]) {
               // add all selected dimensions from the hierarchy
-              stats.hierarchies[memberName].forEach((levelMemberName) => {
+              stats.hierarchies[memberName as string].forEach((levelMemberName) => {
                 if (!folderStats.dimensions.includes(levelMemberName)) {
                   folderStats.dimensions.push(levelMemberName);
                 }
               });
             }
 
-            if (grouping.includes(memberName)) {
-              folderStats.grouping.push(memberName);
+            if (grouping.includes(memberName as string)) {
+              folderStats.grouping.push(memberName as string);
             }
           });
         });

packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts

@@ -259,34 +259,51 @@ export class CubeEvaluator extends CubeSymbols {
 
   private prepareFolders(cube: any, errorReporter: ErrorReporter) {
     const folders = cube.rawFolders();
-    if (folders.length) {
-      cube.folders = folders.map(it => {
-        const includedMembers = this.allMembersOrList(cube, it.includes);
-        const includes = includedMembers.map(memberName => {
-          if (memberName.includes('.')) {
-            errorReporter.error(
-              `Paths aren't allowed in the 'folders' but '${memberName}' has been provided for ${cube.name}`
-            );
-          }
+    if (!folders.length) return;
 
-          const member = cube.includedMembers.find(m => m.name === memberName);
-          if (!member) {
-            errorReporter.error(
-              `Member '${memberName}' included in folder '${it.name}' not found`
-            );
-            return null;
-          }
+    const checkMember = (memberName: string, folderName: string) => {
+      if (memberName.includes('.')) {
+        errorReporter.error(
+          `Paths aren't allowed in the 'folders' but '${memberName}' has been provided for ${cube.name}`
+        );
+      }
+
+      const member = cube.includedMembers.find(m => m.name === memberName);
+      if (!member) {
+        errorReporter.error(
+          `Member '${memberName}' included in folder '${folderName}' not found`
+        );
+        return null;
+      }
 
-          return member;
-        })
-          .filter(Boolean);
+      return member;
+    };
+
+    const processFolder = (folder: any): any => {
+      let includedMembers: string[];
+      let includes: any[] = [];
+
+      if (folder.includes === '*') {
+        includedMembers = this.allMembersOrList(cube, folder.includes);
+        includes = includedMembers.map(m => checkMember(m, folder.name)).filter(Boolean);
+      } else if (Array.isArray(folder.includes)) {
+        includes = folder.includes.map(item => {
+          if (typeof item === 'object' && item !== null) {
+            return processFolder(item);
+          }
 
-        return ({
-          ...it,
-          includes
+          return checkMember(item, folder.name);
         });
-      });
-    }
+      }
+
+      return {
+        ...folder,
+        type: 'folder',
+        includes: includes.filter(Boolean)
+      };
+    };
+
+    cube.folders = folders.map(processFolder);
   }
 
   private prepareHierarchies(cube: any, errorReporter: ErrorReporter): void {

packages/cubejs-schema-compiler/src/compiler/CubeToMetaTransformer.js

@@ -43,6 +43,17 @@ export class CubeToMetaTransformer {
 
     const isCubeVisible = this.isVisible(cube, true);
 
+    const processFolderMember = (member) => {
+      if (member.type === 'folder') {
+        return {
+          name: member.name,
+          members: member.includes.map(processFolderMember),
+        };
+      }
+
+      return `${cube.name}.${member.name}`;
+    };
+
     return {
       config: {
         name: cube.name,
@@ -115,7 +126,7 @@ export class CubeToMetaTransformer {
         })),
         folders: (cube.folders || []).map((it) => ({
           name: it.name,
-          members: it.includes.map(member => `${cube.name}.${member.name}`),
+          members: it.includes.map(processFolderMember),
         })),
       },
     };

packages/cubejs-schema-compiler/src/compiler/CubeValidator.ts

@@ -795,6 +795,19 @@ const cubeSchema = inherit(baseSchema, {
   'object.xor': 'You must use either sql or sqlTable within a model, but not both'
 });
 
+const folderSchema = Joi.object().keys({
+  name: Joi.string().required(),
+  includes: Joi.alternatives([
+    Joi.string().valid('*'),
+    Joi.array().items(
+      Joi.alternatives([
+        Joi.string().required(),
+        Joi.link('#folderSchema'), // Can contain nested folders
+      ]),
+    ),
+  ]).required(),
+}).id('folderSchema');
+
 const viewSchema = inherit(baseSchema, {
   isView: Joi.boolean().strict(),
   cubes: Joi.array().items(
@@ -822,13 +835,7 @@ const viewSchema = inherit(baseSchema, {
       'object.oxor': 'Using split together with prefix is not supported'
     })
   ),
-  folders: Joi.array().items(Joi.object().keys({
-    name: Joi.string().required(),
-    includes: Joi.alternatives([
-      Joi.string().valid('*'),
-      Joi.array().items(Joi.string().required())
-    ]).required(),
-  })),
+  folders: Joi.array().items(folderSchema),
 });
 
 function formatErrorMessageFromDetails(explain, d) {

packages/cubejs-schema-compiler/test/unit/__snapshots__/schema.test.ts.snap

@@ -1990,6 +1990,7 @@ Array [
       },
     ],
     "name": "folder1",
+    "type": "folder",
   },
   Object {
     "includes": Array [
@@ -2005,6 +2006,7 @@ Array [
       },
     ],
     "name": "folder2",
+    "type": "folder",
   },
 ]
 `;

packages/cubejs-schema-compiler/test/unit/fixtures/folders.yml

@@ -105,6 +105,17 @@ views:
         includes:
           - users_city
           - users_renamed_in_view3_gender
+  - name: test_view4
+    extends: test_view3
+    folders:
+      - name: folder3
+        includes:
+          - users_city
+          - name: inner folder 4
+            includes:
+              - renamed_orders_status
+          - name: inner folder 5
+            includes: "*"
 
   # - name: empty_view
   #   cubes:

packages/cubejs-schema-compiler/test/unit/fixtures/folders_invalid_path.yml

@@ -72,6 +72,9 @@ views:
           - age
           - renamed_gender
           - users.age
+          - name: inner folder
+            includes:
+              - users.renamed_gender
       - name: folder2
         includes: '*'
   - name: test_view2

packages/cubejs-schema-compiler/test/unit/folders.test.ts

@@ -43,6 +43,50 @@ describe('Cube Folders', () => {
     );
   });
 
+  it('a nested folders with some * and named members', async () => {
+    const testView = metaTransformer.cubes.find(
+      (it) => it.config.name === 'test_view4'
+    );
+
+    expect(testView.config.folders.length).toBe(3);
+
+    const folder1 = testView.config.folders.find(
+      (it) => it.name === 'folder1'
+    );
+    expect(folder1.members).toEqual([
+      'test_view4.users_age',
+      'test_view4.users_state',
+      'test_view4.renamed_orders_status',
+    ]);
+
+    const folder2 = testView.config.folders.find(
+      (it) => it.name === 'folder2'
+    );
+    expect(folder2.members).toEqual(
+      expect.arrayContaining(['test_view4.users_city', 'test_view4.users_renamed_in_view3_gender'])
+    );
+
+    const folder3 = testView.config.folders.find(
+      (it) => it.name === 'folder3'
+    );
+    expect(folder3.members.length).toBe(3);
+    expect(folder3.members[1]).toEqual(
+      { name: 'inner folder 4', members: ['test_view4.renamed_orders_status'] }
+    );
+    expect(folder3.members[2].name).toEqual('inner folder 5');
+    expect(folder3.members[2].members).toEqual([
+      'test_view4.renamed_orders_count',
+      'test_view4.renamed_orders_id',
+      'test_view4.renamed_orders_number',
+      'test_view4.renamed_orders_status',
+      'test_view4.users_age',
+      'test_view4.users_state',
+      'test_view4.users_gender',
+      'test_view4.users_city',
+      'test_view4.users_renamed_in_view3_gender',
+    ]);
+  });
+
   it('folders from view extending other view', async () => {
     const view2 = metaTransformer.cubes.find(
       (it) => it.config.name === 'test_view2'
@@ -93,6 +137,7 @@ describe('Cube Folders', () => {
       throw new Error('should throw earlier');
     } catch (e: any) {
       expect(e.toString()).toMatch(/Paths aren't allowed in the 'folders' but 'users.age' has been provided for test_view/);
+      expect(e.toString()).toMatch(/Paths aren't allowed in the 'folders' but 'users.renamed_gender' has been provided for test_view/);
       expect(e.toString()).toMatch(/Member 'users.age' included in folder 'folder1' not found/);
     }
   });

rust/cubesql/cubeclient/.openapi-generator/FILES

@@ -5,6 +5,7 @@ src/models/v1_cube_meta.rs
 src/models/v1_cube_meta_dimension.rs
 src/models/v1_cube_meta_dimension_granularity.rs
 src/models/v1_cube_meta_folder.rs
+src/models/v1_cube_meta_folder_members_inner.rs
 src/models/v1_cube_meta_hierarchy.rs
 src/models/v1_cube_meta_join.rs
 src/models/v1_cube_meta_measure.rs

rust/cubesql/cubeclient/.openapi-generator/VERSION

@@ -1 +1 @@
-6.6.0
+7.13.0

rust/cubesql/cubeclient/src/models/mod.rs

@@ -6,6 +6,8 @@ pub mod v1_cube_meta_dimension_granularity;
 pub use self::v1_cube_meta_dimension_granularity::V1CubeMetaDimensionGranularity;
 pub mod v1_cube_meta_folder;
 pub use self::v1_cube_meta_folder::V1CubeMetaFolder;
+pub mod v1_cube_meta_folder_members_inner;
+pub use self::v1_cube_meta_folder_members_inner::V1CubeMetaFolderMembersInner;
 pub mod v1_cube_meta_hierarchy;
 pub use self::v1_cube_meta_hierarchy::V1CubeMetaHierarchy;
 pub mod v1_cube_meta_join;

rust/cubesql/cubeclient/src/models/v1_cube_meta_folder.rs

@@ -8,16 +8,19 @@
  * Generated by: https://openapi-generator.tech
  */
 
-#[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)]
+#[derive(Clone, Default, Debug, PartialEq, Serialize, Deserialize)]
 pub struct V1CubeMetaFolder {
     #[serde(rename = "name")]
     pub name: String,
     #[serde(rename = "members")]
-    pub members: Vec<String>,
+    pub members: Vec<crate::models::V1CubeMetaFolderMembersInner>,
 }
 
 impl V1CubeMetaFolder {
-    pub fn new(name: String, members: Vec<String>) -> V1CubeMetaFolder {
+    pub fn new(
+        name: String,
+        members: Vec<crate::models::V1CubeMetaFolderMembersInner>,
+    ) -> V1CubeMetaFolder {
         V1CubeMetaFolder { name, members }
     }
 }

rust/cubesql/cubeclient/src/models/v1_cube_meta_folder_members_inner.rs

@@ -0,0 +1,22 @@
+/*
+ * Cube.js
+ *
+ * Cube.js Swagger Schema
+ *
+ * The version of the OpenAPI document: 1.0.0
+ *
+ * Generated by: https://openapi-generator.tech
+ */
+
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[serde(untagged)]
+pub enum V1CubeMetaFolderMembersInner {
+    String(String),
+    V1CubeMetaFolder(Box<crate::models::V1CubeMetaFolder>),
+}
+
+impl Default for V1CubeMetaFolderMembersInner {
+    fn default() -> Self {
+        Self::String(Default::default())
+    }
+}