Fix read model queries when selecting deeply-nested arrays (#1552)

* Fix handling of deeply-nested arrays of objects in read model queries

* Handle case of top-level array property followed by sub array prop.

* Add rush change file

* Update docs

* Refactor preprocessProjections

* Update @cdktf/provider-azurerm version

* Refactor some code due to changes in azurerm

---------

Co-authored-by: Castro, Mario <mariocs@optum.com>

common/changes/@boostercloud/framework-core/fix_nested_arrays_2024-09-24-21-22.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@boostercloud/framework-core",
+      "comment": "Correct handling of deeply-nested arrays and sub-array properties in Read Model queries",
+      "type": "patch"
+    }
+  ],
+  "packageName": "@boostercloud/framework-core"
+}

packages/framework-provider-azure-infrastructure/package.json

@@ -29,7 +29,7 @@
     "@boostercloud/framework-core": "workspace:^2.18.1",
     "@boostercloud/framework-provider-azure": "workspace:^2.18.1",
     "@boostercloud/framework-types": "workspace:^2.18.1",
-    "@cdktf/provider-azurerm": "11.2.0",
+    "@cdktf/provider-azurerm": "13.3.0",
     "@cdktf/provider-time": "9.0.2",
     "@types/archiver": "5.1.0",
     "@types/needle": "^2.0.4",

packages/framework-provider-azure-infrastructure/src/infrastructure/synth/application-synth.ts

@@ -52,7 +52,7 @@ export class ApplicationSynth {
   public constructor(terraformStack: TerraformStack) {
     this.config = readProjectConfig(process.cwd())
     const azurermProvider = new AzurermProvider(terraformStack, 'azureFeature', {
-      features: {},
+      features: [{}],
     })
     const appPrefix = buildAppPrefix(this.config)
     const resourceGroupName = createResourceGroupName(this.config.appName, this.config.environmentName)

packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-containers.ts

@@ -85,7 +85,7 @@ export class TerraformContainers {
       resourceGroupName: cosmosdbDatabaseResource.resourceGroupName,
       accountName: cosmosdbDatabaseResource.name,
       databaseName: cosmosdbSqlDatabaseResource.name,
-      partitionKeyPath: `/${eventsStoreAttributes.partitionKey}`,
+      partitionKeyPaths: [`/${eventsStoreAttributes.partitionKey}`],
       partitionKeyVersion: 2,
       autoscaleSettings: {
         maxThroughput: MAX_CONTAINER_THROUGHPUT,
@@ -108,7 +108,7 @@ export class TerraformContainers {
       resourceGroupName: cosmosdbDatabaseResource.resourceGroupName,
       accountName: cosmosdbDatabaseResource.name,
       databaseName: cosmosdbSqlDatabaseResource.name,
-      partitionKeyPath: '/eventId',
+      partitionKeyPaths: ['/eventId'],
       partitionKeyVersion: 2,
       uniqueKey: [{ paths: ['/eventId'] }],
       autoscaleSettings: {
@@ -134,7 +134,7 @@ export class TerraformContainers {
       resourceGroupName: cosmosdbDatabaseResource.resourceGroupName,
       accountName: cosmosdbDatabaseResource.name,
       databaseName: cosmosdbSqlDatabaseResource.name,
-      partitionKeyPath: '/id',
+      partitionKeyPaths: ['/id'],
       partitionKeyVersion: 2,
       autoscaleSettings: {
         maxThroughput: MAX_CONTAINER_THROUGHPUT,
@@ -157,7 +157,7 @@ export class TerraformContainers {
       resourceGroupName: cosmosdbDatabaseResource.resourceGroupName,
       accountName: cosmosdbDatabaseResource.name,
       databaseName: cosmosdbSqlDatabaseResource.name,
-      partitionKeyPath: `/${subscriptionsStoreAttributes.partitionKey}`,
+      partitionKeyPaths: [`/${subscriptionsStoreAttributes.partitionKey}`],
       partitionKeyVersion: 2,
       defaultTtl: -1,
       autoscaleSettings: {
@@ -181,7 +181,7 @@ export class TerraformContainers {
       resourceGroupName: cosmosdbDatabaseResource.resourceGroupName,
       accountName: cosmosdbDatabaseResource.name,
       databaseName: cosmosdbSqlDatabaseResource.name,
-      partitionKeyPath: `/${connectionsStoreAttributes.partitionKey}`,
+      partitionKeyPaths: [`/${connectionsStoreAttributes.partitionKey}`],
       partitionKeyVersion: 2,
       defaultTtl: -1,
       autoscaleSettings: {
@@ -207,7 +207,7 @@ export class TerraformContainers {
       resourceGroupName: cosmosdbDatabase.resourceGroupName,
       accountName: cosmosdbDatabase.name,
       databaseName: cosmosdbSqlDatabase.name,
-      partitionKeyPath: `/${dedupAttributes.partitionKey}`,
+      partitionKeyPaths: [`/${dedupAttributes.partitionKey}`],
       uniqueKey: [{ paths: [`/${dedupAttributes.partitionKey}`] }],
       partitionKeyVersion: 2,
       defaultTtl: -1,

packages/framework-provider-azure-infrastructure/src/infrastructure/synth/terraform-cosmosdb-database.ts

@@ -17,9 +17,9 @@ export class TerraformCosmosdbDatabase {
       resourceGroupName: resourceGroupName,
       offerType: 'Standard',
       kind: 'GlobalDocumentDB',
-      enableMultipleWriteLocations: false,
+      multipleWriteLocationsEnabled: false,
       isVirtualNetworkFilterEnabled: false,
-      enableAutomaticFailover: true,
+      automaticFailoverEnabled: true,
       geoLocation: [
         {
           location: resourceGroup.location,

packages/framework-provider-azure/package.json

@@ -31,7 +31,8 @@
     "@boostercloud/framework-types": "workspace:^2.18.1",
     "tslib": "^2.4.0",
     "@effect-ts/core": "^0.60.4",
-    "@azure/web-pubsub": "~1.1.0"
+    "@azure/web-pubsub": "~1.1.0",
+    "@cdktf/provider-azurerm": "13.3.0"
   },
   "devDependencies": {
     "@boostercloud/eslint-config": "workspace:^2.18.1",

packages/framework-provider-azure/src/helpers/query-helper.ts

@@ -243,6 +243,9 @@ function buildProjections(projections: ProjectionFor<unknown> | string = '*'): s
     return projections
   }
 
+  // Preprocess the projections
+  const preprocessedProjections = preprocessProjections(projections)
+
   // Helper function to convert dot notation to square-bracket notation
   const toSquareBracketsNotation = (path: string): string => {
     return path
@@ -253,7 +256,7 @@ function buildProjections(projections: ProjectionFor<unknown> | string = '*'): s
 
   // Group fields by the root property
   const groupedFields: { [key: string]: string[] } = {}
-  Object.values(projections).forEach((field: string) => {
+  Object.values(preprocessedProjections).forEach((field: string) => {
     const root: string = field.split('.')[0]
     if (!groupedFields[root]) {
       groupedFields[root] = []
@@ -314,6 +317,51 @@ function buildProjections(projections: ProjectionFor<unknown> | string = '*'): s
     .join(', ')
 }
 
+/**
+ * Preprocesses the projections to handle nested arrays and objects.
+ *
+ * @param {ProjectionFor<unknown>} projections - The projections to preprocess.
+ * @returns {ProjectionFor<unknown>} - The preprocessed projections.
+ */
+function preprocessProjections(projections: ProjectionFor<unknown>): ProjectionFor<unknown> {
+  const processed = new Set<string>()
+
+  Object.values(projections).forEach((field: string) => {
+    const parts = field.split('.')
+    const arrayIndices = parts.reduce((acc, part, index) => {
+      if (part.endsWith('[]')) acc.push(index)
+      return acc
+    }, [] as number[])
+
+    if (
+      arrayIndices.length === 0 ||
+      (arrayIndices[0] === 0 && arrayIndices.length === 1) ||
+      (arrayIndices[0] === 1 && arrayIndices.length === 1)
+    ) {
+      // This block is accessed when one of the following occurs:
+      // - No arrays in the projection
+      // - Top-level array not followed by another array
+      // - Array nested within a top-level property, no arrays follow
+      processed.add(field)
+    } else {
+      // Cases with nested arrays or arrays deeper in the structure
+      const processToIndex = arrayIndices[0] === 0 || arrayIndices[0] === 1 ? arrayIndices[1] : arrayIndices[0]
+      const processedField = parts.slice(0, processToIndex + 1).join('.')
+      processed.add(processedField.slice(0, -2)) // Remove the '[]' from the last part
+    }
+  })
+
+  // Convert the Set back to the original type of projections
+  if (Array.isArray(projections)) {
+    return Array.from(processed) as ProjectionFor<unknown>
+  } else {
+    return Array.from(processed).reduce((acc, field) => {
+      ;(acc as any)[field] = field
+      return acc
+    }, {} as ProjectionFor<unknown>)
+  }
+}
+
 /**
  * Transforms the flat properties returned by Cosmos DB into a nested structure. For example, the following object:
  *

packages/framework-provider-azure/test/helpers/query-helper.test.ts

@@ -169,6 +169,46 @@ describe('Query helper', () => {
       )
     })
 
+    it('Executes a SQL query with a projectionFor projection that has a deeply nested array of objects', async () => {
+      await search(
+        mockCosmosDbClient as any,
+        mockConfig,
+        mockReadModelName,
+        {},
+        undefined,
+        undefined,
+        false,
+        undefined,
+        [
+          'id',
+          'x.arr[].z',
+          'foo.bar.items[].id',
+          'foo.bar.baz.items[].id',
+          'arr[].subArr[].id',
+          'arr[].id',
+        ] as ProjectionFor<unknown>
+      )
+
+      expect(mockCosmosDbClient.database).to.have.been.calledWithExactly(mockConfig.resourceNames.applicationStack)
+      expect(
+        mockCosmosDbClient.database(mockConfig.resourceNames.applicationStack).container
+      ).to.have.been.calledWithExactly(`${mockReadModelName}`)
+      expect(
+        mockCosmosDbClient.database(mockConfig.resourceNames.applicationStack).container(`${mockReadModelName}`).items
+          .query
+      ).to.have.been.calledWith(
+        match({
+          query:
+            'SELECT c["id"], ' +
+            'ARRAY(SELECT item["z"] FROM item IN c["x"]["arr"]) AS "x.arr", ' +
+            'c["foo"]["bar"]["items"] AS "foo.bar.items", c["foo"]["bar"]["baz"]["items"] AS "foo.bar.baz.items", ' +
+            'ARRAY(SELECT item["subArr"], item["id"] FROM item IN c["arr"]) AS arr ' +
+            'FROM c ',
+          parameters: [],
+        })
+      )
+    })
+
     it('Executes a SQL query with a star projection in the read model table', async () => {
       await search(
         mockCosmosDbClient as any,

website/docs/06_graphql.md

@@ -891,7 +891,7 @@ Using `select` will skip any Read Models migrations that need to be applied to t
 :::
 
 :::warning
-Support for selecting fields from objects inside arrays is limited to arrays that are at most nested inside another property, e.g., `['category.relatedCategories[].name']`. Selecting fields from arrays that are nested deeper than that (e.g., `['foo.bar.items[].id']`) is not expected to yield the expected results.
+Support for selecting fields from objects inside arrays is limited to arrays that are at most nested inside another property, e.g., `['category.relatedCategories[].name']`. Selecting fields from arrays that are nested deeper than that (e.g., `['foo.bar.items[].id']`) will return the entire object.
 :::
 
 :::warning