Fix SDF API tests and Re-enable queueing the Edda Task for Deployment MVs on Migration

bin/si-sdf-api-test/sdf_api_client.ts

@@ -156,6 +156,21 @@ export const ROUTES = {
       `/v2/workspaces/${vars.workspaceId}/change-sets/${vars.changeSetId}/index/multi_mjolnir`,
     method: "POST",
   },
+  deployment_mjolnir: {
+    path: (vars: ROUTE_VARS) =>
+      `/v2/workspaces/${vars.workspaceId}/mjolnir?kind=${vars.referenceKind}&id=${vars.materializedViewId}`,
+    method: "POST",
+  },
+  deployment_multi_mjolnir: {
+    path: (vars: ROUTE_VARS) =>
+      `/v2/workspaces/${vars.workspaceId}/multi_mjolnir`,
+    method: "POST",
+  },
+  deployment_index: {
+    path: (vars: ROUTE_VARS) =>
+      `/v2/workspaces/${vars.workspaceId}/deployment_index`,
+    method: "GET",
+  },
 
   // Websockets -----------------------------------------
   workspace_updates_ws: {
@@ -329,7 +344,7 @@ export class SdfApiClient {
 
     await retryUntil(
       async () => {
-        const dvuRoots = await this.mjolnir(
+        const dvuRoots = await this.changeSetMjolnir(
           changeSetId,
           "DependentValueComponentList",
           this.workspaceId,
@@ -377,7 +392,7 @@ export class SdfApiClient {
   }
 
   // Helper functions for interacting with MVs
-  public async mjolnir(
+  public async changeSetMjolnir(
     changeSetId: string,
     kind: string,
     id: string,
@@ -414,7 +429,7 @@ export class SdfApiClient {
     return null;
   }
 
-  public async multiMjolnir(
+  public async changeSetMultiMjolnir(
     changeSetId: string,
     mvs: { kind: string; id: string }[],
   ) {
@@ -451,6 +466,86 @@ export class SdfApiClient {
     }
   }
 
+  public async deploymentMultiMjolnir(mvs: { kind: string; id: string }[]) {
+    const response = await this.call(
+      {
+        route: "deployment_multi_mjolnir",
+        body: { requests: mvs },
+      },
+      true,
+    );
+
+    if (response?.status === 200) {
+      try {
+        const json = await response.json();
+        if (json.failed && json.failed.length > 0) {
+          console.warn(
+            "Some MVs were not found during multi mjolnir:",
+            json.failed,
+          );
+        }
+        return json.successful.map((v: any) => v.frontEndObject.data);
+      } catch (err) {
+        console.error("Error trying to parse response body as JSON", err);
+        throw new Error(`Error trying to parse response body as JSON: ${err}`);
+      }
+    } else {
+      // Fail on non-200 errors
+      console.error(
+        `Error ${response.status}: Unable to fetch multiple Deployment MVs:`,
+        await response.text(),
+      );
+      throw new Error(`Error ${response.status}: ${await response.text()}`);
+    }
+  }
+
+  public async fetchDeploymentIndex(): Promise<any> {
+    const response = await this.call(
+      {
+        route: "deployment_index",
+      },
+      true,
+    );
+    if (response?.status === 200) {
+      const json = await response.json();
+      return json;
+    } else {
+      console.warn(`Deployment index missing! This is bad!`);
+      throw new Error(`Deployment index missing! This is bad!`);
+    }
+  }
+
+  public async deploymentMjolnir(
+    kind: string,
+    id: string,
+  ): Promise<any | null> {
+    const response = await this.call(
+      {
+        route: "deployment_mjolnir",
+        routeVars: { materializedViewId: id, referenceKind: kind },
+      },
+      true,
+    );
+    if (response?.status === 200) {
+      try {
+        const json = await response.json();
+        return json.data;
+      } catch (err) {
+        console.error("Error trying to parse response body as JSON", err);
+      }
+    } else {
+      console.warn(
+        `Deployment Materialized view for ${kind} with ID ${id} not found`,
+      );
+      throw new Error(
+        "Deployment Materialized view not found for kind: " +
+          kind +
+          ", id: " +
+          id,
+      );
+    }
+  }
+
   public async fetchChangeSetIndex(
     changeSetId: string,
     timeout_ms: number = 150000,

bin/si-sdf-api-test/test_helpers.ts

@@ -254,40 +254,6 @@ export async function createComponent(
   return newComponentId;
 }
 
-export function createComponentPayload(
-  schemaVariantCategoriesMV: any,
-  schemaName: string,
-) {
-  const installedVariant = schemaVariantCategoriesMV.installed?.find(
-    (sv: any) => sv.schemaName === schemaName,
-  );
-  if (installedVariant) {
-    return {
-      schemaVariantId: installedVariant.schemaVariantId,
-      x: "0",
-      y: "0",
-      height: "0",
-      width: "0",
-      parentId: null,
-      schemaType: "installed",
-    };
-  } else {
-    const uninstalledVariant = schemaVariantCategoriesMV.uninstalled?.find(
-      (sv: any) => sv.schemaName === schemaName,
-    );
-    assert(uninstalledVariant, `Expected to find ${schemaName} schema variant`);
-    return {
-      schemaId: uninstalledVariant.schemaId,
-      schemaType: "uninstalled",
-      x: "0",
-      y: "0",
-      height: "0",
-      width: "0",
-      parentId: null,
-    };
-  }
-}
-
 // Func Helpers ------------------------------------------------------------
 
 export async function getFuncRun(
@@ -419,7 +385,7 @@ export async function abandon_all_change_sets(sdf: SdfApiClient) {
 }
 
 export async function getActions(sdf: SdfApiClient, changeSetId: string) {
-  const actions = await sdf.mjolnir(
+  const actions = await sdf.changeSetMjolnir(
     changeSetId,
     "ActionViewList",
     sdf.workspaceId,
@@ -428,52 +394,85 @@ export async function getActions(sdf: SdfApiClient, changeSetId: string) {
   return actions;
 }
 
-export async function getVariants(sdf: SdfApiClient, changeSetId: string) {
-  const schemas = await sdf.mjolnir(
-    changeSetId,
-    "SchemaVariantCategories",
-    sdf.workspaceId,
+// Used to generate the correct payload for sdf
+// If not given a Schema Id, assume it's a builtin, and find
+// the Id by the name by looking at the CachedSchemas Deployment MV
+// If given a schema Id, just try to find an installed one
+export async function createComponentPayload(
+  sdf: SdfApiClient,
+  changeSetId: string,
+  schemaName: string,
+  schemaId?: string,
+) {
+  const deploymentIndex = await sdf.fetchDeploymentIndex();
+  // console.log(deploymentIndex);
+
+  const cachedSchemasMV = deploymentIndex.frontEndObject.data.mvList.find(
+    (mv: any) => mv.kind === "CachedSchemas",
   );
-  assert(schemas, "Expected to get schemas MV");
-
-  const installedList: { kind: string; id: string }[] =
-    schemas.categories.flatMap((c: any) =>
-      c.schemaVariants
-        .filter((v: any) => v.type === "installed")
-        .map((v: any) => {
-          return {
-            kind: "SchemaVariant",
-            id: v.id,
-          };
-        }),
+  const cachedSchemasMVId = cachedSchemasMV.id;
+  let actualSchemaId = undefined;
+  if (!schemaId) {
+    // Get the builtin version, so we can get the Schema Id as we only have the name
+    const builtins = await sdf.deploymentMjolnir(
+      "CachedSchemas",
+      cachedSchemasMVId,
+    );
+
+    assert(builtins, "Expected to get schemas MV");
+    let foundSchema = builtins.schemas.find(
+      (schema: { id: string; name: string }) => schema.name === schemaName,
     );
-  let installed = [];
-  if (installedList.length > 0) {
-    let installedResp = await sdf.multiMjolnir(changeSetId, installedList);
-    assert(installedResp, "Expected to get installed variants data");
-    installed = installedResp;
+    assert(foundSchema, `Expected to find id for Schema Name ${schemaName}`);
+    actualSchemaId = foundSchema.id;
+    assert(actualSchemaId, `Expected to find id for Schema Name ${schemaName}`);
+  } else {
+    actualSchemaId = schemaId;
   }
 
-  const uninstalled = schemas.categories.flatMap((c: any) =>
-    c.schemaVariants
-      .filter((v: any) => v.type === "uninstalled")
-      .map((v: any) => {
-        const uninstalledMeta = schemas.uninstalled?.[v.id];
-        assert(
-          uninstalledMeta,
-          `Expected uninstalled metadata for schemaId ${v.id}`,
-        );
-        return uninstalledMeta;
-      }),
-  );
-  return { installed, uninstalled };
+  assert(actualSchemaId, "we have a valid schema Id!");
+  try {
+    // First is it installed?
+    const maybeInstalled = await sdf.changeSetMjolnir(
+      changeSetId,
+      "LuminorkDefaultVariant",
+      actualSchemaId,
+    );
+    // if so, return it
+    return {
+      schemaVariantId: maybeInstalled.variantId,
+      x: "0",
+      y: "0",
+      height: "0",
+      width: "0",
+      parentId: null,
+      schemaType: "installed",
+    };
+  } catch (e) {
+    console.log(
+      `LuminorkDefaultVariant not found for ${schemaName}, assuming it's uninstalled`,
+    );
+    return {
+      schemaId: actualSchemaId,
+      schemaType: "uninstalled",
+      x: "0",
+      y: "0",
+      height: "0",
+      width: "0",
+      parentId: null,
+    };
+  }
 }
 
 export async function getViews(sdf: SdfApiClient, changeSetId: string) {
-  const viewsList = await sdf.mjolnir(changeSetId, "ViewList", sdf.workspaceId);
+  const viewsList = await sdf.changeSetMjolnir(
+    changeSetId,
+    "ViewList",
+    sdf.workspaceId,
+  );
   assert(viewsList, "Expected to get views MV");
   let viewsToFetch = viewsList.views;
-  let views = await sdf.multiMjolnir(changeSetId, viewsToFetch);
+  let views = await sdf.changeSetMultiMjolnir(changeSetId, viewsToFetch);
   assert(views, "Expected to get views data");
   return views;
 }
@@ -493,7 +492,7 @@ export async function eventualMVAssert(
   }
   await retryUntil(
     async () => {
-      const mv = await sdf.mjolnir(changeSetId, kind, id);
+      const mv = await sdf.changeSetMjolnir(changeSetId, kind, id);
       if (mv) {
         try {
           if (assertFn(mv)) {

bin/si-sdf-api-test/tests/0-add_action.ts

@@ -1,6 +1,13 @@
 import assert from "node:assert";
 import { SdfApiClient } from "../sdf_api_client.ts";
-import { createComponent, createComponentPayload, eventualMVAssert, getActions, getVariants, getViews, runWithTemporaryChangeset } from "../test_helpers.ts";
+import {
+  createComponent,
+  createComponentPayload,
+  eventualMVAssert,
+  getActions,
+  getViews,
+  runWithTemporaryChangeset,
+} from "../test_helpers.ts";
 
 export default async function add_action(sdfApiClient: SdfApiClient) {
   return runWithTemporaryChangeset(sdfApiClient, add_action_inner);
@@ -14,12 +21,14 @@ export async function add_action_inner(
   // Store the original length of actions to verify later
   assert(Array.isArray(data.actions), "Expected actions to be an array");
 
-
   const actionOriginalLength = data.actions.length;
 
   // Get all Schema Variants
-  let schemaVariants = await getVariants(sdfApiClient, changeSetId);
-  let createComponentBody = createComponentPayload(schemaVariants, "AWS::EC2::Instance");
+  let createComponentBody = await createComponentPayload(
+    sdfApiClient,
+    changeSetId,
+    "AWS::EC2::Instance",
+  );
 
   // Get the views and find the default one
   const views = await getViews(sdfApiClient, changeSetId);
@@ -40,8 +49,13 @@ export async function add_action_inner(
     changeSetId,
     "ActionViewList",
     sdfApiClient.workspaceId,
-    (mv) => { return mv.actions.some((action: any) => action.componentId === newComponentId) && mv.actions.length === actionOriginalLength + 1; },
+    (mv) => {
+      return (
+        mv.actions.some(
+          (action: any) => action.componentId === newComponentId,
+        ) && mv.actions.length === actionOriginalLength + 1
+      );
+    },
     "No action with the expected componentId found",
   );
-
 }