perf(tests): cut per-file import graphs in apps/sim and shard the suite across CI runners (#7367)

* perf(tests): cut per-file import graphs in apps/sim and shard the suite across CI runners

* fix(tests): recheck abort after lazy handler load; list script tests by file

Author: waleedlatif1

.github/workflows/test-build.yml

@@ -180,13 +180,19 @@ jobs:
 
       # Runs the setup CLI's Bun tests plus each workspace's Vitest suite,
       # without `--coverage`. See the Codecov note below.
+      #
+      # apps/sim runs only its first shard here; `test-shard` below runs the
+      # other. That suite is bound by the single Vite server thread that feeds
+      # every worker — wall time is flat from 4 to 13 workers — so a bigger
+      # runner buys nothing and a second runner halves it.
       - name: Run tests
         env:
           NODE_OPTIONS: '--no-warnings --max-old-space-size=8192'
           NEXT_PUBLIC_APP_URL: 'https://www.sim.ai'
           DATABASE_URL: 'postgresql://postgres:postgres@localhost:5432/simstudio'
           ENCRYPTION_KEY: '7cf672e460e430c1fba707575c2b0e2ad5a99dddf9b7b7e3b5646e630861db1c' # dummy key for CI only
           TURBO_CACHE_DIR: .turbo
+          SIM_TEST_SHARD: 1/2
         run: bun run test
 
       - name: Check schema and migrations are in sync
@@ -202,6 +208,67 @@ jobs:
           fi
           echo "✅ Schema and migrations are in sync"
 
+  # The second half of apps/sim's Vitest suite. Everything else — lint, the
+  # audits, type-check, the other workspaces' suites — lives in `test-build`
+  # with shard 1; this job exists only because that suite cannot go faster on
+  # one machine (see the "Run tests" note there). The Turbo cache disk gets
+  # its own key so the two shards' entries do not evict each other.
+  test-shard:
+    name: Test (shard 2)
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }}
+    timeout-minutes: 15
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: 1.3.14
+
+      - name: Setup Node
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
+        with:
+          node-version: 24
+
+      - name: Mount Bun cache
+        uses: ./.github/actions/cache-mount
+        with:
+          provider: ${{ vars.CI_PROVIDER }}
+          key: ${{ github.repository }}-bun-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}
+          path: ~/.bun/install/cache
+
+      - name: Mount node_modules
+        uses: ./.github/actions/cache-mount
+        with:
+          provider: ${{ vars.CI_PROVIDER }}
+          key: ${{ github.repository }}-node-modules-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}-${{ hashFiles('bun.lock') }}
+          path: ./node_modules
+
+      - name: Mount Turbo cache
+        uses: ./.github/actions/cache-mount
+        with:
+          provider: ${{ vars.CI_PROVIDER }}
+          key: ${{ github.repository }}-turbo-cache-shard-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}
+          path: ./.turbo
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile --ignore-scripts
+
+      - name: Install ripgrep
+        run: command -v rg || (sudo apt-get update && sudo apt-get install -y ripgrep)
+
+      - name: Run tests (apps/sim shard 2/2)
+        env:
+          NODE_OPTIONS: '--no-warnings --max-old-space-size=8192'
+          NEXT_PUBLIC_APP_URL: 'https://www.sim.ai'
+          DATABASE_URL: 'postgresql://postgres:postgres@localhost:5432/simstudio'
+          ENCRYPTION_KEY: '0000000000000000000000000000000000000000000000000000000000000000' # dummy key for CI only
+          TURBO_CACHE_DIR: .turbo
+          SIM_TEST_SHARD: 2/2
+        run: bunx turbo run test --filter=@sim/app
+
   # Next.js production build, in parallel with lint + tests. Sticky disks are
   # cloned from the last committed snapshot per job and committed last-writer-
   # wins, so concurrent mounts are safe. The bun/node_modules disks are shared

apps/sim/app/api/copilot/tools/execute/route.ts

@@ -162,7 +162,7 @@ export const POST = withRouteHandler((request: NextRequest) =>
         // glob/read/grep, function execute, ...) plus the server tool router
         // fallback — the plain server-tool adapter alone rejects VFS tools
         // with "Unknown server tool".
-        ensureHandlersRegistered()
+        await ensureHandlersRegistered()
         const result = await executeTool(toolName, params, {
           userId,
           workflowId: workflowId ?? '',

apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-icon.test.ts

@@ -1,11 +1,14 @@
 /** @vitest-environment node */
 import { Workflow } from '@sim/emcn/icons'
-import { describe, expect, it } from 'vitest'
+import { describe, expect, it, vi } from 'vitest'
 import { AgentSkillsIcon } from '@/components/icons'
 import { getDocumentIcon } from '@/components/icons/document-icons'
 import { mentionIcon } from './mention-icon'
 import type { MentionKind } from './types'
 
+/** Compares real icon components by identity; the global `@/components/icons` stub in vitest.setup.ts would make that vacuous. */
+vi.unmock('@/components/icons')
+
 describe('mentionIcon', () => {
   it('uses the product-wide glyph for a known kind', () => {
     expect(mentionIcon('workflow', 'x')).toBe(Workflow)

apps/sim/blocks/blocks/harmonic.test.ts

@@ -3,11 +3,19 @@
  */
 import { describe, expect, it, vi } from 'vitest'
 
-vi.unmock('@/tools/registry')
+/**
+ * Only this service's configs are needed; the full registry is ~6,000 modules.
+ * Registration is asserted through the generated `@/tools/tool-ids`.
+ */
+vi.mock('@/tools/registry', async () => {
+  const { partialToolRegistry } = await import('@sim/testing/mocks/tool-registry.mock')
+  return { tools: partialToolRegistry(await import('@/tools/harmonic')) }
+})
 
 import { HarmonicBlock, HarmonicBlockMeta } from '@/blocks/blocks/harmonic'
 import { BLOCK_META_REGISTRY, BLOCK_REGISTRY } from '@/blocks/registry-maps'
 import { tools } from '@/tools/registry'
+import { hasToolId } from '@/tools/tool-ids'
 
 describe('HarmonicBlock', () => {
   const buildParams = HarmonicBlock.tools.config!.params!
@@ -44,7 +52,8 @@ describe('HarmonicBlock', () => {
 
     for (const operation of operationIds) {
       const tool = tools[operation]
-      expect(tool?.id, `missing registry entry ${operation}`).toBe(operation)
+      expect(hasToolId(operation), `missing registry entry ${operation}`).toBe(true)
+      expect(tool?.id).toBe(operation)
 
       const blockOutputs = Object.entries(HarmonicBlock.outputs)
         .filter(([, output]) => {

apps/sim/blocks/blocks/outlook.test.ts

@@ -1,15 +1,18 @@
 /**
  * @vitest-environment node
  */
-import { describe, expect, it } from 'vitest'
+import { describe, expect, it, vi } from 'vitest'
 import { tools as toolRegistry } from '@/tools/registry'
 import { OutlookBlock } from './outlook'
 
 /**
- * Uses the real tool registry: these assertions are about tool registration and
- * params, which the global `@/tools/registry` mock in vitest.setup.ts empties.
+ * Only this service's configs are needed; the full registry is ~6,000 modules.
+ * Registration is asserted through the generated `@/tools/tool-ids`.
  */
-vi.unmock('@/tools/registry')
+vi.mock('@/tools/registry', async () => {
+  const { partialToolRegistry } = await import('@sim/testing/mocks/tool-registry.mock')
+  return { tools: partialToolRegistry(await import('@/tools/outlook')) }
+})
 
 const block = OutlookBlock
 

apps/sim/blocks/brand-icon.test.tsx

@@ -8,6 +8,9 @@ import { OAUTH_PROVIDERS } from '@/lib/oauth'
 import { BrandIcon, withBrandIcon } from '@/blocks/brand-icon'
 import { getAllBlocks } from '@/blocks/registry'
 
+/** Compares real icon components by identity; the global `@/components/icons` stub in vitest.setup.ts would make that vacuous. */
+vi.unmock('@/components/icons')
+
 vi.mocked(getAllBlocks).mockReturnValue([
   { icon: DropboxIcon, iconColor: '#0061FF' },
 ] as unknown as ReturnType<typeof getAllBlocks>)

apps/sim/lib/billing/core/limit-notifications.ts

@@ -3,15 +3,13 @@ import { member, organization, settings, user, userStats } from '@sim/db/schema'
 import { createLogger } from '@sim/logger'
 import { isOrgAdminRole } from '@sim/platform-authz/workspace'
 import { and, eq, sql } from 'drizzle-orm'
-import { getLimitEmailSubject, renderLimitThresholdEmail } from '@/components/emails'
 import type { HighestPrioritySubscription } from '@/lib/billing/core/plan'
 import { getHighestPrioritySubscription } from '@/lib/billing/core/subscription'
 import type { BillingEntity } from '@/lib/billing/core/usage-log'
 import { isOrgScopedSubscription } from '@/lib/billing/subscriptions/utils'
 import { buildUpgradeHref, type UpgradeReason } from '@/lib/billing/upgrade-reasons'
 import { isBillingEnabled } from '@/lib/core/config/env-flags'
 import { getBaseUrl } from '@/lib/core/utils/urls'
-import { sendEmail } from '@/lib/messaging/email/mailer'
 import { getEmailPreferences } from '@/lib/messaging/email/unsubscribe'
 
 const logger = createLogger('LimitNotifications')
@@ -218,6 +216,11 @@ export async function maybeSendLimitThresholdEmail(params: {
     const percentUsed = Math.min(100, Math.round(percent))
     const upgradeLink = `${getBaseUrl()}${buildUpgradeHref(params.workspaceId, category)}`
 
+    const [{ getLimitEmailSubject, renderLimitThresholdEmail }, { sendEmail }] = await Promise.all([
+      import('@/components/emails'),
+      import('@/lib/messaging/email/mailer'),
+    ])
+
     let sent = 0
     for (const r of recipients) {
       try {

apps/sim/lib/billing/core/usage.ts

@@ -4,14 +4,6 @@ import { createLogger } from '@sim/logger'
 import { isOrgAdminRole } from '@sim/platform-authz/workspace'
 import { generateId } from '@sim/utils/id'
 import { and, eq, isNull, sql } from 'drizzle-orm'
-import {
-  getEmailSubject,
-  getLimitEmailSubject,
-  renderCreditsExhaustedEmail,
-  renderFreeTierUpgradeEmail,
-  renderUsageLimitReachedEmail,
-  renderUsageThresholdEmail,
-} from '@/components/emails'
 import { getEffectiveBillingStatus } from '@/lib/billing/core/access'
 import { defaultBillingPeriod } from '@/lib/billing/core/billing-period'
 import {
@@ -45,11 +37,23 @@ import { Decimal, toDecimal, toNumber } from '@/lib/billing/utils/decimal'
 import { isBillingEnabled } from '@/lib/core/config/env-flags'
 import { getBaseUrl } from '@/lib/core/utils/urls'
 import type { DbClient } from '@/lib/db/types'
-import { sendEmail } from '@/lib/messaging/email/mailer'
 import { getEmailPreferences } from '@/lib/messaging/email/unsubscribe'
 
 const logger = createLogger('UsageManagement')
 
+/**
+ * Email rendering pulls the React templates and every mail provider into the
+ * module graph, which is ~1.2s of imports on every route that reaches billing
+ * attribution. Load it only when a threshold email is actually being sent.
+ */
+async function loadEmailDelivery() {
+  const [emails, mailer] = await Promise.all([
+    import('@/components/emails'),
+    import('@/lib/messaging/email/mailer'),
+  ])
+  return { ...emails, sendEmail: mailer.sendEmail }
+}
+
 export interface OrgUsageLimitResult {
   limit: number
   minimum: number
@@ -773,6 +777,7 @@ export async function maybeSendUsageThresholdEmail(params: {
         const prefs = await getEmailPreferences(email)
         if (prefs?.unsubscribeAll || prefs?.unsubscribeNotifications) return
 
+        const { renderUsageThresholdEmail, getEmailSubject, sendEmail } = await loadEmailDelivery()
         const html = await renderUsageThresholdEmail({
           userName: name,
           planName: params.planName,
@@ -798,6 +803,7 @@ export async function maybeSendUsageThresholdEmail(params: {
         const prefs = await getEmailPreferences(email)
         if (prefs?.unsubscribeAll || prefs?.unsubscribeNotifications) return
 
+        const { renderFreeTierUpgradeEmail, getEmailSubject, sendEmail } = await loadEmailDelivery()
         const html = await renderFreeTierUpgradeEmail({
           userName: name,
           percentUsed: Math.min(100, Math.round(params.percentAfter)),
@@ -830,6 +836,13 @@ export async function maybeSendUsageThresholdEmail(params: {
         const prefs = await getEmailPreferences(email)
         if (prefs?.unsubscribeAll || prefs?.unsubscribeNotifications) return
 
+        const {
+          renderCreditsExhaustedEmail,
+          renderUsageLimitReachedEmail,
+          getEmailSubject,
+          getLimitEmailSubject,
+          sendEmail,
+        } = await loadEmailDelivery()
         const html = useFreeCopy
           ? await renderCreditsExhaustedEmail({
               userName: name,

apps/sim/lib/catalog/projection/catalog-sweep.test.ts

@@ -41,6 +41,13 @@ import { getBlockRegistry } from '@/blocks/registry'
 import { CONNECTOR_META_REGISTRY } from '@/connectors/registry'
 import { getToolIds } from '@/tools/tool-ids'
 
+/**
+ * Asserts real tool params and outputs, which the global `@/tools/metadata`
+ * and `@/tools/metadata-outputs` mocks in vitest.setup.ts empty.
+ */
+vi.unmock('@/tools/metadata')
+vi.unmock('@/tools/metadata-outputs')
+
 /** Hosted deployment: the state under which every declared hosted key is published. */
 const HOSTED: CatalogDeployment = { hostedKeys: true }
 

apps/sim/lib/copilot/request/tools/executor.ts

@@ -505,6 +505,8 @@ async function executeToolAndReportInner(
     })
   }
 
+  // Loads the handler map on first use; the abort check below covers that wait.
+  await ensureHandlersRegistered()
   if (abortRequested(context, execContext, options)) {
     markToolCallCancelled('Request aborted before tool execution')
     markToolResultSeen(toolCall.id)
@@ -608,7 +610,6 @@ async function executeToolAndReportInner(
   }
 
   try {
-    ensureHandlersRegistered()
     let result = await executeToolWithWatchdog(toolCall, toolExecutionContext)
     if (toolCall.endTime || isTerminalToolCallStatus(toolCall.status)) {
       endToolSpanFromTerminalState()