Skip to content

fix(logs-page): optimize logs retrieval queries, consolidate useEffects to prevent dup calls #845

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
icecrasher321 merged 3 commits into from
Aug 1, 2025
Merged

fix(logs-page): optimize logs retrieval queries, consolidate useEffects to prevent dup calls #845

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4b204c9
fix(logs-page): optimize loading times by improving query, removing u…
icecrasher321 Aug 1, 2025
6ef4066
add migration files
icecrasher321 Aug 1, 2025
0906c6d
remove fake min loading time
icecrasher321 Aug 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 70 additions & 65 deletions apps/sim/app/api/logs/route.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,33 +75,50 @@ export async function GET(request: NextRequest) {
const { searchParams } = new URL(request.url)
const params = QueryParamsSchema.parse(Object.fromEntries(searchParams.entries()))

const workflowConditions = and(
eq(workflow.workspaceId, params.workspaceId),
eq(permissions.userId, userId),
eq(permissions.entityType, 'workspace')
)

const userWorkflows = await db
.select({ id: workflow.id, folderId: workflow.folderId })
.from(workflow)
.leftJoin(
const baseQuery = db
.select({
id: workflowExecutionLogs.id,
workflowId: workflowExecutionLogs.workflowId,
executionId: workflowExecutionLogs.executionId,
stateSnapshotId: workflowExecutionLogs.stateSnapshotId,
level: workflowExecutionLogs.level,
message: workflowExecutionLogs.message,
trigger: workflowExecutionLogs.trigger,
startedAt: workflowExecutionLogs.startedAt,
endedAt: workflowExecutionLogs.endedAt,
totalDurationMs: workflowExecutionLogs.totalDurationMs,
blockCount: workflowExecutionLogs.blockCount,
successCount: workflowExecutionLogs.successCount,
errorCount: workflowExecutionLogs.errorCount,
skippedCount: workflowExecutionLogs.skippedCount,
totalCost: workflowExecutionLogs.totalCost,
totalInputCost: workflowExecutionLogs.totalInputCost,
totalOutputCost: workflowExecutionLogs.totalOutputCost,
totalTokens: workflowExecutionLogs.totalTokens,
metadata: workflowExecutionLogs.metadata,
createdAt: workflowExecutionLogs.createdAt,
workflowName: workflow.name,
workflowDescription: workflow.description,
workflowColor: workflow.color,
workflowFolderId: workflow.folderId,
workflowUserId: workflow.userId,
workflowWorkspaceId: workflow.workspaceId,
workflowCreatedAt: workflow.createdAt,
workflowUpdatedAt: workflow.updatedAt,
})
.from(workflowExecutionLogs)
.innerJoin(workflow, eq(workflowExecutionLogs.workflowId, workflow.id))
.innerJoin(
permissions,
and(
eq(permissions.entityType, 'workspace'),
eq(permissions.entityId, workflow.workspaceId),
eq(permissions.userId, userId)
)
)
.where(workflowConditions)

const userWorkflowIds = userWorkflows.map((w) => w.id)

if (userWorkflowIds.length === 0) {
return NextResponse.json({ data: [], total: 0 }, { status: 200 })
}

// Build conditions for logs
let conditions: SQL | undefined = inArray(workflowExecutionLogs.workflowId, userWorkflowIds)
// Build conditions for the joined query
let conditions: SQL | undefined = eq(workflow.workspaceId, params.workspaceId)

// Filter by level
if (params.level && params.level !== 'all') {
Expand All @@ -111,27 +128,16 @@ export async function GET(request: NextRequest) {
// Filter by specific workflow IDs
if (params.workflowIds) {
const workflowIds = params.workflowIds.split(',').filter(Boolean)
const filteredWorkflowIds = workflowIds.filter((id) => userWorkflowIds.includes(id))
if (filteredWorkflowIds.length > 0) {
conditions = and(
conditions,
inArray(workflowExecutionLogs.workflowId, filteredWorkflowIds)
)
if (workflowIds.length > 0) {
conditions = and(conditions, inArray(workflow.id, workflowIds))
}
}

// Filter by folder IDs
if (params.folderIds) {
const folderIds = params.folderIds.split(',').filter(Boolean)
const workflowsInFolders = userWorkflows
.filter((w) => w.folderId && folderIds.includes(w.folderId))
.map((w) => w.id)

if (workflowsInFolders.length > 0) {
conditions = and(
conditions,
inArray(workflowExecutionLogs.workflowId, workflowsInFolders)
)
if (folderIds.length > 0) {
conditions = and(conditions, inArray(workflow.folderId, folderIds))
}
}

Expand Down Expand Up @@ -166,21 +172,30 @@ export async function GET(request: NextRequest) {
)
}

// Execute the query
const logs = await db
.select()
.from(workflowExecutionLogs)
// Execute the query using the optimized join
const logs = await baseQuery
.where(conditions)
.orderBy(desc(workflowExecutionLogs.startedAt))
.limit(params.limit)
.offset(params.offset)

// Get total count for pagination
const countResult = await db
// Get total count for pagination using the same join structure
const countQuery = db
.select({ count: sql<number>`count(*)` })
.from(workflowExecutionLogs)
.innerJoin(workflow, eq(workflowExecutionLogs.workflowId, workflow.id))
.innerJoin(
permissions,
and(
eq(permissions.entityType, 'workspace'),
eq(permissions.entityId, workflow.workspaceId),
eq(permissions.userId, userId)
)
)
.where(conditions)

const countResult = await countQuery

const count = countResult[0]?.count || 0

// Block executions are now extracted from trace spans instead of separate table
Expand Down Expand Up @@ -271,7 +286,7 @@ export async function GET(request: NextRequest) {
}
}

// Transform to clean log format
// Transform to clean log format with workflow data included
const enhancedLogs = logs.map((log) => {
const blockExecutions = blockExecutionsByExecution[log.executionId] || []

Expand All @@ -298,6 +313,19 @@ export async function GET(request: NextRequest) {
models: (log.metadata as any)?.models || {},
}

// Build workflow object from joined data
const workflow = {
id: log.workflowId,
name: log.workflowName,
description: log.workflowDescription,
color: log.workflowColor,
folderId: log.workflowFolderId,
userId: log.workflowUserId,
workspaceId: log.workflowWorkspaceId,
createdAt: log.workflowCreatedAt,
updatedAt: log.workflowUpdatedAt,
}

return {
id: log.id,
workflowId: log.workflowId,
Expand All @@ -307,6 +335,7 @@ export async function GET(request: NextRequest) {
duration: log.totalDurationMs ? `${log.totalDurationMs}ms` : null,
trigger: log.trigger,
createdAt: log.startedAt.toISOString(),
workflow: params.includeWorkflow ? workflow : undefined,
metadata: {
totalDuration: log.totalDurationMs,
cost: costSummary,
Expand All @@ -323,30 +352,6 @@ export async function GET(request: NextRequest) {
}
})

if (params.includeWorkflow) {
const workflowIds = [...new Set(logs.map((log) => log.workflowId))]
const workflowConditions = inArray(workflow.id, workflowIds)

const workflowData = await db.select().from(workflow).where(workflowConditions)
const workflowMap = new Map(workflowData.map((w) => [w.id, w]))

const logsWithWorkflow = enhancedLogs.map((log) => ({
...log,
workflow: workflowMap.get(log.workflowId) || null,
}))

return NextResponse.json(
{
data: logsWithWorkflow,
total: Number(count),
page: Math.floor(params.offset / params.limit) + 1,
pageSize: params.limit,
totalPages: Math.ceil(Number(count) / params.limit),
},
{ status: 200 }
)
}

// Include block execution data if requested
if (params.includeBlocks) {
// Block executions are now extracted from stored trace spans in metadata
Expand Down
Loading