Database tools

.gitignore

@@ -65,4 +65,5 @@ docs/.contentlayer
 docs/.content-collections
 
 # database instantiation
-**/postgres_data/
+**/postgres_data/
+.qodo

Dockerfile

@@ -1,4 +1,7 @@
-FROM node:20-alpine
+FROM node:18-alpine
+
+# Install build dependencies
+RUN apk add --no-cache python3 make g++
 
 # Set working directory
 WORKDIR /app

packages/sdk/src/generated/registry.ts

@@ -77,10 +77,14 @@ export const availableBlocks = [
   "google_drive",
   "google_sheets",
   "jina",
+  "mongodb",
+  "mysql",
   "notion",
   "openai",
   "pinecone",
+  "postgresql",
   "reddit",
+  "redis",
   "router",
   "serper",
   "slack",

sim/app/api/databases/mysql/route.ts

@@ -0,0 +1,163 @@
+import { createPool, PoolOptions, ResultSetHeader, RowDataPacket } from 'mysql2/promise'
+import { NextResponse } from 'next/server'
+import { getMySQLConfig } from '@/config/database'
+
+export async function POST(request: Request) {
+  try {
+    const body = await request.json()
+    console.log('[MySQL API] Received request:', {
+      ...body,
+      password: body.password ? '[REDACTED]' : undefined,
+      connection: body.connection ? { 
+        ...body.connection,
+        password: body.connection.password ? '[REDACTED]' : undefined 
+      } : undefined
+    })
+
+    // Get connection parameters from either:
+    // 1. Request body
+    // 2. Environment variables
+    let connection = body.connection || {
+      host: body.host,
+      port: body.port,
+      user: body.user || body.username,
+      password: body.password,
+      database: body.database,
+      ssl: body.ssl === 'true'
+    }
+
+    // If no connection details provided in the request, try to get from environment
+    if (!connection.host || !connection.user || !connection.password || !connection.database) {
+      const envConfig = getMySQLConfig()
+      if (envConfig) {
+        connection = {
+          ...connection,
+          host: connection.host || envConfig.host,
+          port: connection.port || envConfig.port,
+          user: connection.user || envConfig.user,
+          password: connection.password || envConfig.password,
+          database: connection.database || envConfig.database,
+          ssl: connection.ssl !== undefined ? connection.ssl : envConfig.ssl
+        }
+      }
+    }
+
+    // Check for required connection parameters
+    const hasRequiredParams = {
+      hasConnection: !!connection,
+      hasHost: !!connection?.host,
+      hasUsername: !!(connection?.user || connection?.username),
+      hasPassword: !!connection?.password,
+      hasDatabase: !!connection?.database
+    }
+
+    console.log('[MySQL API] Connection parameters check:', hasRequiredParams)
+
+    if (!hasRequiredParams.hasConnection || !hasRequiredParams.hasHost || 
+        !hasRequiredParams.hasUsername || !hasRequiredParams.hasPassword || 
+        !hasRequiredParams.hasDatabase) {
+      console.log('[MySQL API] Missing required connection parameters:', hasRequiredParams)
+      throw new Error('Missing required connection parameters')
+    }
+
+    // Create MySQL connection pool
+    console.log('[MySQL API] Creating connection pool with config:', {
+      ...connection,
+      password: '[REDACTED]'
+    })
+
+    const poolConfig: PoolOptions = {
+      host: connection.host,
+      port: parseInt(connection.port || '3306'),
+      user: connection.user || connection.username,
+      password: connection.password,
+      database: connection.database,
+      ssl: connection.ssl === 'true' ? { rejectUnauthorized: false } : undefined
+    }
+
+    console.log('[MySQL API] Attempting to create connection pool with config:', {
+      ...poolConfig,
+      password: '[REDACTED]'
+    })
+
+    const pool = createPool(poolConfig)
+
+    // Test the connection with a simple query
+    console.log('[MySQL API] Testing connection with SELECT 1')
+    try {
+      await pool.query('SELECT 1')
+      console.log('[MySQL API] Connection test successful')
+    } catch (testError: any) {
+      console.error('[MySQL API] Connection test failed:', testError)
+      throw testError
+    }
+
+    // Execute query based on operation
+    const { operation, query, params, options } = body
+    console.log('[MySQL API] Executing query:', { operation, query, params, options })
+
+    let result
+    switch (operation?.toLowerCase()) {
+      case 'select':
+        result = await pool.query<RowDataPacket[]>(query, params)
+        return NextResponse.json({ rows: result[0], fields: result[1] })
+
+      case 'insert':
+      case 'update':
+      case 'delete':
+        result = await pool.query<ResultSetHeader>(query, params)
+        return NextResponse.json({ affectedRows: result[0].affectedRows })
+
+      case 'execute':
+        result = await pool.query(query, params)
+        return NextResponse.json({ result })
+
+      default:
+        throw new Error(`Unsupported operation: ${operation}`)
+    }
+  } catch (error: any) {
+    console.error('[MySQL API] Error:', error)
+    console.error('[MySQL API] Error details:', {
+      name: error.name,
+      message: error.message,
+      code: error.code,
+      errno: error.errno,
+      sqlState: error.sqlState,
+      sqlMessage: error.sqlMessage
+    })
+
+    // Handle specific MySQL errors
+    if (error.code === 'ER_ACCESS_DENIED_ERROR') {
+      return NextResponse.json(
+        { error: 'Access denied. Please check your username and password.' },
+        { status: 401 }
+      )
+    }
+
+    if (error.code === 'ECONNREFUSED') {
+      return NextResponse.json(
+        { error: 'Could not connect to MySQL server. Please check if the server is running and accessible.' },
+        { status: 503 }
+      )
+    }
+
+    if (error.code === 'ER_BAD_DB_ERROR') {
+      return NextResponse.json(
+        { error: 'Database does not exist.' },
+        { status: 404 }
+      )
+    }
+
+    if (error.code === 'ER_NO_SUCH_TABLE') {
+      return NextResponse.json(
+        { error: 'Table does not exist.' },
+        { status: 404 }
+      )
+    }
+
+    return NextResponse.json(
+      { error: error.message || 'An error occurred while executing the query.' },
+      { status: 500 }
+    )
+  }
+} 

sim/app/api/databases/postgresql/route.ts

@@ -0,0 +1,94 @@
+import { Pool, FieldDef } from 'pg'
+import { NextResponse } from 'next/server'
+import { getPostgreSQLConfig } from '@/config/database'
+
+export async function POST(request: Request) {
+  try {
+    console.log('[PostgreSQL API] Received request')
+
+    const params = await request.json()
+    console.log('[PostgreSQL API] Parsed request params:', {
+      ...params,
+      connection: {
+        ...params.connection,
+        password: '[REDACTED]'
+      }
+    })
+
+    let { connection, operation, query, params: queryParams, options } = params
+
+    // If no connection details provided in the request, try to get from environment
+    if (!connection || !connection.host || !connection.username || !connection.password || !connection.database) {
+      const envConfig = getPostgreSQLConfig()
+      if (envConfig) {
+        connection = {
+          ...connection,
+          host: connection?.host || envConfig.host,
+          port: connection?.port || envConfig.port,
+          username: connection?.username || envConfig.user,
+          password: connection?.password || envConfig.password,
+          database: connection?.database || envConfig.database,
+          ssl: connection?.ssl !== undefined ? connection.ssl : envConfig.ssl
+        }
+      }
+    }
+
+    // Check for required connection parameters
+    if (!connection || !connection.host || !connection.username || !connection.password || !connection.database) {
+      console.error('[PostgreSQL API] Missing required connection parameters')
+      throw new Error('Missing required connection parameters')
+    }
+
+    // Create connection pool
+    const { host, port, username, password, database, ssl } = connection
+    console.log('[PostgreSQL API] Creating connection pool:', { host, port, database, ssl })
+
+    const pool = new Pool({
+      host,
+      port,
+      user: username,
+      password,
+      database,
+      ssl: ssl ? { rejectUnauthorized: false } : undefined
+    })
+
+    // Test connection
+    try {
+      console.log('[PostgreSQL API] Testing connection...')
+      await pool.query('SELECT 1')
+      console.log('[PostgreSQL API] Connection test successful')
+    } catch (error) {
+      console.error('[PostgreSQL API] Connection test failed:', error)
+      throw error
+    }
+
+    // Execute query
+    console.log('[PostgreSQL API] Executing query:', query)
+    const result = await pool.query(query, queryParams || [])
+    console.log('[PostgreSQL API] Query result:', {
+      rowCount: result.rowCount,
+      fieldCount: result.fields?.length,
+      rows: result.rows
+    })
+
+    // Close pool
+    await pool.end()
+    console.log('[PostgreSQL API] Connection pool closed')
+
+    return NextResponse.json({
+      rows: result.rows,
+      rowCount: result.rowCount,
+      fields: result.fields?.map((field: FieldDef) => ({
+        name: field.name,
+        type: field.dataTypeID
+      })) || []
+    })
+
+  } catch (error) {
+    console.error('[PostgreSQL API] Error:', error)
+    return NextResponse.json(
+      { error: error instanceof Error ? error.message : 'Unknown error occurred' },
+      { status: 500 }
+    )
+  }
+} 

sim/app/api/webhooks/trigger/[path]/route.test.ts

@@ -18,11 +18,10 @@ describe('Webhook Trigger API Route', () => {
     // Mock all dependencies
     mockExecutionDependencies()
 
-    // Mock Redis for duplicate detection
-    vi.doMock('@/lib/redis', () => ({
+    // Mock deduplication system
+    vi.doMock('@/lib/deduplication', () => ({
       hasProcessedMessage: vi.fn().mockResolvedValue(false),
-      markMessageAsProcessed: vi.fn().mockResolvedValue(true),
-      closeRedisConnection: vi.fn().mockResolvedValue(undefined),
+      markMessageAsProcessed: vi.fn().mockResolvedValue(undefined),
     }))
 
     // Mock database with webhook data
@@ -252,16 +251,6 @@ describe('Webhook Trigger API Route', () => {
     // Mock the path param
     const params = Promise.resolve({ path: 'test-path' })
 
-    // Import Redis mocks
-    const hasProcessedMessageMock = vi.fn().mockResolvedValue(false)
-    const markMessageAsProcessedMock = vi.fn().mockResolvedValue(true)
-
-    vi.doMock('@/lib/redis', () => ({
-      hasProcessedMessage: hasProcessedMessageMock,
-      markMessageAsProcessed: markMessageAsProcessedMock,
-      closeRedisConnection: vi.fn().mockResolvedValue(undefined),
-    }))
-
     // Import the handler after mocks are set up
     const { POST } = await import('./route')
 
@@ -323,10 +312,9 @@ describe('Webhook Trigger API Route', () => {
     const markMessageAsProcessedMock = vi.fn().mockResolvedValue(true)
 
     // Mock hasProcessedMessage to return true (duplicate)
-    vi.doMock('@/lib/redis', () => ({
+    vi.doMock('@/lib/deduplication', () => ({
       hasProcessedMessage: hasProcessedMessageMock,
       markMessageAsProcessed: markMessageAsProcessedMock,
-      closeRedisConnection: vi.fn().mockResolvedValue(undefined),
     }))
 
     // Create executor mock to verify it's not called