Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
74a8ad5
feat: enhance resolution search with high-res imagery and image display
google-labs-jules[bot] Jan 31, 2026
76ace98
feat: add google maps image render to resolution search and geospatia…
google-labs-jules[bot] Jan 31, 2026
830d53a
feat: enhance resolution search with multi-image previews and fix fol…
google-labs-jules[bot] Jan 31, 2026
0732290
feat: enhance resolution search with multi-image previews and fix fol…
google-labs-jules[bot] Feb 1, 2026
cd27290
feat: reset branch to 76ace98 and synchronize with main
google-labs-jules[bot] Feb 3, 2026
f89e3f7
feat: fix follow-ups and integrate drawing context
google-labs-jules[bot] Feb 3, 2026
07f633d
feat: synchronize with main and fix refresh ReferenceError in Chat co…
google-labs-jules[bot] Feb 3, 2026
4b9e1e0
fix: resolution search, specialized queries, and follow-up functionality
google-labs-jules[bot] Feb 4, 2026
649e109
Merge pull request #492 from QueueLab/fix/resolution-search-and-ui-is…
ngoiyaeric Feb 4, 2026
6ea0456
Fix React Server Components CVE vulnerabilities
vercel[bot] Feb 4, 2026
b4c7ca0
Merge pull request #495 from QueueLab/vercel/react-server-components-…
ngoiyaeric Feb 4, 2026
6b7118f
fix: add global safety fallback for HTMX event handlers to prevent ex…
ngoiyaeric Feb 5, 2026
5c25e9f
Merge pull request #496 from QueueLab/fix/sse-error-extension-conflict
ngoiyaeric Feb 5, 2026
8fb9809
Merge origin/main and resolve conflicts
ngoiyaeric Feb 5, 2026
6289c7f
chore: update bun.lock after synchronization
ngoiyaeric Feb 8, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 71 additions & 37 deletions app/actions.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -13,28 +13,26 @@ import { Spinner } from '@/components/ui/spinner'
import { Section } from '@/components/section'
import { FollowupPanel } from '@/components/followup-panel'
import { inquire, researcher, taskManager, querySuggestor, resolutionSearch, type DrawnFeature } from '@/lib/agents'
// Removed import of useGeospatialToolMcp as it no longer exists and was incorrectly used here.
// The geospatialTool (if used by agents like researcher) now manages its own MCP client.
import { writer } from '@/lib/agents/writer'
import { saveChat, getSystemPrompt } from '@/lib/actions/chat' // Added getSystemPrompt
import { saveChat, getSystemPrompt } from '@/lib/actions/chat'
import { Chat, AIMessage } from '@/lib/types'
import { UserMessage } from '@/components/user-message'
import { BotMessage } from '@/components/message'
import { SearchSection } from '@/components/search-section'
import SearchRelated from '@/components/search-related'
import { GeoJsonLayer } from '@/components/map/geojson-layer'
import { MapDataUpdater } from '@/components/map/map-data-updater'
import { ResolutionImage } from '@/components/resolution-image'
import { CopilotDisplay } from '@/components/copilot-display'
import RetrieveSection from '@/components/retrieve-section'
import { VideoSearchSection } from '@/components/video-search-section'
import { MapQueryHandler } from '@/components/map/map-query-handler' // Add this import
import { MapQueryHandler } from '@/components/map/map-query-handler'

// Define the type for related queries
type RelatedQueries = {
items: { query: string }[]
}

// Removed mcp parameter from submit, as geospatialTool now handles its client.
async function submit(formData?: FormData, skip?: boolean) {
'use server'

Expand All @@ -44,16 +42,17 @@ async function submit(formData?: FormData, skip?: boolean) {
const isCollapsed = createStreamableValue(false)

const action = formData?.get('action') as string;
const drawnFeaturesString = formData?.get('drawnFeatures') as string;
let drawnFeatures: DrawnFeature[] = [];
try {
drawnFeatures = drawnFeaturesString ? JSON.parse(drawnFeaturesString) : [];
} catch (e) {
console.error('Failed to parse drawnFeatures:', e);
}

if (action === 'resolution_search') {
const file = formData?.get('file') as File;
const timezone = (formData?.get('timezone') as string) || 'UTC';
const drawnFeaturesString = formData?.get('drawnFeatures') as string;
let drawnFeatures: DrawnFeature[] = [];
try {
drawnFeatures = drawnFeaturesString ? JSON.parse(drawnFeaturesString) : [];
} catch (e) {
console.error('Failed to parse drawnFeatures:', e);
}

if (!file) {
throw new Error('No file provided for resolution search.');
Expand All @@ -62,7 +61,6 @@ async function submit(formData?: FormData, skip?: boolean) {
const buffer = await file.arrayBuffer();
const dataUrl = `data:${file.type};base64,${Buffer.from(buffer).toString('base64')}`;

// Get the current messages, excluding tool-related ones.
const messages: CoreMessage[] = [...(aiState.get().messages as any[])].filter(
message =>
message.role !== 'tool' &&
Expand All @@ -72,16 +70,12 @@ async function submit(formData?: FormData, skip?: boolean) {
message.type !== 'resolution_search_result'
);

// The user's prompt for this action is static.
const userInput = 'Analyze this map view.';

// Construct the multimodal content for the user message.
const content: CoreMessage['content'] = [
{ type: 'text', text: userInput },
{ type: 'image', image: dataUrl, mimeType: file.type }
];

// Add the new user message to the AI state.
aiState.update({
...aiState.get(),
messages: [
Expand All @@ -91,12 +85,11 @@ async function submit(formData?: FormData, skip?: boolean) {
});
messages.push({ role: 'user', content });

// Create a streamable value for the summary.
const summaryStream = createStreamableValue<string>('');
const summaryStream = createStreamableValue<string>('Analyzing map view...');
const groupeId = nanoid();

async function processResolutionSearch() {
try {
// Call the simplified agent, which now returns a stream.
const streamResult = await resolutionSearch(messages, timezone, drawnFeatures);

let fullSummary = '';
Expand All @@ -108,22 +101,41 @@ async function submit(formData?: FormData, skip?: boolean) {
}

const analysisResult = await streamResult.object;

// Mark the summary stream as done with the result.
summaryStream.done(analysisResult.summary || 'Analysis complete.');

if (analysisResult.geoJson) {
uiStream.append(
<GeoJsonLayer
id={groupeId}
data={analysisResult.geoJson as FeatureCollection}
/>
);
}

messages.push({ role: 'assistant', content: analysisResult.summary || 'Analysis complete.' });

const sanitizedMessages: CoreMessage[] = messages.map(m => {
if (Array.isArray(m.content)) {
return {
...m,
content: m.content.filter(part => part.type !== 'image')
content: m.content.filter((part: any) => part.type !== 'image')
} as CoreMessage
}
return m
})

const currentMessages = aiState.get().messages;
const sanitizedHistory = currentMessages.map(m => {
if (m.role === "user" && Array.isArray(m.content)) {
return {
...m,
content: m.content.map((part: any) =>
part.type === "image" ? { ...part, image: "IMAGE_PROCESSED" } : part
)
}
}
return m
});
const relatedQueries = await querySuggestor(uiStream, sanitizedMessages);
uiStream.append(
<Section title="Follow-up">
Expand All @@ -133,8 +145,6 @@ async function submit(formData?: FormData, skip?: boolean) {

await new Promise(resolve => setTimeout(resolve, 500));

const groupeId = nanoid();

aiState.done({
...aiState.get(),
messages: [
Expand All @@ -148,7 +158,10 @@ async function submit(formData?: FormData, skip?: boolean) {
{
id: groupeId,
role: 'assistant',
content: JSON.stringify(analysisResult),
content: JSON.stringify({
...analysisResult,
image: dataUrl
}),
type: 'resolution_search_result'
},
{
Expand All @@ -174,12 +187,11 @@ async function submit(formData?: FormData, skip?: boolean) {
}
}

// Start the background process without awaiting it.
processResolutionSearch();

// Immediately update the UI stream with the BotMessage component.
uiStream.update(
<Section title="response">
<ResolutionImage src={dataUrl} />
<BotMessage content={summaryStream.value} />
</Section>
);
Expand All @@ -199,7 +211,17 @@ async function submit(formData?: FormData, skip?: boolean) {
message.type !== 'related' &&
message.type !== 'end' &&
message.type !== 'resolution_search_result'
)
).map(m => {
if (Array.isArray(m.content)) {
return {
...m,
content: m.content.filter((part: any) =>
part.type !== "image" || (typeof part.image === "string" && part.image.startsWith("data:"))
)
} as any
}
return m
})

const groupeId = nanoid()
const useSpecificAPI = process.env.USE_SPECIFIC_API_FOR_WRITER === 'true'
Expand Down Expand Up @@ -273,9 +295,8 @@ async function submit(formData?: FormData, skip?: boolean) {
</Section>
);

uiStream.append(answerSection);
uiStream.update(answerSection);

const groupeId = nanoid();
const relatedQueries = { items: [] };

aiState.done({
Expand Down Expand Up @@ -392,7 +413,6 @@ async function submit(formData?: FormData, skip?: boolean) {
}

const hasImage = messageParts.some(part => part.type === 'image')
// Properly type the content based on whether it contains images
const content: CoreMessage['content'] = hasImage
? messageParts as CoreMessage['content']
: messageParts.map(part => part.text).join('\n')
Expand Down Expand Up @@ -426,7 +446,6 @@ async function submit(formData?: FormData, skip?: boolean) {

const userId = 'anonymous'
const currentSystemPrompt = (await getSystemPrompt(userId)) || ''

const mapProvider = formData?.get('mapProvider') as 'mapbox' | 'google'

async function processEvents() {
Expand Down Expand Up @@ -475,7 +494,8 @@ async function submit(formData?: FormData, skip?: boolean) {
streamText,
messages,
mapProvider,
useSpecificAPI
useSpecificAPI,
drawnFeatures
)
answer = fullResponse
toolOutputs = toolResponses
Expand Down Expand Up @@ -717,12 +737,10 @@ export const getUIStateFromAIState = (aiState: AIState): UIState => {
case 'input_related':
let messageContent: string | any[]
try {
// For backward compatibility with old messages that stored a JSON string
const json = JSON.parse(content as string)
messageContent =
type === 'input' ? json.input : json.related_query
} catch (e) {
// New messages will store the content array or string directly
messageContent = content
}
return {
Expand Down Expand Up @@ -780,11 +798,13 @@ export const getUIStateFromAIState = (aiState: AIState): UIState => {
case 'resolution_search_result': {
const analysisResult = JSON.parse(content as string);
const geoJson = analysisResult.geoJson as FeatureCollection;
const image = analysisResult.image as string;

return {
id,
component: (
<>
{image && <ResolutionImage src={image} />}
{geoJson && (
<GeoJsonLayer id={id} data={geoJson} />
)}
Expand All @@ -811,9 +831,23 @@ export const getUIStateFromAIState = (aiState: AIState): UIState => {
toolOutput.type === 'MAP_QUERY_TRIGGER' &&
name === 'geospatialQueryTool'
) {
const mapUrl = toolOutput.mcp_response?.mapUrl;
const placeName = toolOutput.mcp_response?.location?.place_name;

return {
id,
component: <MapQueryHandler toolOutput={toolOutput} />,
component: (
<>
{mapUrl && (
<ResolutionImage
src={mapUrl}
className="mb-0"
alt={placeName ? `Map of ${placeName}` : 'Map Preview'}
/>
)}
<MapQueryHandler toolOutput={toolOutput} />
</>
),
isCollapsed: false
}
}
Expand Down
23 changes: 23 additions & 0 deletions app/layout.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,29 @@ export default function RootLayout({
}>) {
return (
<html lang="en" suppressHydrationWarning>
<head>
<script
dangerouslySetInnerHTML={{
__html: `
(function() {
const htmxEvents = [
'sseError', 'sseOpen', 'swapError', 'targetError', 'timeout',
'validation:validate', 'validation:failed', 'validation:halted',
'xhr:abort', 'xhr:loadend', 'xhr:loadstart'
];
htmxEvents.forEach(event => {
const funcName = 'func ' + event;
if (typeof window[funcName] === 'undefined') {
window[funcName] = function() {
console.warn('HTMX event handler "' + funcName + '" was called but not defined. Providing safety fallback.');
};
}
});
})();
`,
}}
/>
</head>
<body
className={cn(
'font-sans antialiased',
Expand Down
54 changes: 54 additions & 0 deletions app_actions_diff.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
--- app/actions.tsx
+++ app/actions.tsx
@@ -84,7 +84,7 @@
});
messages.push({ role: 'user', content });

- const summaryStream = createStreamableValue<string>('');
+ const summaryStream = createStreamableValue<string>('Analyzing map view...');
const groupeId = nanoid();

async function processResolutionSearch() {
@@ -143,15 +143,26 @@
return m
})

+ const currentMessages = aiState.get().messages;
+ const sanitizedHistory = currentMessages.map(m => {
+ if (m.role === 'user' && Array.isArray(m.content)) {
+ return {
+ ...m,
+ content: m.content.map(part =>
+ part.type === 'image' ? { ...part, image: 'IMAGE_PROCESSED' } : part
+ )
+ } as AIMessage
+ }
+ return m
+ });
+
const relatedQueries = await querySuggestor(uiStream, sanitizedMessages);
uiStream.append(
<Section title="Follow-up">
<FollowupPanel />
</Section>
);

await new Promise(resolve => setTimeout(resolve, 500));

aiState.done({
...aiState.get(),
- messages: [
- ...aiState.get().messages,
+ messages: [
+ ...sanitizedHistory,
{
id: groupeId,
role: 'assistant',
@@ -241,7 +252,7 @@
</Section>
);

- uiStream.append(answerSection);
+ uiStream.update(answerSection);

const relatedQueries = { items: [] };
Loading