event classification

README.md

@@ -8,13 +8,13 @@ A real-time global situational awareness platform that plots security events, ge
 
 ## Features
 
-                                                                                    ### Core Features
+### Core Features
 
-                                                                                    - **Real-Time Event Mapping** - Plot breaking news events (conflicts, protests, natural disasters) on a world map with color-coded threat levels
-                                                                                    - **Interactive Mapbox Map** - Dark-themed map with clustering, heatmap visualization, and smooth navigation
-                                                                                    - **Event Feed** - Real-time filterable feed of global events with category and threat level filters
-                                                                                    - **Entity Search** - Research organizations, people, countries, and groups using Valyu's intelligence APIs
-                                                                                    - **Alert System** - Configure keyword and region-based alerts with real-time notifications
+- **Real-Time Event Mapping** - Plot breaking news events (conflicts, protests, natural disasters) on a world map with color-coded threat levels
+- **Interactive Mapbox Map** - Dark-themed map with clustering, heatmap visualization, and smooth navigation
+- **Event Feed** - Real-time filterable feed of global events with category and threat level filters
+- **Entity Search** - Research organizations, people, countries, and groups using Valyu's intelligence APIs
+- **Alert System** - Configure keyword and region-based alerts with real-time notifications
 
 ### Country Intelligence
 
@@ -192,7 +192,7 @@ This app uses [Valyu](https://valyu.ai) for intelligence data:
 - **Answer API** - Synthesizing conflict intelligence and military base data
 - **Deep Research** - Comprehensive entity analysis
 
-All Valyu queries exclude Wikipedia to ensure higher-quality source citations.
+Wikipedia is excluded from search results.
 
 ## Authentication
 
@@ -203,7 +203,7 @@ Global Threat Map supports two app modes controlled by the `NEXT_PUBLIC_APP_MODE
 | Mode | Description |
 |------|-------------|
 | `self-hosted` | Default mode. No authentication required. All features are freely accessible. |
-| `valyu` | OAuth mode. Users sign in with Valyu to access premium features. |
+| `valyu` | OAuth mode. Users sign in with Valyu to access additional features. |
 
 ### Self-Hosted Mode (Default)
 
@@ -245,7 +245,7 @@ When running in valyu mode, certain features require authentication:
 | Entity search | ❌ Blocked | ✅ Unlimited |
 | Military bases | ✅ Free | ✅ Free |
 
-After users exhaust their free usage, a sign-in modal prompts them to authenticate with Valyu. New Valyu accounts receive **$10 in free credits**.
+After users exhaust their free usage, a sign-in modal prompts them to authenticate.
 
 ### OAuth Flow
 

app/api/events/route.ts

@@ -1,8 +1,9 @@
 import { NextResponse } from "next/server";
 import { searchEvents } from "@/lib/valyu";
 import { isSelfHostedMode } from "@/lib/app-mode";
-import { geocodeLocationsFromText } from "@/lib/geocoding";
-import { createThreatEvent } from "@/lib/event-classifier";
+import { classifyEvent, isAIClassificationEnabled } from "@/lib/ai-classifier";
+import { generateEventId } from "@/lib/utils";
+import { extractKeywords, extractEntities } from "@/lib/event-classifier";
 import type { ThreatEvent } from "@/types";
 
 export const dynamic = "force-dynamic";
@@ -17,34 +18,49 @@ const THREAT_QUERIES = [
   "diplomatic summit sanctions",
 ];
 
+// Clean boilerplate from content
+function cleanContent(text: string): string {
+  return text
+    .replace(/skip to (?:main |primary )?content/gi, "")
+    .replace(/keyboard shortcuts?/gi, "")
+    .replace(/\n{3,}/g, "\n\n")
+    .replace(/\s{2,}/g, " ")
+    .trim();
+}
+
 async function processSearchResults(
   results: Array<{ title: string; url: string; content: string; publishedDate?: string; source?: string }>
 ): Promise<ThreatEvent[]> {
   const eventsWithLocations = await Promise.all(
     results.map(async (result) => {
-      const locations = await geocodeLocationsFromText(
-        `${result.title} ${result.content}`,
-        result.title
-      );
+      const cleanedTitle = cleanContent(result.title);
+      const cleanedContent = cleanContent(result.content);
+      const fullText = `${cleanedTitle} ${cleanedContent}`;
 
-      const location = locations[0] || {
-        latitude: 0,
-        longitude: 0,
-        placeName: "Unknown",
-      };
+      // Use AI classification (falls back to keywords if OpenAI not available)
+      const classification = await classifyEvent(cleanedTitle, cleanedContent);
 
-      if (location.latitude === 0 && location.longitude === 0) {
+      // Skip events without valid locations
+      if (!classification.location) {
         return null;
       }
 
-      return createThreatEvent(
-        result.title,
-        result.content,
-        location,
-        result.source || "web",
-        result.url,
-        result.publishedDate
-      );
+      const event: ThreatEvent = {
+        id: generateEventId(),
+        title: cleanedTitle,
+        summary: cleanedContent.slice(0, 500),
+        category: classification.category,
+        threatLevel: classification.threatLevel,
+        location: classification.location,
+        timestamp: result.publishedDate || new Date().toISOString(),
+        source: result.source || "web",
+        sourceUrl: result.url,
+        entities: extractEntities(fullText),
+        keywords: extractKeywords(fullText),
+        rawContent: cleanedContent,
+      };
+
+      return event;
     })
   );
 

components/auth/sign-in-modal.tsx

@@ -75,14 +75,8 @@ export function SignInModal({ open, onOpenChange }: SignInModalProps) {
         <DialogTitle className="text-center text-xl">Sign in</DialogTitle>
       </DialogHeader>
       <DialogContent className="space-y-6">
-        <p className="text-center text-muted-foreground">
-          Valyu is the information backbone of Global Threat Map, giving our app
-          access to real-time data across web, academic, and proprietary
-          sources.
-        </p>
-
         <p className="text-center text-sm text-muted-foreground">
-          Free to use.
+          Sign in to access all features.
         </p>
 
         {/* Error Message */}

components/map/country-conflicts-modal.tsx

@@ -13,13 +13,12 @@ import { Skeleton } from "@/components/ui/skeleton";
 import {
   Swords,
   ExternalLink,
-  Globe,
   History,
   AlertTriangle,
-  MessageSquare,
   Database,
   RotateCw,
 } from "lucide-react";
+import { Favicon } from "@/components/ui/favicon";
 import { cn } from "@/lib/utils";
 
 interface CountryConflictsModalProps {
@@ -350,9 +349,9 @@ export function CountryConflictsModal({
                           href={source.url}
                           target="_blank"
                           rel="noopener noreferrer"
-                          className="flex items-start gap-2 rounded-lg border border-border bg-card p-3 text-sm transition-colors hover:bg-muted/50"
+                          className="flex items-start gap-3 rounded-lg border border-border bg-card p-3 text-sm transition-colors hover:bg-muted/50"
                         >
-                          <Globe className="mt-0.5 h-4 w-4 shrink-0 text-muted-foreground" />
+                          <Favicon url={source.url} size={20} className="mt-0.5" />
                           <div className="flex-1 min-w-0">
                             <span className="line-clamp-2 text-foreground">
                               {source.title}

components/map/event-popup.tsx

@@ -1,18 +1,21 @@
 "use client";
 
+import { useState } from "react";
 import type { ThreatEvent } from "@/types";
 import { Badge } from "@/components/ui/badge";
-import { Markdown } from "@/components/ui/markdown";
 import { formatRelativeTime } from "@/lib/utils";
-import { ExternalLink, MapPin } from "lucide-react";
+import { ExternalLink, MapPin, ArrowDownRight, ChevronUp } from "lucide-react";
+import { Streamdown } from "streamdown";
 
 interface EventPopupProps {
   event: ThreatEvent;
 }
 
 export function EventPopup({ event }: EventPopupProps) {
+  const [isExpanded, setIsExpanded] = useState(false);
+
   return (
-    <div className="min-w-[250px] max-w-[300px] p-2">
+    <div className={`min-w-[250px] p-2 ${isExpanded ? "max-w-[500px]" : "max-w-[300px]"}`}>
       <div className="mb-2 flex items-start justify-between gap-2">
         <h3 className="text-sm font-semibold text-foreground line-clamp-2">
           <a
@@ -32,9 +35,17 @@ export function EventPopup({ event }: EventPopupProps) {
         </Badge>
       </div>
 
-      <div className="mb-2 text-xs text-muted-foreground line-clamp-3">
-        <Markdown content={event.summary} />
-      </div>
+      {!isExpanded ? (
+        <div className="mb-2 text-xs text-muted-foreground line-clamp-3">
+          {event.summary}
+        </div>
+      ) : (
+        <div className="mb-2 max-h-[400px] overflow-y-auto rounded-md bg-muted/30 p-3">
+          <div className="prose prose-sm prose-invert max-w-none text-xs">
+            <Streamdown>{event.rawContent || event.summary}</Streamdown>
+          </div>
+        </div>
+      )}
 
       <div className="flex items-center gap-2 text-xs text-muted-foreground">
         <MapPin className="h-3 w-3" />
@@ -47,16 +58,36 @@ export function EventPopup({ event }: EventPopupProps) {
         <span className="text-muted-foreground">
           {formatRelativeTime(event.timestamp)}
         </span>
-        {event.sourceUrl && (
-          <a
-            href={event.sourceUrl}
-            target="_blank"
-            rel="noopener noreferrer"
-            className="flex items-center gap-1 text-primary hover:underline"
-          >
-            Source <ExternalLink className="h-3 w-3" />
-          </a>
-        )}
+        <div className="flex items-center gap-2">
+          {event.rawContent && (
+            <button
+              onClick={() => setIsExpanded(!isExpanded)}
+              className="flex items-center gap-1 text-muted-foreground hover:text-foreground transition-colors"
+            >
+              {isExpanded ? (
+                <>
+                  <ChevronUp className="h-3 w-3" />
+                  Collapse
+                </>
+              ) : (
+                <>
+                  <ArrowDownRight className="h-3 w-3" />
+                  Expand
+                </>
+              )}
+            </button>
+          )}
+          {event.sourceUrl && (
+            <a
+              href={event.sourceUrl}
+              target="_blank"
+              rel="noopener noreferrer"
+              className="flex items-center gap-1 text-primary hover:underline"
+            >
+              Source <ExternalLink className="h-3 w-3" />
+            </a>
+          )}
+        </div>
       </div>
 
       <div className="mt-2 flex flex-wrap gap-1">

components/search/entity-search.tsx

@@ -15,11 +15,11 @@ import {
   Globe,
   Users,
   FileText,
-  ExternalLink,
   MapPin,
   Navigation,
   Lock,
 } from "lucide-react";
+import { Favicon } from "@/components/ui/favicon";
 import { useMapStore } from "@/stores/map-store";
 import { useAuthStore } from "@/stores/auth-store";
 import { Markdown } from "@/components/ui/markdown";
@@ -252,10 +252,10 @@ export function EntitySearch() {
                           href={source.url}
                           target="_blank"
                           rel="noopener noreferrer"
-                          className="flex items-center gap-2 text-sm text-primary hover:underline"
+                          className="flex items-center gap-2 text-sm text-foreground hover:text-primary transition-colors"
                         >
-                          <ExternalLink className="h-3 w-3" />
-                          {source.title}
+                          <Favicon url={source.url} size={16} />
+                          <span className="truncate">{source.title}</span>
                         </a>
                       ))}
                     </div>

components/ui/favicon.tsx

@@ -0,0 +1,40 @@
+"use client";
+
+import { useState } from "react";
+import { Globe } from "lucide-react";
+import { cn } from "@/lib/utils";
+
+interface FaviconProps {
+  url: string;
+  size?: number;
+  className?: string;
+}
+
+export function Favicon({ url, size = 16, className }: FaviconProps) {
+  const [error, setError] = useState(false);
+
+  let hostname: string;
+  try {
+    hostname = new URL(url).hostname;
+  } catch {
+    return <Globe className={cn("text-muted-foreground", className)} style={{ width: size, height: size }} />;
+  }
+
+  if (error) {
+    return <Globe className={cn("text-muted-foreground", className)} style={{ width: size, height: size }} />;
+  }
+
+  // Use Google's favicon service for reliable favicon fetching
+  const faviconUrl = `https://www.google.com/s2/favicons?domain=${hostname}&sz=${size * 2}`;
+
+  return (
+    <img
+      src={faviconUrl}
+      alt=""
+      width={size}
+      height={size}
+      className={cn("shrink-0 rounded-sm", className)}
+      onError={() => setError(true)}
+    />
+  );
+}