Skip to content

Major update for DevDocs #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Feb 11, 2025
Merged

Major update for DevDocs #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1cfcff5
Major update for DevDocs web scraper
Shubham-Khichi Feb 11, 2025
017da3e
Update page.tsx
Shubham-Khichi Feb 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions app/api/discover/route.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { discoverSubdomains } from '@/lib/crawl-service'

export async function POST(request: Request) {
try {
const { url } = await request.json()
const { url, depth = 3 } = await request.json()

if (!url) {
return NextResponse.json(
Expand All @@ -12,8 +12,11 @@ export async function POST(request: Request) {
)
}

console.log('Making discover request for URL:', url)
const pages = await discoverSubdomains(url)
// Validate depth is between 1 and 5
const validatedDepth = Math.min(5, Math.max(1, parseInt(String(depth)) || 3))

console.log('Making discover request for URL:', url, 'with depth:', validatedDepth)
const pages = await discoverSubdomains({ url, depth: validatedDepth })
console.log('Received pages from backend:', pages)

// Even if we get an empty array, we should still return it with a 200 status
Expand Down
125 changes: 54 additions & 71 deletions app/page.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ export default function Home() {
})
const { toast } = useToast()

const handleSubmit = async (submittedUrl: string) => {
const handleSubmit = async (submittedUrl: string, depth: number) => {
if (!validateUrl(submittedUrl)) {
toast({
title: "Invalid URL",
Expand All @@ -42,8 +42,8 @@ export default function Home() {
setDiscoveredPages([])

try {
console.log('Discovering pages for:', submittedUrl)
const pages = await discoverSubdomains(submittedUrl)
console.log('Discovering pages for:', submittedUrl, 'with depth:', depth)
const pages = await discoverSubdomains({ url: submittedUrl, depth })
console.log('Discovered pages:', pages)

setDiscoveredPages(pages)
Expand All @@ -54,7 +54,7 @@ export default function Home() {

toast({
title: "Pages Discovered",
description: `Found ${pages.length} related pages`
description: `Found ${pages.length} related pages at depth ${depth}`
})
} catch (error) {
console.error('Error discovering pages:', error)
Expand All @@ -71,7 +71,6 @@ export default function Home() {
const handleCrawlSelected = async (selectedUrls: string[]) => {
setIsCrawling(true)
try {
// Filter pages to only selected ones
const selectedPages = discoveredPages.filter(page => selectedUrls.includes(page.url))
console.log('Starting crawl for selected pages:', selectedPages)

Expand All @@ -83,7 +82,6 @@ export default function Home() {
}

try {
// Save markdown to storage
await saveMarkdown(url, result.markdown)
console.log('Saved content for:', url)

Expand All @@ -107,7 +105,6 @@ export default function Home() {
})
}

// Update only selected page statuses to crawled
setDiscoveredPages(pages =>
pages.map(page => ({
...page,
Expand Down Expand Up @@ -137,7 +134,6 @@ export default function Home() {

return (
<main className="min-h-screen bg-gradient-to-b from-gray-900 via-gray-800 to-gray-900">
{/* Hero Header */}
<header className="w-full py-12 bg-gradient-to-r from-gray-900/50 to-gray-800/50 backdrop-blur-sm border-b border-gray-700">
<div className="container mx-auto px-4 text-center">
<h1 className="text-5xl font-bold bg-clip-text text-transparent bg-gradient-to-r from-blue-400 to-purple-500 mb-4">
Expand All @@ -150,78 +146,65 @@ export default function Home() {
</div>
</header>

{/* Main Content */}
<div className="container mx-auto px-4 py-12">
<div className="grid grid-cols-1 lg:grid-cols-2 gap-8">
{/* Left Column - Input and Processing */}
<div className="space-y-8">
<section className="bg-gray-800/50 backdrop-blur-lg rounded-2xl p-6 border border-gray-700 shadow-xl">
<h2 className="text-2xl font-semibold mb-6 text-blue-400">Start Exploration</h2>
<UrlInput onSubmit={handleSubmit} />
</section>

<section className="bg-gray-800/50 backdrop-blur-lg rounded-2xl p-6 border border-gray-700 shadow-xl">
<h2 className="text-2xl font-semibold mb-6 text-purple-400">Processing Status</h2>
<ProcessingBlock
isProcessing={isProcessing || isCrawling}
stats={stats}
/>
</section>

<section className="bg-gray-800/50 backdrop-blur-lg rounded-2xl p-6 border border-gray-700 shadow-xl">
<h2 className="text-2xl font-semibold mb-6 text-white">MCP Server</h2>
<MCPServerStatus />
</section>

<section className="bg-gray-800/50 backdrop-blur-lg rounded-2xl p-6 border border-gray-700 shadow-xl">
<h2 className="text-2xl font-semibold mb-6 text-blue-400">Stored Files</h2>
<StoredFiles />
</section>
<div className="container mx-auto px-4 py-8 space-y-6">
<div className="bg-gray-800/50 backdrop-blur-lg rounded-2xl p-6 border border-gray-700 shadow-xl">
<h2 className="text-2xl font-semibold mb-4 text-purple-400">Processing Status</h2>
<div className="flex gap-4">
<ProcessingBlock
isProcessing={isProcessing || isCrawling}
stats={stats}
/>
</div>
</div>

{/* Right Column - Results */}
<div className="space-y-8">
<section className="bg-gray-800/50 backdrop-blur-lg rounded-2xl p-6 border border-gray-700 shadow-xl">
<h2 className="text-2xl font-semibold mb-6 text-green-400">Discovered Pages</h2>
<SubdomainList
subdomains={discoveredPages}
onCrawlSelected={handleCrawlSelected}
isProcessing={isCrawling}
/>
</section>
<div className="bg-gray-800/50 backdrop-blur-lg rounded-2xl p-6 border border-gray-700 shadow-xl">
<h2 className="text-2xl font-semibold mb-4 text-blue-400">Start Exploration</h2>
<UrlInput onSubmit={handleSubmit} />
</div>

<section className="bg-gray-800/50 backdrop-blur-lg rounded-2xl p-6 border border-gray-700 shadow-xl">
<h2 className="text-2xl font-semibold mb-6 text-yellow-400">Extracted Content</h2>
<MarkdownOutput
markdown={markdown}
isVisible={markdown !== ''}
/>
</section>
</div>
<div className="bg-gray-800/50 backdrop-blur-lg rounded-2xl p-6 border border-gray-700 shadow-xl">
<SubdomainList
subdomains={discoveredPages}
onCrawlSelected={handleCrawlSelected}
isProcessing={isCrawling}
/>
</div>

{/* Features Grid */}
<div className="grid grid-cols-1 md:grid-cols-3 gap-8 mt-16">
<div className="bg-gray-800/30 backdrop-blur-sm rounded-xl p-6 border border-gray-700 shadow-lg">
<h3 className="text-xl font-semibold text-blue-400 mb-3">Advanced Extraction</h3>
<p className="text-gray-300">
Smart content targeting with specific selectors and quality filtering
</p>
</div>
<div className="bg-gray-800/30 backdrop-blur-sm rounded-xl p-6 border border-gray-700 shadow-lg">
<h3 className="text-xl font-semibold text-purple-400 mb-3">Bypass Restrictions</h3>
<p className="text-gray-300">
Handle SSL certificates and security policies with ease
</p>
<div className="bg-gray-800/50 backdrop-blur-lg rounded-2xl p-6 border border-gray-700 shadow-xl">
<h2 className="text-2xl font-semibold mb-4 text-yellow-400">Extracted Content</h2>
<MarkdownOutput
markdown={markdown}
isVisible={markdown !== ''}
/>
</div>

<div className="grid grid-cols-2 gap-6">
<div className="bg-gray-800/50 backdrop-blur-lg rounded-2xl p-6 border border-gray-700 shadow-xl">
<h2 className="text-2xl font-semibold mb-4 text-white">MCP Server</h2>
<MCPServerStatus />
</div>
<div className="bg-gray-800/30 backdrop-blur-sm rounded-xl p-6 border border-gray-700 shadow-lg">
<h3 className="text-xl font-semibold text-green-400 mb-3">Error Recovery</h3>
<p className="text-gray-300">
Automatic retries and enhanced session management
</p>

<div className="bg-gray-800/50 backdrop-blur-lg rounded-2xl p-6 border border-gray-700 shadow-xl">
<h2 className="text-2xl font-semibold mb-4 text-blue-400">Stored Files</h2>
<StoredFiles />
</div>
</div>
</div>

<footer className="py-8 text-center text-gray-400">
<p className="flex items-center justify-center gap-2">
Made with <span className="text-red-500">❤️</span> by{' '}
<a
href="https://www.cyberagi.ai/"
target="_blank"
rel="noopener noreferrer"
className="text-blue-400 hover:text-blue-300 transition-colors"
>
CyberAGI Inc
</a>{' '}
in <span>🇺🇸</span>
</p>
</footer>
</main>
)
}
Loading