Fix/browsify scraping

app/api/parse-url/route.ts

@@ -16,8 +16,11 @@ const handler = async (request: Request) => {
 		);
 	}
 
+	// Get client's cookies from the incoming request
+	const clientCookies = request.headers.get('cookie') || '';
+
 	const journeyDetails = extractJourneyDetails(
-		await getResolvedUrlBrowserless(url)
+		await getResolvedUrlBrowserless(url, clientCookies)
 	);
 
 	if ("error" in journeyDetails) {
@@ -151,7 +154,7 @@ function displayJourneyInfo(journeyDetails: ExtractedData) {
 	console.log(formatInfo);
 }
 
-async function getResolvedUrlBrowserless(url: string) {
+async function getResolvedUrlBrowserless(url: string, clientCookies: string = '') {
 	const vbid = new URL(url).searchParams.get("vbid");
 
 	if (!vbid) {
@@ -163,8 +166,29 @@ async function getResolvedUrlBrowserless(url: string) {
 		schema: vbidSchema,
 	});
 
-	const cookies = vbidRequest.response.headers.getSetCookie();
-	const { data } = await parseHinfahrtReconWithAPI(vbidRequest.data, cookies);
+	// Get cookies from the vbidRequest response
+	const responseCookies = vbidRequest.response.headers.getSetCookie()
+		.map(cookie => cookie.split(';')[0]) // Take only the name=value part
+		.filter(cookie => cookie.trim());
+
+	// Properly format cookies with semicolons and spaces between them
+	const allCookiesSet = new Set<string>();
+
+	// Add client cookies (split and clean if needed)
+	if (clientCookies) {
+		clientCookies.split(';').forEach(cookie => {
+			const trimmed = cookie.trim();
+			if (trimmed) allCookiesSet.add(trimmed);
+		});
+	}
+
+	// Add response cookies
+	responseCookies.forEach(cookie => allCookiesSet.add(cookie));
+
+	const allCookies = Array.from(allCookiesSet);
+	console.log("Final cookie header:", allCookies.join('; '));
+
+	const { data } = await parseHinfahrtReconWithAPI(vbidRequest.data, allCookies)
 
 	const newUrl = new URL("https://www.bahn.de/buchung/fahrplan/suche");
 

next-env.d.ts

@@ -1,5 +1,6 @@
 /// <reference types="next" />
 /// <reference types="next/image-types/global" />
+/// <reference path="./.next/types/routes.d.ts" />
 
 // NOTE: This file should not be edited
 // see https://nextjs.org/docs/app/api-reference/config/typescript for more information.

utils/fetchAndValidateJson.ts

@@ -32,6 +32,24 @@ export const fetchAndValidateJson = async <
 	const response = await fetch(url, init);
 
 	if (!response.ok) {
+		if (response.status === 403) {
+			console.error(`403 Forbidden error accessing ${url}`);
+			console.error(`Request headers:`, init.headers);
+
+			// Try to get response body for additional error details
+			let responseText = '';
+			try {
+				responseText = await response.text();
+				console.error(`Response body: ${responseText.substring(0, 500)}`);
+			} catch (e) {
+				console.error(`Could not read response body: ${e}`);
+			}
+
+			throw new Error(
+				`Access forbidden (403) to ${url} - API may have blocked the request due to missing authentication or anti-scraping measures`
+			);
+		}
+
 		throw new Error(
 			`Failed to fetch ${url}: ${response.status} ${response.statusText}`
 		);

utils/parseHinfahrtRecon.ts

@@ -116,7 +116,15 @@ export const parseHinfahrtReconWithAPI = async (
 		method: "POST",
 		headers: {
 			"Content-Type": "application/json",
-			"Cookie": cookies.join(" "),
+			"Cookie": cookies.join("; "),
+			"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+			"Accept": "application/json, text/plain, */*",
+			"Accept-Language": "en-US,en;q=0.9,de;q=0.8",
+			"Origin": "https://www.bahn.de",
+			"Referer": "https://www.bahn.de/buchung/fahrplan/suche",
+			"Sec-Fetch-Dest": "empty",
+			"Sec-Fetch-Mode": "cors",
+			"Sec-Fetch-Site": "same-origin",
 		},
 		body: {
 			klasse: "KLASSE_2",