Skip to content

revert(docs): undo loader refactor revert while keeping FDR fallback fixes #5711

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
coltondotio wants to merge 1 commit into from
Closed

revert(docs): undo loader refactor revert while keeping FDR fallback fixes #5711

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 98 additions & 13 deletions packages/commons/docs-loader/src/readonly-docs-loader.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,70 @@ const loadWithUrl = async (domainKey: string): Promise<DocsV2Read.LoadDocsForUrl
};
const loadDynamicIRWithUrl = uncachedLoadDynamicIRWithUrl;

type DocsBranchPathSegment = string | number;

type DocsBranchPath = DocsBranchPathSegment[];

type DocsBranchSelectorInput = {
branch: unknown;
path: DocsBranchPath;
branches: { path: DocsBranchPath; branch: unknown }[];
paths: DocsBranchPath[];
response: DocsV2Read.LoadDocsForUrlResponse;
};

type DocsBranchRequest<T> = {
paths?: DocsBranchPath[];
selector?: (input: DocsBranchSelectorInput) => T;
};

const loadDocsResponse = cache(async (domainKey: string) => loadWithUrl(domainKey));

function identityBranchSelector<T>({ branch }: DocsBranchSelectorInput): T {
return branch as T;
}

function getBranchAtPath(target: unknown, path: DocsBranchPath) {
return path.reduce<unknown>((current, segment) => {
if (current == null) {
return undefined;
}
if (typeof segment === "number") {
return (current as unknown[])[segment];
}
return (current as Record<string, unknown>)[segment];
}, target);
}

async function loadDocsBranch<T>(domainKey: string, request: DocsBranchRequest<T>): Promise<T> {
const hasExplicitPaths = request.paths != null && request.paths.length > 0;
const paths = request.paths ?? [[]];

// Load the full response
const response = await loadDocsResponse(domainKey);

const branches = paths.map((path) => ({
path,
branch: getBranchAtPath(response, path)
}));

const primaryBranch = branches[0]?.branch;
const primaryPath = branches[0]?.path ?? [];

if (hasExplicitPaths && primaryBranch === undefined) {
throw new Error(`Path not found in docs response: ${JSON.stringify(primaryPath)}`);
}

const selector = request.selector ?? identityBranchSelector<T>;
return selector({
branch: hasExplicitPaths ? primaryBranch : response,
path: primaryPath,
branches,
paths,
response
});
}

/*
* Domain key decoder/encoder functions
*/
Expand Down Expand Up @@ -306,16 +370,17 @@ const cachedGetEdgeFlags = cache(async (domainKey: string) => {

export const getMetadataFromResponse = async (
domainKey: string,
responsePromise: AsyncOrSync<DocsV2Read.LoadDocsForUrlResponse>
baseUrlPromise: AsyncOrSync<Pick<DocsV2Read.LoadDocsForUrlResponse, "baseUrl">>
): Promise<DocsMetadata> => {
assertDocsDomain(domainKey);
const [response, docsUrlMetadata] = await Promise.all([
responsePromise,
const [baseUrlResponse, docsUrlMetadata] = await Promise.all([
baseUrlPromise,
getDocsUrlMetadata(deriveDomainFromDomainKey(domainKey))
]);
const baseUrl = baseUrlResponse.baseUrl;

const isSelfHostedMode = isSelfHosted();
const fdrBasePath = response.baseUrl.basePath;
const fdrBasePath = baseUrl.basePath;
const nextBasePath = process.env.NEXT_PUBLIC_BASE_PATH;
const finalBasePath = isSelfHostedMode ? cleanBasePath(nextBasePath) : cleanBasePath(fdrBasePath);

Expand All @@ -324,11 +389,11 @@ export const getMetadataFromResponse = async (
fdrBasePath,
nextBasePath,
finalBasePath,
domain: response.baseUrl.domain
domain: baseUrl.domain
});

return {
domain: response.baseUrl.domain,
domain: baseUrl.domain,
// In self-hosted mode, use the Next.js basePath instead of the FDR basePath
// This allows the app to be served from a single basePath for all routes
basePath: finalBasePath,
Expand Down Expand Up @@ -366,7 +431,25 @@ export const getMetadata = (cacheConfig: Required<CacheConfig>) =>

const loadStart = Date.now();
console.debug(`[DocsLoader] getMetadata loadWithUrl start - domain: ${domainKey}`);
const metadata = await getMetadataFromResponse(domainKey, loadWithUrl(domainKey));

// Try to use getDocsFields API directly
const { domain } = decodeDocsLoaderDomainKey(domainKey);
const fieldsResponse = await runWithSpan(
"docs.getDocsFields.baseUrl",
() => getDocsFieldsFromServer(domain, [DocsDefinitionField.BaseUrl]),
{ "fern.docs.domain": domain }
);

let baseUrlResponse: Pick<DocsV2Read.LoadDocsForUrlResponse, "baseUrl">;
if (fieldsResponse?.baseUrl != null) {
baseUrlResponse = { baseUrl: fieldsResponse.baseUrl };
} else {
// Fallback to full load
const response = await loadDocsResponse(domainKey);
baseUrlResponse = { baseUrl: response.baseUrl };
}

const metadata = await getMetadataFromResponse(domainKey, baseUrlResponse);
const loadDuration = Date.now() - loadStart;
console.debug(`[DocsLoader] getMetadata loadWithUrl done in ${loadDuration}ms - domain: ${domainKey}`);
kvSet(domainKey, CACHE_KEY_METADATA, metadata, cacheConfig.kvTtl, cacheConfig.cacheKeySuffix);
Expand Down Expand Up @@ -490,7 +573,7 @@ const getFiles = (cacheConfig: Required<CacheConfig>) =>
}

// Fallback to full load
const response = await loadWithUrl(domainKey);
const response = await loadDocsResponse(domainKey);
const basePath = response.baseUrl?.basePath ?? "";
const files = transformFilesToFileData(response.definition.filesV2, basePath);

Expand Down Expand Up @@ -538,7 +621,7 @@ const getApi = async (domainKey: string, id: string) => {
apis = fieldsResponse.apis;
} else {
// Fallback to full load
const response = await loadWithUrl(domainKey);
const response = await loadDocsResponse(domainKey);
apisV2 = response.definition.apisV2;
apis = response.definition.apis;
}
Expand Down Expand Up @@ -804,7 +887,9 @@ const unsafe_getFullRoot = async (domainKey: string) => {

// Fallback to S3 if getDocsFields didn't return a root
console.debug(`[DocsLoader] unsafe_getFullRoot falling back to S3 for domain: ${domainKey}`);
const response = await loadWithUrl(domainKey);
const response = await loadDocsBranch(domainKey, {
selector: ({ response }) => response
});
const root = convertResponseToRootNode(response, await cachedGetEdgeFlags(domainKey));
if (root == null) {
console.error("Could not find root node for domainKey", domainKey);
Expand Down Expand Up @@ -1074,7 +1159,7 @@ const getPage = (cacheConfig: Required<CacheConfig>) =>
pages = fieldsResponse.pages as Record<PageId, DocsV1Read.PageContent> | undefined;
} else {
// Fallback to full load
const response = await loadWithUrl(domainKey);
const response = await loadDocsResponse(domainKey);
pages = response.definition.pages;
}

Expand Down Expand Up @@ -1130,7 +1215,7 @@ const getMdxBundlerFiles = (cacheConfig: Required<CacheConfig>) =>
files = fieldsResponse.jsFiles ?? {};
} else {
// Fallback to full load
const response = await loadWithUrl(domainKey);
const response = await loadDocsResponse(domainKey);
files = response.definition.jsFiles ?? {};
}

Expand Down Expand Up @@ -1479,7 +1564,7 @@ const getTypes = () =>
apis = fieldsResponse.apis;
} else {
// Fallback to full load
const response = await loadWithUrl(domainKey);
const response = await loadDocsResponse(domainKey);
apisV2 = response.definition.apisV2;
apis = response.definition.apis;
}
Expand Down
14 changes: 11 additions & 3 deletions packages/fern-docs/bundle/src/app/[host]/[domain]/api/fern-docs/revalidate/route.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,11 +95,13 @@ async function performRevalidation(params: {
controller.log(`revalidating:${domain}\n`);

const loadWithUrlPromise = loadWithUrl(domain);
// TODO: use the FDR API to get the baseUrl
const baseUrlPromise = loadWithUrlPromise.then((docs) => ({ baseUrl: docs.baseUrl }));

const [docs, edgeFlags, metadata, authConfig] = await Promise.all([
loadWithUrlPromise,
getEdgeFlags(domain),
getMetadataFromResponse(withoutStaging(domain), loadWithUrlPromise),
getMetadataFromResponse(withoutStaging(domain), baseUrlPromise),
getAuthEdgeConfig(domain)
]);

Expand Down Expand Up @@ -440,7 +442,10 @@ export async function GET(
};

try {
const metadata = await getMetadataFromResponse(withoutStaging(domain), loadWithUrl(domain));
const metadata = await getMetadataFromResponse(
withoutStaging(domain),
loadWithUrl(domain).then((docs) => ({ baseUrl: docs.baseUrl }))
);
const doReindex = !metadata.isPreview && req.nextUrl.searchParams.get("reindex") !== "false";
const doRegenerate = !metadata.isPreview && req.nextUrl.searchParams.get("regenerate") !== "false";
const useGetRequests = req.nextUrl.searchParams.get("useGetRequests") === "true";
Expand Down Expand Up @@ -492,7 +497,10 @@ export async function GET(
}
};

const metadata = await getMetadataFromResponse(withoutStaging(domain), loadWithUrl(domain));
const metadata = await getMetadataFromResponse(
withoutStaging(domain),
loadWithUrl(domain).then((docs) => ({ baseUrl: docs.baseUrl }))
);
const doReindex = !metadata.isPreview && req.nextUrl.searchParams.get("reindex") !== "false";
const doRegenerate = !metadata.isPreview && req.nextUrl.searchParams.get("regenerate") !== "false";
const useGetRequests = req.nextUrl.searchParams.get("useGetRequests") === "true";
Expand Down
35 changes: 28 additions & 7 deletions packages/fern-docs/bundle/src/components/PageHeader.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@ import { FernLink } from "@fern-docs/components/FernLink";
import { Separator } from "@fern-docs/components/Separator";
import { ChevronLeft } from "lucide-react";
import React from "react";
import { MdxContent } from "@/mdx/components/MdxContent";
import { MdxServerComponent } from "@/mdx/components/server-component";
import type { MdxSerializer } from "@/server/mdx-serializer";
import type { MdxSerializer, SerializedMdx } from "@/server/mdx-serializer";

import { PageActionsDropdown } from "./PageActionsDropdown";
import { PageFilters } from "./PageFilters";
Expand All @@ -19,10 +20,12 @@ export function PageHeader({
serialize,
breadcrumb,
title,
titleMdx,
titleHref,
action,
tags,
subtitle,
subtitleMdx,
children,
markdownPromise,
pageActionOptions,
Expand All @@ -36,9 +39,11 @@ export function PageHeader({
serialize: MdxSerializer;
breadcrumb: readonly FernNavigation.BreadcrumbItem[];
title: string;
titleMdx?: SerializedMdx;
titleHref?: string;
action?: React.ReactNode;
subtitle?: string;
subtitleMdx?: SerializedMdx;
tags?: React.ReactNode;
children?: React.ReactNode;
markdownPromise?: Promise<{ content: string; contentType: "markdown" | "mdx" } | undefined>;
Expand Down Expand Up @@ -68,20 +73,32 @@ export function PageHeader({
>
<ChevronLeft className="size-icon-md text-(color:--grayscale-a11)" />
<h1 className="fern-page-heading text-balance break-words">
<MdxServerComponent serialize={serialize} mdx={title} slug={slug} />
{titleMdx ? (
<MdxContent mdx={titleMdx} fallback={title} engine={titleMdx.engine} />
) : (
<MdxServerComponent serialize={serialize} mdx={title} slug={slug} />
)}
</h1>
</div>
</FernLink>
) : (
<div className="flex flex-row items-center" style={{ gap: "8px" }}>
{titleHref == null ? (
<h1 className="fern-page-heading text-balance break-words">
<MdxServerComponent serialize={serialize} mdx={title} slug={slug} />
{titleMdx ? (
<MdxContent mdx={titleMdx} fallback={title} engine={titleMdx.engine} />
) : (
<MdxServerComponent serialize={serialize} mdx={title} slug={slug} />
)}
</h1>
) : (
<FernLink href={titleHref} scroll={true}>
<h1 className="fern-page-heading text-balance break-words">
<MdxServerComponent serialize={serialize} mdx={title} slug={slug} />
{titleMdx ? (
<MdxContent mdx={titleMdx} fallback={title} engine={titleMdx.engine} />
) : (
<MdxServerComponent serialize={serialize} mdx={title} slug={slug} />
)}
</h1>
</FernLink>
)}
Expand Down Expand Up @@ -109,9 +126,13 @@ export function PageHeader({

{subtitle && (
<div className="prose-p:text-(color:--grayscale-a11) mt-2 break-words leading-7">
<React.Suspense fallback={subtitle}>
<MdxServerComponent serialize={serialize} mdx={subtitle} slug={slug} />
</React.Suspense>
{subtitleMdx ? (
<MdxContent mdx={subtitleMdx} fallback={subtitle} engine={subtitleMdx.engine} />
) : (
<React.Suspense fallback={subtitle}>
<MdxServerComponent serialize={serialize} mdx={subtitle} slug={slug} />
</React.Suspense>
)}
</div>
)}

Expand Down
40 changes: 34 additions & 6 deletions packages/fern-docs/bundle/src/components/changelog/ChangelogPage.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,14 @@ import { FernNavigation } from "@fern-api/fdr-sdk";
import { isNonNullish } from "@fern-api/ui-core-utils";
import { FernLink } from "@fern-docs/components/FernLink";
import { t } from "@fern-docs/i18n";
import { makeToc, type TableOfContentsItem, toTree } from "@fern-docs/mdx";
import {
getFrontmatter,
makeToc,
sanitizeBreaks,
sanitizeMdxExpression,
type TableOfContentsItem,
toTree
} from "@fern-docs/mdx";
import { compact } from "es-toolkit/compat";
import { notFound } from "next/navigation";
import { PageHeader } from "@/components/PageHeader";
Expand Down Expand Up @@ -162,16 +169,37 @@ export async function ChangelogPageEntry({
node: FernNavigation.ChangelogEntryNode;
}) {
const page = await loader.getPage(node.pageId);
const mdx = await serialize(page.markdown, {
filename: page.filename,
slug: node.slug
});

const title = await serialize(mdx?.frontmatter?.title, {
// Extract frontmatter title without full MDX bundling (much faster)
const sanitized = sanitizeMdxExpression(sanitizeBreaks(page.markdown))[0];
const { data: frontmatter } = getFrontmatter(sanitized);
const frontmatterTitle = frontmatter?.title;

// Start body serialization immediately
const mdxPromise = serialize(page.markdown, {
filename: page.filename,
slug: node.slug
});

// If frontmatter title exists, serialize it in parallel (fast path)
// Otherwise, wait for full MDX to run remarkExtractTitle, then serialize that title (fallback path)
const titlePromise = frontmatterTitle
? serialize(frontmatterTitle, {
filename: page.filename,
slug: node.slug
})
: mdxPromise.then((mdx) => {
const extractedTitle = mdx?.frontmatter?.title;
return extractedTitle
? serialize(extractedTitle, {
filename: page.filename,
slug: node.slug
})
: undefined;
});

const [mdx, title] = await Promise.all([mdxPromise, titlePromise]);

return (
<Markdown
mdx={mdx}
Expand Down
Loading
Loading