Fix issues with revalidateTag/revalidatePath (#470)

* fix clearing data cache with revalidatePath

* fix revalidatePath for fetch with no tag

* fix revalidateTag for next 15

* e2e test

* fix incorrectly writing to tag cache on set

* added comment

* Create gold-paws-laugh.md

Author: conico974

.changeset/gold-paws-laugh.md

@@ -0,0 +1,5 @@
+---
+"open-next": patch
+---
+
+Fix issues with revalidateTag/revalidatePath

examples/app-router/app/api/revalidate-path/route.ts

@@ -0,0 +1,9 @@
+import { revalidatePath } from "next/cache";
+
+export const dynamic = "force-dynamic";
+
+export async function GET() {
+  revalidatePath("/revalidate-path");
+
+  return new Response("ok");
+}

examples/app-router/app/api/revalidate-tag/route.ts

@@ -1,5 +1,7 @@
 import { revalidateTag } from "next/cache";
 
+export const dynamic = "force-dynamic";
+
 export async function GET() {
   revalidateTag("revalidate");
 

examples/app-router/app/revalidate-path/page.tsx

@@ -0,0 +1,26 @@
+export default async function Page() {
+  const timeInParis = await fetch(
+    "http://worldtimeapi.org/api/timezone/Europe/Paris",
+    {
+      next: {
+        tags: ["path"],
+      },
+    },
+  );
+  // This one doesn't have a tag
+  const timeInLondon = await fetch(
+    "http://worldtimeapi.org/api/timezone/Europe/London",
+  );
+  const timeInParisJson = await timeInParis.json();
+  const parisTime = timeInParisJson.datetime;
+  const timeInLondonJson = await timeInLondon.json();
+  const londonTime = timeInLondonJson.datetime;
+  return (
+    <div>
+      <h1>Time in Paris</h1>
+      <p>Paris: {parisTime}</p>
+      <h1>Time in London</h1>
+      <p>London: {londonTime}</p>
+    </div>
+  );
+}

examples/app-router/middleware.ts

@@ -43,7 +43,10 @@ export function middleware(request: NextRequest) {
   }
 
   // It is so that cloudfront doesn't cache the response
-  if (path.startsWith("/revalidate-tag")) {
+  if (
+    path.startsWith("/revalidate-tag") ||
+    path.startsWith("/revalidate-path")
+  ) {
     responseHeaders.set(
       "cache-control",
       "private, no-cache, no-store, max-age=0, must-revalidate",

packages/open-next/src/adapters/cache.ts

@@ -104,7 +104,12 @@ export default class S3Cache {
     // fetchCache is for next 13.5 and above, kindHint is for next 14 and above and boolean is for earlier versions
     options?:
       | boolean
-      | { fetchCache?: boolean; kindHint?: "app" | "pages" | "fetch" },
+      | {
+          fetchCache?: boolean;
+          kindHint?: "app" | "pages" | "fetch";
+          tags?: string[];
+          softTags?: string[];
+        },
   ) {
     if (globalThis.disableIncrementalCache) {
       return null;
@@ -115,13 +120,16 @@ export default class S3Cache {
           ? options.kindHint === "fetch"
           : options.fetchCache
         : options;
+
+    const softTags = typeof options === "object" ? options.softTags : [];
+    const tags = typeof options === "object" ? options.tags : [];
     return isFetchCache
-      ? this.getFetchCache(key)
+      ? this.getFetchCache(key, softTags, tags)
       : this.getIncrementalCache(key);
   }
 
-  async getFetchCache(key: string) {
-    debug("get fetch cache", { key });
+  async getFetchCache(key: string, softTags?: string[], tags?: string[]) {
+    debug("get fetch cache", { key, softTags, tags });
     try {
       const { value, lastModified } = await globalThis.incrementalCache.get(
         key,
@@ -139,6 +147,31 @@ export default class S3Cache {
 
       if (value === undefined) return null;
 
+      // For cases where we don't have tags, we need to ensure that we insert at least an entry
+      // for this specific paths, otherwise we might not be able to invalidate it
+      if ((tags ?? []).length === 0) {
+        // First we check if we have any tags for the given key
+        const storedTags = await globalThis.tagCache.getByPath(key);
+        if (storedTags.length === 0) {
+          // Then we need to find the path for the given key
+          const path = softTags?.find(
+            (tag) =>
+              tag.startsWith("_N_T_/") &&
+              !tag.endsWith("layout") &&
+              !tag.endsWith("page"),
+          );
+          if (path) {
+            // And write the path with the tag
+            await globalThis.tagCache.writeTags([
+              {
+                path: key,
+                tag: path,
+              },
+            ]);
+          }
+        }
+      }
+
       return {
         lastModified: _lastModified,
         value: value,
@@ -317,22 +350,45 @@ export default class S3Cache {
     }
   }
 
-  public async revalidateTag(tag: string) {
+  public async revalidateTag(tags: string | string[]) {
     if (globalThis.disableDynamoDBCache || globalThis.disableIncrementalCache) {
       return;
     }
     try {
-      debug("revalidateTag", tag);
-      // Find all keys with the given tag
-      const paths = await globalThis.tagCache.getByTag(tag);
-      debug("Items", paths);
-      // Update all keys with the given tag with revalidatedAt set to now
-      await globalThis.tagCache.writeTags(
-        paths?.map((path) => ({
-          path: path,
-          tag: tag,
-        })) ?? [],
-      );
+      const _tags = Array.isArray(tags) ? tags : [tags];
+      for (const tag of _tags) {
+        debug("revalidateTag", tag);
+        // Find all keys with the given tag
+        const paths = await globalThis.tagCache.getByTag(tag);
+        debug("Items", paths);
+        const toInsert = paths.map((path) => ({
+          path,
+          tag,
+        }));
+
+        // If the tag is a soft tag, we should also revalidate the hard tags
+        if (tag.startsWith("_N_T_/")) {
+          for (const path of paths) {
+            // We need to find all hard tags for a given path
+            const _tags = await globalThis.tagCache.getByPath(path);
+            const hardTags = _tags.filter((t) => !t.startsWith("_N_T_/"));
+            // For every hard tag, we need to find all paths and revalidate them
+            for (const hardTag of hardTags) {
+              const _paths = await globalThis.tagCache.getByTag(hardTag);
+              debug({ hardTag, _paths });
+              toInsert.push(
+                ..._paths.map((path) => ({
+                  path,
+                  tag: hardTag,
+                })),
+              );
+            }
+          }
+        }
+
+        // Update all keys with the given tag with revalidatedAt set to now
+        await globalThis.tagCache.writeTags(toInsert);
+      }
     } catch (e) {
       error("Failed to revalidate tag", e);
     }

packages/open-next/src/cache/tag/dynamodb-lite.ts

@@ -70,7 +70,8 @@ const tagCache: TagCache = {
 
       const tags = Items?.map((item: any) => item.tag.S ?? "") ?? [];
       debug("tags for path", path, tags);
-      return tags;
+      // We need to remove the buildId from the path
+      return tags.map((tag: string) => tag.replace(`${NEXT_BUILD_ID}/`, ""));
     } catch (e) {
       error("Failed to get tags by path", e);
       return [];

packages/open-next/src/cache/tag/dynamodb.ts

@@ -59,7 +59,8 @@ const tagCache: TagCache = {
       );
       const tags = result.Items?.map((item) => item.tag.S ?? "") ?? [];
       debug("tags for path", path, tags);
-      return tags;
+      // We need to remove the buildId from the path
+      return tags.map((tag) => tag.replace(`${NEXT_BUILD_ID}/`, ""));
     } catch (e) {
       error("Failed to get tags by path", e);
       return [];

packages/tests-e2e/tests/appRouter/revalidateTag.test.ts

@@ -64,3 +64,28 @@ test("Revalidate tag", async ({ page, request }) => {
   response = await responsePromise;
   expect(response.headers()["x-nextjs-cache"]).toEqual("HIT");
 });
+
+test("Revalidate path", async ({ page, request }) => {
+  await page.goto("/revalidate-path");
+
+  let elLayout = page.getByText("Paris:");
+  const initialParis = await elLayout.textContent();
+
+  elLayout = page.getByText("London:");
+  const initialLondon = await elLayout.textContent();
+
+  // Send revalidate path request
+  const result = await request.get("/api/revalidate-path");
+  expect(result.status()).toEqual(200);
+  const text = await result.text();
+  expect(text).toEqual("ok");
+
+  await page.goto("/revalidate-path");
+  elLayout = page.getByText("Paris:");
+  const newParis = await elLayout.textContent();
+  expect(newParis).not.toEqual(initialParis);
+
+  elLayout = page.getByText("London:");
+  const newLondon = await elLayout.textContent();
+  expect(newLondon).not.toEqual(initialLondon);
+});