Dylan/s3en 2086 add pagination to dataset api

app/(protected)/project/[project_id]/datasets/dataset/[dataset_id]/page.tsx

@@ -2,25 +2,73 @@
 
 import { CreateData } from "@/components/project/dataset/create-data";
 import { EditData } from "@/components/project/dataset/edit-data";
+import { Spinner } from "@/components/shared/spinner";
 import { Button } from "@/components/ui/button";
 import { Separator } from "@/components/ui/separator";
+import { PAGE_SIZE } from "@/lib/constants";
+import { Data } from "@prisma/client";
 import { ChevronLeft } from "lucide-react";
 import { useParams } from "next/navigation";
+import { useState } from "react";
+import { useBottomScrollListener } from "react-bottom-scroll-listener";
 import { useQuery } from "react-query";
 
 export default function Dataset() {
   const dataset_id = useParams()?.dataset_id as string;
+  const [page, setPage] = useState<number>(1);
+  const [totalPages, setTotalPages] = useState<number>(1);
+  const [showLoader, setShowLoader] = useState(false);
+  const [currentData, setCurrentData] = useState<Data[]>([]);
+
+  useBottomScrollListener(() => {
+    if (fetchDataset.isRefetching) {
+      return;
+    }
+    if (page <= totalPages) {
+      setShowLoader(true);
+      fetchDataset.refetch();
+    }
+  });
 
   const fetchDataset = useQuery({
     queryKey: [dataset_id],
     queryFn: async () => {
-      const response = await fetch(`/api/dataset?dataset_id=${dataset_id}`);
+      const response = await fetch(
+        `/api/dataset?dataset_id=${dataset_id}&page=${page}&pageSize=${PAGE_SIZE}`
+      );
       const result = await response.json();
       return result;
     },
+    onSuccess: (data) => {
+      // Get the newly fetched data and metadata
+      const newData: Data[] = data?.datasets?.Data || [];
+      const metadata = data?.metadata || {};
+
+      // Update the total pages and current page number
+      setTotalPages(parseInt(metadata?.total_pages) || 1);
+      if (parseInt(metadata?.page) <= parseInt(metadata?.total_pages)) {
+        setPage(parseInt(metadata?.page) + 1);
+      }
+
+      // Merge the new data with the existing data
+      if (currentData.length > 0) {
+        const updatedData = [...currentData, ...newData];
+
+        // Remove duplicates
+        const uniqueData = updatedData.filter(
+          (v: any, i: number, a: any) =>
+            a.findIndex((t: any) => t.id === v.id) === i
+        );
+
+        setCurrentData(uniqueData);
+      } else {
+        setCurrentData(newData);
+      }
+      setShowLoader(false);
+    },
   });
 
-  if (fetchDataset.isLoading || !fetchDataset.data) {
+  if (fetchDataset.isLoading || !fetchDataset.data || !currentData) {
     return <div>Loading...</div>;
   } else {
     return (
@@ -39,16 +87,16 @@ export default function Dataset() {
             <p className="text-xs font-medium">Output</p>
             <p className="text-xs font-medium text-end">Note</p>
           </div>
-          {fetchDataset.data?.datasets &&
-            fetchDataset.data?.datasets?.Data?.length === 0 && (
-              <div className="flex items-center justify-center">
-                <p className="text-muted-foreground">
-                  No data found in this dataset
-                </p>
-              </div>
-            )}
-          {fetchDataset.data?.datasets &&
-            fetchDataset.data?.datasets?.Data?.map((data: any, i: number) => {
+          {fetchDataset.isLoading && currentData.length === 0 && (
+            <div className="flex items-center justify-center">
+              <p className="text-muted-foreground">
+                No data found in this dataset
+              </p>
+            </div>
+          )}
+          {!fetchDataset.isLoading &&
+            currentData.length > 0 &&
+            currentData.map((data: any, i: number) => {
               return (
                 <div className="flex flex-col" key={i}>
                   <div className="grid grid-cols-5 items-start justify-stretch gap-3 py-3 px-4">
@@ -61,13 +109,22 @@ export default function Dataset() {
                     </p>
                     <p className="text-xs text-end">{data.note}</p>
                     <div className="text-end">
-                      <EditData key={data.id} idata={data} datasetId={dataset_id} />
+                      <EditData
+                        key={data.id}
+                        idata={data}
+                        datasetId={dataset_id}
+                      />
                     </div>
                   </div>
                   <Separator orientation="horizontal" />
                 </div>
               );
             })}
+          {showLoader && (
+            <div className="flex justify-center py-8">
+              <Spinner className="h-8 w-8 text-center" />
+            </div>
+          )}
         </div>
       </div>
     );

app/(protected)/project/[project_id]/datasets/promptset/[promptset_id]/page.tsx

@@ -64,7 +64,6 @@ export default function Promptset() {
       } else {
         setCurrentData(newData);
       }
-
       setShowLoader(false);
     },
   });

app/api/dataset/route.ts

@@ -12,6 +12,11 @@ export async function GET(req: NextRequest) {
 
   const id = req.nextUrl.searchParams.get("id") as string;
   const datasetId = req.nextUrl.searchParams.get("dataset_id") as string;
+  const pageParam = req.nextUrl.searchParams.get("page");
+  let page = pageParam ? parseInt(pageParam, 10) : 1;
+  const pageSizeParam = req.nextUrl.searchParams.get("pageSize");
+  const pageSize = pageSizeParam ? parseInt(pageSizeParam, 10) : 10;
+
   if (!datasetId && !id) {
     return NextResponse.json(
       {
@@ -41,6 +46,20 @@ export async function GET(req: NextRequest) {
       );
     }
 
+    const totalLen = await prisma.data.count({
+      where: {
+        datasetId: dataset.id,
+      },
+    });
+
+    const totalPages =
+      Math.ceil(totalLen / pageSize) === 0 ? 1 : Math.ceil(totalLen / pageSize);
+    const md = { page, page_size: pageSize, total_pages: totalPages };
+
+    if (page! > totalPages) {
+      page = totalPages;
+    }
+
     // If dataset exists, fetch related Data records ordered by createdAt
     const relatedData = await prisma.data.findMany({
       where: {
@@ -49,6 +68,8 @@ export async function GET(req: NextRequest) {
       orderBy: {
         createdAt: "desc",
       },
+      take: pageSize,
+      skip: (page - 1) * pageSize,
     });
 
     // Combine dataset with its related, ordered Data
@@ -59,6 +80,7 @@ export async function GET(req: NextRequest) {
 
     return NextResponse.json({
       datasets: datasetWithOrderedData,
+      metadata: md,
     });
   }