feat: revamp catalog sdk and its react-query (#1578)

Because

@thewbuk Please take a look at this PR, since this is the clean up task
for your implementation, I would like to point out several crucial
points that might be helpful for your future implementation

- You need to keep your writing consistent and clean, there have many
unnecessary syntax, such as
- `return Promise.resolve()` -> The async/await already wrap the
un-returned function for you
- You didn't cast `use client` in your react-query, they are hook so can
only work on the client side
- You need to follow what BE provided, there are dozens of endpoint
didn't provide pagination but you still expose pageSize and pageToken on
them such as listNamespaceCatalogs
- It's pretty clear that you didn't understand the API you are working
with well enough

Besides all the above code smell issues, I revamped some structure and
make it stay consistent with our other query-hook

- Centralized queryKeyStore for better key cache management
- Separate paginated endpoint and non-paginated endpoint in the sdk
- Retain the response structure in the sdk response, so user need to
pour them out like how we did it in the react-query, `res.chunk`

This commit

- revamp catalog sdk and its react-query

packages/sdk/src/catalog/types.ts

@@ -1,4 +1,4 @@
-import { Nullable } from "../types";
+import { GeneralRecord } from "../types";
 
 export type Catalog = {
   chunks: Chunk[];
@@ -21,6 +21,39 @@ export type Catalog = {
   catalogUid: string;
 };
 
+export type CatalogRunAction =
+  | "CATALOG_RUN_ACTION_CREATE"
+  | "CATALOG_RUN_ACTION_UPDATE"
+  | "CATALOG_RUN_ACTION_DELETE"
+  | "CATALOG_RUN_ACTION_CREATE_FILE"
+  | "CATALOG_RUN_ACTION_PROCESS_FILE"
+  | "CATALOG_RUN_ACTION_DELETE_FILE";
+
+export type CatalogRunStatus =
+  | "RUN_STATUS_PROCESSING"
+  | "RUN_STATUS_COMPLETED"
+  | "RUN_STATUS_FAILED"
+  | "RUN_STATUS_QUEUED";
+
+export type CatalogRunSource = "RUN_SOURCE_CONSOLE" | "RUN_SOURCE_API";
+
+export type CatalogRun = {
+  uid: string;
+  catalogUid: string;
+  fileUids: string[];
+  action: CatalogRunAction;
+  status: CatalogRunStatus;
+  source: CatalogRunSource;
+  totalDuration: number;
+  runnerId: string;
+  namespaceId: string;
+  payload: GeneralRecord;
+  startTime: string;
+  completeTime: string;
+  error?: string;
+  creditAmount: number;
+};
+
 export type Chunk = {
   chunkUid: string;
   content: string;
@@ -32,6 +65,14 @@ export type Chunk = {
   type: string;
 };
 
+export type SimilarChunk = {
+  chunkUid: string;
+  similarityScore: number;
+  textContent: string;
+  sourceFile: string;
+  chunkMetadata: GeneralRecord;
+};
+
 export type FileStatus =
   | "FILE_PROCESS_STATUS_NOTSTARTED"
   | "FILE_PROCESS_STATUS_WAITING"
@@ -41,6 +82,20 @@ export type FileStatus =
   | "FILE_PROCESS_STATUS_COMPLETED"
   | "FILE_PROCESS_STATUS_FAILED";
 
+export type FileType =
+  | "FILE_TYPE_TEXT"
+  | "FILE_TYPE_PDF"
+  | "FILE_TYPE_MARKDOWN"
+  | "FILE_TYPE_HTML"
+  | "FILE_TYPE_DOCX"
+  | "FILE_TYPE_DOC"
+  | "FILE_TYPE_PPT"
+  | "FILE_TYPE_PPTX"
+  | "FILE_TYPE_XLS"
+  | "FILE_TYPE_XLSX"
+  | "FILE_TYPE_CSV"
+  | "FILE_TYPE_UNSPECIFIED";
+
 export type File = {
   fileUid: string;
   name: string;
@@ -73,11 +128,8 @@ export type CatalogFile = {
   retrievable?: boolean;
 };
 
-export type ListCatalogsRequest = {
+export type ListNamespaceCatalogsRequest = {
   namespaceId: string;
-  pageSize?: number;
-  pageToken?: string;
-  view?: string;
 };
 
 export type ListCatalogsResponse = {
@@ -86,100 +138,159 @@ export type ListCatalogsResponse = {
   totalSize: number;
 };
 
-export type CreateCatalogRequest = {
+export type CreateNamespaceCatalogRequest = {
   namespaceId: string;
-  payload: {
-    name: string;
-    description?: string;
-    tags?: string[];
-    namespaceId: string;
-  };
+  name: string;
+  description?: string;
+  tags?: string[];
 };
 
-export type CreateCatalogResponse = Catalog;
+export type CreateNamespaceCatalogResponse = {
+  catalog: Catalog;
+};
 
-export type UpdateCatalogRequest = {
+export type UpdateNamespaceCatalogRequest = {
   namespaceId: string;
   catalogId: string;
   description?: string;
+  tags?: string[];
 };
 
-export type UpdateCatalogResponse = Catalog;
-
-export type DeleteCatalogRequest = {
-  namespaceId: string;
-  catalogId: string;
+export type UpdateNamespaceCatalogResponse = {
+  catalog: Catalog;
 };
 
-export type GetFileDetailsRequest = {
+export type DeleteNamespaceCatalogRequest = {
   namespaceId: string;
   catalogId: string;
-  fileId: string;
 };
 
-export type GetFileDetailsResponse = File;
-
-export type GetCatalogSingleSourceOfTruthFileRequest = {
+export type GetNamespaceCatalogSingleSourceOfTruthFileRequest = {
   namespaceId: string;
   catalogId: string;
   fileUid: string;
 };
 
-export type GetCatalogSingleSourceOfTruthFileResponse = FileContent;
+export type GetNamespaceCatalogSingleSourceOfTruthFileResponse = {
+  sourceFile: FileContent;
+};
 
-export type UploadCatalogFileRequest = {
+export type CreateNamespaceCatalogFileRequest = {
   namespaceId: string;
   catalogId: string;
-  payload: {
-    name: string;
-    type: string;
-    content: string;
-  };
+  name: string;
+  type: FileType;
+  content?: string;
 };
 
-export type UploadCatalogFileResponse = File;
+export type CreateNamespaceCatalogFileResponse = {
+  file: File;
+};
 
-export type ListCatalogFilesRequest = {
+export type ListPaginatedNamespaceCatalogFilesRequest = {
   namespaceId: string;
   catalogId: string;
   pageSize?: number;
   pageToken?: string;
 };
 
-export type ListCatalogFilesResponse = {
+export type ListPaginatedNamespaceCatalogFilesResponse = {
   files: File[];
   nextPageToken: string;
   totalSize: number;
 };
 
+export type ListNamespaceCatalogFilesRequest = {
+  namespaceId: string;
+  catalogId: string;
+  pageSize?: number;
+  pageToken?: string;
+};
+
+export type ListNamespaceCatalogFilesResponse = File[];
+
 export type DeleteCatalogFileRequest = {
   fileUid: string;
 };
 
 export type ProcessCatalogFilesRequest = {
   fileUids: string[];
-  requesterUid: Nullable<string>;
+  requesterUid?: string;
 };
 
-export type ProcessCatalogFilesResponse = File[];
+export type ProcessCatalogFilesResponse = {
+  files: File[];
+};
 
-export type ListChunksRequest = {
+export type ListNamespaceCatalogChunksRequest = {
   namespaceId: string;
   catalogId: string;
-  fileUid: string;
-  pageSize?: number;
-  pageToken?: string;
+  fileUid?: string;
+  chunkUids?: string[];
 };
 
-export type ListChunksResponse = {
+export type ListNamespaceCatalogChunksResponse = {
   chunks: Chunk[];
-  nextPageToken: string;
-  totalSize: number;
 };
 
-export type UpdateChunkRequest = {
+export type UpdateCatalogChunkRequest = {
   chunkUid: string;
   retrievable: boolean;
 };
 
-export type UpdateChunkResponse = Chunk;
+export type UpdateCatalogChunkResponse = {
+  chunk: Chunk;
+};
+
+export type RetrieveSimilarNamespaceCatalogChunksRequest = {
+  namespaceId: string;
+  catalogId: string;
+  textPrompt?: string;
+  topK?: number;
+  requesterUid?: string;
+};
+
+export type RetrieveSimilarNamespaceCatalogChunksResponse = {
+  similarChunks: SimilarChunk[];
+};
+
+export type AskNamespaceCatalogQuestionRequest = {
+  namespaceId: string;
+  catalogId: string;
+  question?: string;
+  topK?: number;
+  requesterUid?: string;
+};
+
+export type AskNamespaceCatalogQuestionResponse = {
+  answer: {
+    similarChunks: SimilarChunk[];
+  };
+};
+
+export type ListNamespaceCatalogRunsRequest = {
+  namespaceId: string;
+  catalogId: string;
+  pageSize?: number;
+  page?: number;
+  filter?: string;
+  orderBy?: string;
+};
+
+export type ListNamespaceCatalogRunsResponse = CatalogRun[];
+
+export type ListPaginatedNamespaceCatalogRunsRequest = {
+  namespaceId: string;
+  catalogId: string;
+  pageSize?: number;
+  page?: number;
+  filter?: string;
+  orderBy?: string;
+};
+
+export type ListPaginatedNamespaceCatalogRunsResponse = {
+  catalogRuns: CatalogRun[];
+  totalSize: number;
+  pageSize: number;
+  page: number;
+};

packages/sdk/src/helper/getQueryString.ts

@@ -15,6 +15,8 @@ export const getQueryString = ({
   stop,
   aggregationWindow,
   showDeleted,
+  fileUid,
+  chunkUids,
 }: {
   baseURL: string;
   pageSize?: number;
@@ -32,6 +34,8 @@ export const getQueryString = ({
   stop?: string;
   aggregationWindow?: string;
   showDeleted?: boolean;
+  fileUid?: string;
+  chunkUids?: string[];
 }) => {
   let url = baseURL;
 
@@ -110,6 +114,16 @@ export const getQueryString = ({
     url += `showDeleted=${showDeleted}&`;
   }
 
+  if (fileUid) {
+    url += `fileUid=${fileUid}&`;
+  }
+
+  if (chunkUids) {
+    for (const chunkUid of chunkUids) {
+      url += `chunkUids=${chunkUid}&`;
+    }
+  }
+
   if (url.endsWith("&")) {
     url = url.slice(0, -1);
   }

packages/toolkit/src/lib/react-query-service/catalog/index.ts

@@ -1,13 +1,11 @@
-export * from "./useCreateCatalog";
-export * from "./useDeleteCatalog";
-export * from "./useUpdateCatalog";
-export * from "./useGetCatalogs";
-export * from "./useUploadCatalogFile";
+export * from "./useCreateNamespaceCatalog";
+export * from "./useDeleteNamespaceCatalog";
+export * from "./useUpdateNamespaceCatalog";
+export * from "./useListNamespaceCatalogs";
+export * from "./useCreateNamespaceCatalogFile";
 export * from "./useProcessCatalogFiles";
-export * from "./useListCatalogFiles";
+export * from "./useListNamespaceCatalogFiles";
 export * from "./useDeleteCatalogFile";
-export * from "./useGetFileDetails";
-export * from "./useGetCatalogSingleSourceOfTruthFile";
-export * from "./useListChunks";
-export * from "./useUpdateChunk";
-export * from "./useGetAllChunks";
+export * from "./useGetNamespaceCatalogSingleSourceOfTruthFile";
+export * from "./useUpdateCatalogChunk";
+export * from "./useListNamespaceCatalogChunks";