Merge pull request #4 from wizenheimer/dev

restructure exports
wizenheimer · Dec 18, 2023 · 61364b0 · 61364b0
2 parents 79f2bc1 + ded34ad
commit 61364b0
Show file tree

Hide file tree

Showing 14 changed files with 309 additions and 122 deletions.
diff --git a/extras/scripts/index.html → extras/benchmark/index.html b/extras/scripts/index.html → extras/benchmark/index.html
diff --git a/extras/scripts/script.ts → extras/benchmark/script.ts b/extras/scripts/script.ts → extras/benchmark/script.ts
@@ -1,5 +1,6 @@
 import { VectorStore } from "../src/store";
 import { SimilarityMetric } from "../src/metric";
+import { faker } from "@faker-js/faker";
 
 export const testStore = async (
     dataSizes: number[], // Vary the size of the dataset
@@ -20,9 +21,10 @@ export const testStore = async (
                         { length: dataSize },
                         (_, index) => ({
                             id: index + 1,
-                            vector: Array.from({ length: dimension }, () =>
+                            embedding: Array.from({ length: dimension }, () =>
                                 Math.random()
-                            )
+                            ),
+                            content: faker.hacker.phrase()
                         })
                     );
 
@@ -32,11 +34,12 @@ export const testStore = async (
                     });
 
                     // Measure index build time
-                    console.time("VectorStoreIndexBuildTime");
-                    for (const item of data) {
-                        await vectorStore.addVector(item.id, item.vector);
-                    }
-                    console.timeEnd("VectorStoreIndexBuildTime");
+                    // console.time("VectorStoreIndexBuildTime");
+                    vectorStore.buildIndex(data);
+                    // console.timeEnd("VectorStoreIndexBuildTime");
+                    // for (const item of data) {
+                    //     await vectorStore.addVector(item.id, item.vector);
+                    // }
 
                     // Save the index
                     await vectorStore.saveIndex();
@@ -47,9 +50,9 @@ export const testStore = async (
 };
 
 // Example usage
-const dataSizes = [1000];
+const dataSizes = [10];
 const dimensions = [128];
-const kValues = [1];
+const kValues = [5];
 const similarityMetric = [SimilarityMetric.cosine];
 
 console.log("loading scripts...");

diff --git a/extras/scripts/testIndex.ts → extras/benchmark/testIndex.ts b/extras/scripts/testIndex.ts → extras/benchmark/testIndex.ts
@@ -22,7 +22,7 @@ export const testIndex = async (
                         { length: dataSize },
                         (_, index) => ({
                             id: index + 1,
-                            vector: Array.from({ length: dimension }, () =>
+                            embedding: Array.from({ length: dimension }, () =>
                                 Math.random()
                             )
                         })
@@ -63,3 +63,11 @@ export const testIndex = async (
         }
     }
 };
+
+const dataSizes = [10];
+const dimensions = [128];
+const kValues = [5];
+const similarityMetric = [SimilarityMetric.cosine];
+
+console.log("loading scripts...");
+testIndex(dataSizes, dimensions, kValues, similarityMetric, true);
diff --git a/extras/scripts/testStore.ts → extras/benchmark/testStore.ts b/extras/scripts/testStore.ts → extras/benchmark/testStore.ts
@@ -1,5 +1,4 @@
 import { VectorStore } from "../../src/store";
-import { CacheOptions } from "../../src/cache";
 import { SimilarityMetric } from "../../src/metric";
 import fs from "fs/promises";
 import { HNSW } from "../../src/hnsw";
@@ -9,8 +8,7 @@ export const testStore = async (
     dimensions: number[], // Vary the dimensionality of vectors
     kValues: number[], // Vary the number of neighbors to search for
     similarityMetrics: SimilarityMetric[] = [SimilarityMetric.cosine], // Similarity metrics as needed
-    checkSerialization: boolean = false,
-    cacheOptions: CacheOptions | null = null
+    checkSerialization: boolean = false
 ) => {
     for (const dataSize of dataSizes) {
         for (const dimension of dimensions) {
@@ -25,7 +23,7 @@ export const testStore = async (
                         { length: dataSize },
                         (_, index) => ({
                             id: index + 1,
-                            vector: Array.from({ length: dimension }, () =>
+                            embedding: Array.from({ length: dimension }, () =>
                                 Math.random()
                             )
                         })
@@ -66,14 +64,13 @@ export const testStore = async (
 
                     // Test VectorStore
                     const vectorStore = await VectorStore.create({
-                        collectionName: "test-collection",
-                        cacheOptions: cacheOptions
+                        collectionName: "test-collection"
                     });
 
                     // Measure index build time
                     console.time("VectorStoreIndexBuildTime");
                     for (const item of data) {
-                        await vectorStore.addVector(item.id, item.vector);
+                        await vectorStore.addVector(item.id, item.embedding);
                     }
                     console.timeEnd("VectorStoreIndexBuildTime");