Merge pull request #5582 from continuedev/pe/ripgrep-download

feat: manual ripgrep downloads for JB

Author: Patrick-Erichsen

binary/build.js

@@ -8,13 +8,15 @@ const {
   execCmdSync,
   autodetectPlatformAndArch,
 } = require("../scripts/util");
+const { downloadRipgrep } = require("./utils/ripgrep");
+const { ALL_TARGETS, TARGET_TO_LANCEDB } = require("./utils/targets");
 
 const bin = path.join(__dirname, "bin");
 const out = path.join(__dirname, "out");
 const build = path.join(__dirname, "build");
 
 function cleanSlate() {
-// Clean slate
+  // Clean slate
   rimrafSync(bin);
   rimrafSync(out);
   rimrafSync(build);
@@ -25,13 +27,7 @@ function cleanSlate() {
 }
 
 const esbuildOutputFile = "out/index.js";
-let targets = [
-  "darwin-x64",
-  "darwin-arm64",
-  "linux-x64",
-  "linux-arm64",
-  "win32-x64",
-];
+let targets = [...ALL_TARGETS];
 
 const [currentPlatform, currentArch] = autodetectPlatformAndArch();
 
@@ -50,15 +46,6 @@ for (let i = 2; i < process.argv.length; i++) {
   }
 }
 
-const targetToLanceDb = {
-  "darwin-arm64": "@lancedb/vectordb-darwin-arm64",
-  "darwin-x64": "@lancedb/vectordb-darwin-x64",
-  "linux-arm64": "@lancedb/vectordb-linux-arm64-gnu",
-  "linux-x64": "@lancedb/vectordb-linux-x64-gnu",
-  "win32-x64": "@lancedb/vectordb-win32-x64-msvc",
-  "win32-arm64": "@lancedb/vectordb-win32-arm64-msvc",
-};
-
 // Bundles the extension into one file
 async function buildWithEsbuild() {
   console.log("[info] Building with esbuild...");
@@ -89,7 +76,6 @@ async function buildWithEsbuild() {
     inject: ["./importMetaUrl.js"],
     define: { "import.meta.url": "importMetaUrl" },
   });
-
 }
 
 async function installNodeModuleInTempDirAndCopyToCurrent(packageName, toCopy) {
@@ -153,13 +139,31 @@ async function installNodeModuleInTempDirAndCopyToCurrent(packageName, toCopy) {
   }
 }
 
+/**
+ * Downloads and installs ripgrep binaries for the specified target
+ *
+ * @param {string} target - Target platform-arch (e.g., 'darwin-x64')
+ * @param {string} targetDir - Directory to install ripgrep to
+ * @returns {Promise<void>}
+ */
+async function downloadRipgrepForTarget(target, targetDir) {
+  console.log(`[info] Downloading ripgrep for ${target}...`);
+  try {
+    await downloadRipgrep(target, targetDir);
+    console.log(`[info] Successfully installed ripgrep for ${target}`);
+  } catch (error) {
+    console.error(`[error] Failed to download ripgrep for ${target}:`, error);
+    throw error;
+  }
+}
+
 (async () => {
   if (esbuildOnly) {
     await buildWithEsbuild();
     return;
   }
 
-  cleanSlate()
+  cleanSlate();
 
   // Informs of where to look for node_sqlite3.node https://www.npmjs.com/package/bindings#:~:text=The%20searching%20for,file%20is%20found
   // This is only needed for our `pkg` command at build time
@@ -179,10 +183,10 @@ async function installNodeModuleInTempDirAndCopyToCurrent(packageName, toCopy) {
 
   console.log("[info] Downloading prebuilt lancedb...");
   for (const target of targets) {
-    if (targetToLanceDb[target]) {
+    if (TARGET_TO_LANCEDB[target]) {
       console.log(`[info] Downloading for ${target}...`);
       await installNodeModuleInTempDirAndCopyToCurrent(
-        targetToLanceDb[target],
+        TARGET_TO_LANCEDB[target],
         "@lancedb",
       );
     }
@@ -290,10 +294,13 @@ async function installNodeModuleInTempDirAndCopyToCurrent(packageName, toCopy) {
     // copy @lancedb to bin folders
     console.log("[info] Copying @lancedb files to bin");
     fs.copyFileSync(
-      `node_modules/${targetToLanceDb[target]}/index.node`,
+      `node_modules/${TARGET_TO_LANCEDB[target]}/index.node`,
       `${targetDir}/index.node`,
     );
 
+    // Download and install ripgrep for the target
+    await downloadRipgrepForTarget(target, targetDir);
+
     // Informs the `continue-binary` of where to look for node_sqlite3.node
     // https://www.npmjs.com/package/bindings#:~:text=The%20searching%20for,file%20is%20found
     fs.writeFileSync(`${targetDir}/package.json`, "");
@@ -310,6 +317,7 @@ async function installNodeModuleInTempDirAndCopyToCurrent(packageName, toCopy) {
       `${targetDir}/continue-binary${exe}`,
       `${targetDir}/index.node`, // @lancedb
       `${targetDir}/build/Release/node_sqlite3.node`,
+      `${targetDir}/rg${exe}`, // ripgrep binary
     );
   }
 

binary/package-lock.json

@@ -42,6 +42,7 @@
   },
   "dependencies": {
     "@octokit/rest": "^20.0.2",
+    "adm-zip": "^0.5.16",
     "commander": "^12.0.0",
     "core": "file:../core",
     "follow-redirects": "^1.15.5",
@@ -50,6 +51,7 @@
     "node-fetch": "^3.3.2",
     "posthog-node": "^3.6.3",
     "system-ca": "^1.0.2",
+    "tar": "^7.4.3",
     "uuid": "^9.0.1",
     "vectordb": "^0.4.20",
     "win-ca": "^3.5.1"