Using Espressif download mirrors (#514)

* rm error encapsulation, add deps cleaning

* fix lint

* get esp-idf version without git

* add github assets mirror for tools download

gulpfile.js

@@ -117,13 +117,22 @@ async function addExtensionDependencies(done) {
   done();
 }
 
+async function removeExtensionDependencies(done) {
+  const packageJson = await readJSON("package.json");
+  if (packageJson.extensionDependencies) {
+    packageJson.extensionDependencies = undefined;
+    await writeJSON("package.json", packageJson, { spaces: 2});
+  }
+  done();
+}
+
 const preBuild = gulp.series(clean, addI18n, validateLocalizationFiles);
 const build = gulp.series(preBuild, addExtensionDependencies);
-exports.clean = clean;
+exports.clean = gulp.series(clean, removeExtensionDependencies);
 exports.build = build;
 exports.validateLocalization = validateLocalizationFiles;
 exports.publish = gulp.series(build, vscePublish);
 exports.vscePkg = gulp.series(build, vscePackage);
 exports.default = build;
 
-exports.noDepBuild = preBuild;
+exports.noDepBuild = gulp.series(preBuild, removeExtensionDependencies);

src/config.ts

@@ -46,6 +46,7 @@ export namespace ESP {
   }
 
   export namespace URL {
+    export const IDF_GITHUB_ASSETS = "https://dl.espressif.com/github_assets";
     export namespace IDF_EMBED_GIT {
       export const IDF_EMBED_GIT_URL = `https://dl.espressif.com/dl/idf-git/idf-git-2.30.1-win64.zip`;
       export const VERSION = "2.30.1";

src/downloadManager.ts

@@ -28,6 +28,7 @@ import { PackageError } from "./packageError";
 import { PackageProgress } from "./PackageProgress";
 import { PackageManagerWebError } from "./packageWebError";
 import * as utils from "./utils";
+import { IdfMirror } from "./views/setup/types";
 
 export class DownloadManager {
   constructor(
@@ -42,6 +43,7 @@ export class DownloadManager {
   public downloadPackages(
     idfToolsManager: IdfToolsManager,
     progress: vscode.Progress<{ message?: string; increment?: number }>,
+    mirror: IdfMirror,
     pkgsProgress?: PackageProgress[],
     cancelToken?: vscode.CancellationToken
   ): Promise<void> {
@@ -58,6 +60,7 @@ export class DownloadManager {
           }
           const p: Promise<void> = this.downloadPackage(
             idfToolsManager,
+            mirror,
             pkg,
             `${count}/${packages.length}`,
             progress,
@@ -73,6 +76,7 @@ export class DownloadManager {
 
   public async downloadPackage(
     idfToolsManager: IdfToolsManager,
+    mirror: IdfMirror,
     pkg: IPackage,
     progressCount: string,
     progress: vscode.Progress<{ message?: string; increment?: number }>,
@@ -82,6 +86,11 @@ export class DownloadManager {
     progress.report({ message: `Downloading ${progressCount}: ${pkg.name}` });
     this.appendChannel(`Downloading ${pkg.description}`);
     const urlInfoToUse = idfToolsManager.obtainUrlInfoForPlatform(pkg);
+    if (mirror == IdfMirror.Espressif) {
+      urlInfoToUse.url = idfToolsManager.applyGithubAssetsMapping(
+        urlInfoToUse.url
+      );
+    }
     await this.downloadPackageWithRetries(
       pkg,
       urlInfoToUse,

src/idfToolsManager.ts

@@ -21,6 +21,7 @@ import * as utils from "./utils";
 import { Logger } from "./logger/logger";
 import { OutputChannel } from "./logger/outputChannel";
 import { readJSON } from "fs-extra";
+import { ESP } from "./config";
 
 export interface IEspIdfTool {
   actual: string;
@@ -151,6 +152,13 @@ export class IdfToolsManager {
     return linkInfo;
   }
 
+  public applyGithubAssetsMapping(urlToReplace: string) {
+    return urlToReplace.replace(
+      /https:\/\/github.com/g,
+      ESP.URL.IDF_GITHUB_ASSETS
+    );
+  }
+
   public getVersionToUse(pkg: IPackage): string {
     const versions = pkg.versions.filter((value) => {
       return (

src/setup/SetupPanel.ts

@@ -149,12 +149,18 @@ export class SetupPanel {
           }
           break;
         case "installEspIdfTools":
-          if (message.espIdf && message.pyPath && message.toolsPath) {
+          if (
+            message.espIdf &&
+            message.pyPath &&
+            message.toolsPath &&
+            typeof message.mirror !== undefined
+          ) {
             await this.installEspIdfTools(
               message.espIdf,
               message.pyPath,
               message.toolsPath,
-              setupArgs.gitPath
+              setupArgs.gitPath,
+              message.mirror
             );
           }
           break;
@@ -432,7 +438,8 @@ export class SetupPanel {
     idfPath: string,
     pyPath: string,
     toolsPath: string,
-    gitPath: string
+    gitPath: string,
+    mirror: IdfMirror
   ) {
     return await vscode.window.withProgress(
       {
@@ -455,6 +462,7 @@ export class SetupPanel {
             toolsPath,
             pyPath,
             gitPath,
+            mirror,
             progress,
             cancelToken
           );

src/setup/embedGitPy.ts

@@ -64,7 +64,7 @@ export async function installIdfGit(
   if (gitPathExists) {
     const gitDirectory = join(idfGitDestPath, "cmd");
     const binVersion = await checkGitExists(gitDirectory, resultGitPath);
-    if (binVersion) {
+    if (!binVersion || binVersion === "Not found") {
       OutputChannel.appendLine(`Using existing ${idfGitDestPath}`);
       return resultGitPath;
     }

src/setup/espIdfDownloadStep.ts

@@ -106,6 +106,7 @@ export async function expressInstall(
     toolsPath,
     pyPath,
     gitPath,
+    mirror,
     progress,
     cancelToken
   );

src/setup/toolInstall.ts

@@ -25,10 +25,12 @@ import {
   sendPkgDownloadFailed,
 } from "./webviewMsgMethods";
 import { CancellationToken, Progress } from "vscode";
+import { IdfMirror } from "../views/setup/types";
 
 export async function downloadEspIdfTools(
   installDir: string,
   idfToolsManager: IdfToolsManager,
+  mirror: IdfMirror,
   progress: Progress<{ message: string; increment?: number }>,
   cancelToken?: CancellationToken
 ) {
@@ -55,6 +57,7 @@ export async function downloadEspIdfTools(
   await downloadManager.downloadPackages(
     idfToolsManager,
     progress,
+    mirror,
     pkgProgress,
     cancelToken
   );

src/setup/toolsDownloadStep.ts

@@ -17,14 +17,15 @@ import * as vscode from "vscode";
 import { IdfToolsManager } from "../idfToolsManager";
 import { SetupPanel } from "./SetupPanel";
 import { downloadEspIdfTools } from "./toolInstall";
-import { StatusType } from "../views/setup/types";
+import { IdfMirror, StatusType } from "../views/setup/types";
 import { createPyReqs } from "./pyReqsInstallStep";
 
 export async function downloadIdfTools(
   idfPath: string,
   toolsPath: string,
   pyPath: string,
   gitPath: string,
+  mirror: IdfMirror,
   progress?: vscode.Progress<{ message: string; increment?: number }>,
   cancelToken?: vscode.CancellationToken
 ) {
@@ -46,7 +47,7 @@ export async function downloadIdfTools(
     command: "setRequiredToolsInfo",
     toolsInfo: requiredTools,
   });
-  await downloadEspIdfTools(toolsPath, idfToolsManager, progress, cancelToken);
+  await downloadEspIdfTools(toolsPath, idfToolsManager, mirror, progress, cancelToken);
   SetupPanel.postMessage({
     command: "updateEspIdfToolsStatus",
     status: StatusType.installed,

src/utils.ts

@@ -138,17 +138,15 @@ export function spawn(
     child.stdout.on("data", sendToOutputChannel);
     child.stderr.on("data", sendToOutputChannel);
 
-    child.on("error", (error) => reject({ error }));
+    child.on("error", (error) => reject(error));
 
     child.on("exit", (code) => {
       if (code === 0) {
         resolve(buff);
       } else {
-        const msg = "non zero exit code " + code + EOL + EOL + buff;
-        Logger.error(msg, new Error(msg));
-        reject({
-          error: new Error("non zero exit code " + code + "\n\n" + buff),
-        });
+        const err = new Error("non zero exit code " + code + EOL + EOL + buff);
+        Logger.error(err.message, err);
+        reject(err);
       }
     });
   });
@@ -614,11 +612,17 @@ export function getSubProjects(dir: string): string[] {
 
 export async function getEspIdfVersion(workingDir: string, gitPath: string) {
   try {
-    const canCheck = await checkGitExists(
-      extensionContext.extensionPath,
-      gitPath
-    );
-    if (canCheck === "Not found") {
+    const doesWorkingDirExists = await pathExists(workingDir);
+    if (!doesWorkingDirExists) {
+      Logger.info(`${workingDir} does not exists to get ESP-IDF version.`);
+      return "x.x";
+    }
+    const gitVersion = await checkGitExists(workingDir, gitPath);
+    if (!gitVersion || gitVersion === "Not found") {
+      const espIdfVersionFromCmake = await getEspIdfFromCMake(workingDir);
+      if (espIdfVersionFromCmake) {
+        return espIdfVersionFromCmake;
+      }
       Logger.errorNotify(
         "Git is not found in current environment",
         Error("git is not found")
@@ -649,28 +653,56 @@ export async function getEspIdfVersion(workingDir: string, gitPath: string) {
   }
 }
 
+export async function getEspIdfFromCMake(espIdfPath: string) {
+  const versionFilePath = path.join(
+    espIdfPath,
+    "tools",
+    "cmake",
+    "version.cmake"
+  );
+  const doesVersionFileExists = await pathExists(versionFilePath);
+  if (!doesVersionFileExists) {
+    Logger.info(`${versionFilePath} does not exist to get ESP-IDF version.`);
+    return "x.x";
+  }
+  const versionFileContent = await readFile(versionFilePath, "utf8");
+  let versionMatches: RegExpExecArray;
+  let espVersion = {};
+  const cmakeVersionRegex = new RegExp(
+    /\s*set\s*\(\s*IDF_VERSION_([A-Z]{5})\s+(\d+)/gm
+  );
+  while ((versionMatches = cmakeVersionRegex.exec(versionFileContent))) {
+    espVersion[versionMatches[1]] = versionMatches[2];
+  }
+  if (Object.keys(espVersion).length) {
+    return `${espVersion["MAJOR"]}.${espVersion["MINOR"]}.${espVersion["PATCH"]}`;
+  } else {
+    return "x.x";
+  }
+}
+
 export async function checkGitExists(workingDir: string, gitPath: string) {
-  return await execChildProcess(`${gitPath} --version`, workingDir)
-    .then((result) => {
-      if (result) {
-        const match = result.match(
-          /(?:git\sversion\s)(\d+)(.\d+)?(.\d+)?(?:.windows.\d+)?/g
-        );
-        if (match && match.length > 0) {
-          return match[0].replace("git version ", "");
-        } else {
-          Logger.errorNotify(
-            "Git is not found in current environment",
-            Error("")
-          );
-          return "Not found";
-        }
-      }
-    })
-    .catch((err) => {
-      Logger.errorNotify("Git is not found in current environment", err);
+  try {
+    const gitBinariesExists = await pathExists(gitPath);
+    if (!gitBinariesExists) {
       return "Not found";
-    });
+    }
+    const gitRawVersion = await execChildProcess(
+      `${gitPath} --version`,
+      workingDir
+    );
+    const match = gitRawVersion.match(
+      /(?:git\sversion\s)(\d+)(.\d+)?(.\d+)?(?:.windows.\d+)?/g
+    );
+    if (match && match.length) {
+      return match[0].replace("git version ", "");
+    } else {
+      return "Not found";
+    }
+  } catch (error) {
+    Logger.errorNotify("Git is not found in current environment", error);
+    return "Not found";
+  }
 }
 
 export function buildPromiseChain<TItem, TPromise>(

src/views/setup/components/selectEspIdf.vue

@@ -1,9 +1,6 @@
 <template>
   <div id="select-esp-idf-version">
-    <div
-      class="field"
-      v-if="selectedIdfVersion && selectedIdfVersion.filename !== 'manual'"
-    >
+    <div class="field">
       <label for="idf-mirror-select" class="label"
         >Select download server:</label
       >

src/views/setup/store/index.ts

@@ -167,6 +167,7 @@ export const actions: ActionTree<IState, any> = {
     vscode.postMessage({
       command: "installEspIdfTools",
       espIdf: context.state.espIdf,
+      mirror: context.state.selectedIdfMirror,
       pyPath,
       toolsPath: context.state.toolsFolder,
     });