From b6edd77c78a583fd7af8c318fa2ae6e98d9bbe4e Mon Sep 17 00:00:00 2001 From: "Yuichiro Tachibana (Tsuchiya)" Date: Thu, 30 May 2024 05:43:55 +0100 Subject: [PATCH] Add --pyodide-source option and remove manual downloading of the prebuilt packages relying on Pyodide's caching mechanism (#937) * Add --pyodide-source option and remove manual downloading of the prebuilt packages relying on Pyodide's caching mechanism * Update README.md * Refactoring * Specify indexURL * Revert "Specify indexURL" This reverts commit 63223b570584ab085b302d11acc5f47d30731cb2. --- packages/desktop/README.md | 13 ++ .../desktop/bin-src/dump_artifacts/index.ts | 129 +++++++++--------- .../dump_artifacts/pyodide_packages.ts | 69 ++++++---- .../desktop/bin-src/dump_artifacts/url.ts | 5 - 4 files changed, 120 insertions(+), 96 deletions(-) delete mode 100644 packages/desktop/bin-src/dump_artifacts/url.ts diff --git a/packages/desktop/README.md b/packages/desktop/README.md index 6861b5410..7a6d7f7e6 100644 --- a/packages/desktop/README.md +++ b/packages/desktop/README.md @@ -95,6 +95,19 @@ See the [./samples](./samples) directory for sample projects. To make your app secure, be sure to use the latest version of Electron. This is [announced](https://www.electronjs.org/docs/latest/tutorial/security#16-use-a-current-version-of-electron) as one of the security best practices in the Electron document too. +## Use a custom Pyodide source + +The `dump` command downloads some Pyodide resources such as the prebuilt package wheel files from [the JsDelivr CDN](https://pyodide.org/en/stable/usage/downloading-and-deploying.html#cdn) by default. +If you want to use a different Pyodide source, for example when accessing JsDelivr (`cdn.jsdelivr.net`) is restricted in your environment, +you can specify a URL or a path to the Pyodide source by setting the `--pyodide-source` option of the `dump` command. + +For example, if you downloaded a Pyodide package from the [Pyodide releases](https://pyodide.org/en/stable/usage/downloading-and-deploying.html#github-releases) and saved it in `/path/to/pyodide/`, you can specify the URL to the Pyodide package like below. + +```sh +npm run dump -- --pyodide-source /path/to/pyodide/ +yarn dump --pyodide-source /path/to/pyodide/ +``` + ## Configure the app ### Hide the toolbar, hamburger menu, and the footer diff --git a/packages/desktop/bin-src/dump_artifacts/index.ts b/packages/desktop/bin-src/dump_artifacts/index.ts index 205d835e6..c5f9a2edd 100755 --- a/packages/desktop/bin-src/dump_artifacts/index.ts +++ b/packages/desktop/bin-src/dump_artifacts/index.ts @@ -5,9 +5,12 @@ import { hideBin } from "yargs/helpers"; import path from "node:path"; import fsPromises from "node:fs/promises"; import fsExtra from "fs-extra"; -import { loadPyodide, type PyodideInterface } from "pyodide"; -import { makePyodideUrl } from "./url"; -import { PrebuiltPackagesData } from "./pyodide_packages"; +import { + loadPyodide, + version as pyodideVersion, + type PyodideInterface, +} from "pyodide"; +import { PrebuiltPackagesDataReader } from "./pyodide_packages"; import { dumpManifest } from "./manifest"; import { readConfig } from "./config"; import { validateRequirements, parseRequirementsTxt } from "@stlite/common"; @@ -76,22 +79,30 @@ async function copyBuildDirectory(options: CopyBuildDirectoryOptions) { await fsExtra.copy(sourceDir, options.copyTo, { errorOnExist: true }); } -interface InspectUsedPrebuiltPackagesOptions { +interface LoadUsedPrebuiltPackagesOptions { + pyodideSource: string; + pyodideRuntimeDir: string; requirements: string[]; } /** - * Get the list of the prebuilt packages used by the given requirements. - * These package files (`pyodide/*.whl`) will be vendored in the app executable - * and loaded at runtime to avoid problems such as https://github.com/whitphx/stlite/issues/558 + * Load the Pyodide runtime and install the given requirements to load the prebuilt packages used by the requirements. + * Those prebuilt package wheels will be downloaded/copied to a local directory, `pyodideRuntimeDir`. + * Pyodide's caching mechanism available in the Node environment is used here as the wheel file downloader. + * `pyodideRuntimeDir` should be "build/pyodide" so that the downloaded/copied files will be vendored in the app executable. + * This vendoring and runtime-loading mechanism is necessary to avoid problems such as https://github.com/whitphx/stlite/issues/558 */ -async function inspectUsedPrebuiltPackages( - options: InspectUsedPrebuiltPackagesOptions +async function saveUsedPrebuiltPackages( + options: LoadUsedPrebuiltPackagesOptions ): Promise { if (options.requirements.length === 0) { return []; } - const pyodide = await loadPyodide(); + const pyodide = await loadPyodide({ + packageCacheDir: options.pyodideRuntimeDir, + }); + // @ts-ignore + pyodide._api.setCdnUrl(options.pyodideSource); await installPackages(pyodide, { requirements: options.requirements, @@ -148,6 +159,8 @@ async function installPackages( interface CreateSitePackagesSnapshotOptions { requirements: string[]; usedPrebuiltPackages: string[]; + pyodideRuntimeDir: string; + pyodideSource: string; saveTo: string; } async function createSitePackagesSnapshot( @@ -155,20 +168,28 @@ async function createSitePackagesSnapshot( ) { logger.info("Create the site-packages snapshot file..."); - const pyodide = await loadPyodide(); + const pyodide = await loadPyodide({ + packageCacheDir: options.pyodideRuntimeDir, + }); + // @ts-ignore + pyodide._api.setCdnUrl(options.pyodideSource); await ensureLoadPackage(pyodide, "micropip"); const micropip = pyodide.pyimport("micropip"); - const prebuiltPackagesData = await PrebuiltPackagesData.getInstance(); + const prebuiltPackagesDataReader = new PrebuiltPackagesDataReader( + options.pyodideSource + ); const mockedPackages: string[] = []; if (options.usedPrebuiltPackages.length > 0) { logger.info( "Mocking prebuilt packages so that they will not be included in the site-packages snapshot because these will be installed from the vendored wheel files at runtime..." ); - options.usedPrebuiltPackages.forEach((pkg) => { - const packageInfo = prebuiltPackagesData.getPackageInfoByName(pkg); + for (const pkg of options.usedPrebuiltPackages) { + const packageInfo = await prebuiltPackagesDataReader.getPackageInfoByName( + pkg + ); if (packageInfo == null) { throw new Error(`Package ${pkg} is not found in the lock file.`); } @@ -176,7 +197,7 @@ async function createSitePackagesSnapshot( logger.debug(`Mock ${packageInfo.name} ${packageInfo.version}`); micropip.add_mock_package(packageInfo.name, packageInfo.version); mockedPackages.push(packageInfo.name); - }); + } } logger.info(`Install the requirements %j`, options.requirements); @@ -215,7 +236,7 @@ async function createSitePackagesSnapshot( interface CopyAppDirectoryOptions { cwd: string; filePathPatterns: string[]; - buildAppDirectory: string; + destAppDir: string; } async function copyAppDirectory(options: CopyAppDirectoryOptions) { @@ -238,7 +259,7 @@ async function copyAppDirectory(options: CopyAppDirectoryOptions) { await Promise.all( fileRelPaths.map(async (relPath) => { const srcPath = path.resolve(options.cwd, relPath); - const destPath = path.resolve(options.buildAppDirectory, relPath); + const destPath = path.resolve(options.destAppDir, relPath); logger.debug(`Copy ${srcPath} to ${destPath}`); await fsExtra.copy(srcPath, destPath, { errorOnExist: true, @@ -249,12 +270,12 @@ async function copyAppDirectory(options: CopyAppDirectoryOptions) { ); } -async function assertAppDirectoryContainsEntrypoint( - appDirectory: string, +async function assertAppDirContainsEntrypoint( + appDir: string, entrypoint: string ) { try { - await fsPromises.access(path.resolve(appDirectory, entrypoint)); + await fsPromises.access(path.resolve(appDir, entrypoint)); } catch { throw new Error( `The entrypoint file "${entrypoint}" is not included in the bundled files.` @@ -281,42 +302,6 @@ async function writePrebuiltPackagesTxt( }); } -interface DownloadPrebuiltPackageWheelsOptions { - packages: string[]; - destDir: string; -} -async function downloadPrebuiltPackageWheels( - options: DownloadPrebuiltPackageWheelsOptions -) { - const prebuiltPackagesData = await PrebuiltPackagesData.getInstance(); - const usedPrebuiltPackages = options.packages.map((pkgName) => - prebuiltPackagesData.getPackageInfoByName(pkgName) - ); - const usedPrebuiltPackageUrls = usedPrebuiltPackages.map((pkg) => - makePyodideUrl(pkg.file_name) - ); - - logger.info("Downloading the used prebuilt packages..."); - await Promise.all( - usedPrebuiltPackageUrls.map(async (pkgUrl) => { - const dstPath = path.resolve( - options.destDir, - "./pyodide", - path.basename(pkgUrl) - ); - logger.debug(`Download ${pkgUrl} to ${dstPath}`); - const res = await fetch(pkgUrl); - if (!res.ok) { - throw new Error( - `Failed to download ${pkgUrl}: ${res.status} ${res.statusText}` - ); - } - const buf = await res.arrayBuffer(); - await fsPromises.writeFile(dstPath, Buffer.from(buf)); - }) - ); -} - yargs(hideBin(process.argv)) .command( "* [appHomeDirSource] [packages..]", @@ -355,6 +340,18 @@ yargs(hideBin(process.argv)) alias: "k", describe: "Keep the existing build directory contents except appHomeDir.", }) + .options("pyodideSource", { + type: "string", + describe: + "The base URL or path of the Pyodide files to download or copy, such as the prebuild package wheels and pyodide-lock.json", + default: `https://cdn.jsdelivr.net/pyodide/v${pyodideVersion}/full/`, + coerce: (urlOrPath) => { + if (!urlOrPath.endsWith("/")) { + urlOrPath += "/"; + } + return urlOrPath; + }, + }) .options("logLevel", { type: "string", default: "info", @@ -406,7 +403,11 @@ yargs(hideBin(process.argv)) ]); logger.info("Validated dependency list: %j", dependencies); - const usedPrebuiltPackages = await inspectUsedPrebuiltPackages({ + await copyBuildDirectory({ copyTo: destDir, keepOld: args.keepOldBuild }); + + const usedPrebuiltPackages = await saveUsedPrebuiltPackages({ + pyodideSource: args.pyodideSource, + pyodideRuntimeDir: path.resolve(destDir, "./pyodide"), requirements: dependencies, }); logger.info( @@ -414,19 +415,19 @@ yargs(hideBin(process.argv)) usedPrebuiltPackages ); - await copyBuildDirectory({ copyTo: destDir, keepOld: args.keepOldBuild }); - - const buildAppDirectory = path.resolve(destDir, "./app_files"); // This path will be loaded in the `readStreamlitAppDirectory` handler in electron/main.ts. + const destAppDir = path.resolve(destDir, "./app_files"); // This path will be loaded in the `readStreamlitAppDirectory` handler in electron/main.ts. await copyAppDirectory({ cwd: projectDir, filePathPatterns: config.files, - buildAppDirectory, + destAppDir, }); - assertAppDirectoryContainsEntrypoint(buildAppDirectory, config.entrypoint); + assertAppDirContainsEntrypoint(destAppDir, config.entrypoint); await createSitePackagesSnapshot({ requirements: dependencies, usedPrebuiltPackages, + pyodideSource: args.pyodideSource, + pyodideRuntimeDir: path.resolve(destDir, "./pyodide"), saveTo: path.resolve(destDir, "./site-packages-snapshot.tar.gz"), // This path will be loaded in the `readSitePackagesSnapshot` handler in electron/main.ts. }); // These prebuilt packages will be vendored in the build artifact by `downloadPrebuiltPackageWheels()` @@ -439,10 +440,6 @@ yargs(hideBin(process.argv)) path.resolve(destDir, "./prebuilt-packages.txt"), // This path will be loaded in the `readRequirements` handler in electron/main.ts. usedPrebuiltPackages ); - await downloadPrebuiltPackageWheels({ - packages: usedPrebuiltPackages, - destDir, - }); await dumpManifest({ packageJsonStliteDesktopField: packageJson.stlite?.desktop, manifestFilePath: path.resolve(destDir, "./stlite-manifest.json"), diff --git a/packages/desktop/bin-src/dump_artifacts/pyodide_packages.ts b/packages/desktop/bin-src/dump_artifacts/pyodide_packages.ts index 423b749db..55662e5d4 100644 --- a/packages/desktop/bin-src/dump_artifacts/pyodide_packages.ts +++ b/packages/desktop/bin-src/dump_artifacts/pyodide_packages.ts @@ -1,4 +1,5 @@ -import { makePyodideUrl } from "./url"; +import path from "node:path"; +import fsPromises from "node:fs/promises"; import { logger } from "./logger"; interface PackageInfo { @@ -7,39 +8,57 @@ interface PackageInfo { file_name: string; depends: string[]; } -export class PrebuiltPackagesData { - private static _instance: PrebuiltPackagesData; +export class PrebuiltPackagesDataReader { + private sourceUrl: string; + private isRemote: boolean; private _data: Record | null = null; - private constructor() {} - - private static async loadPrebuiltPackageData(): Promise< - Record - > { - const url = makePyodideUrl("pyodide-lock.json"); + constructor(sourceUrl: string) { + // These path logics are based on https://github.com/pyodide/pyodide/blob/0.25.1/src/js/compat.ts#L122 + if (sourceUrl.startsWith("file://")) { + // handle file:// with filesystem operations rather than with fetch. + sourceUrl = sourceUrl.slice("file://".length); + } + this.sourceUrl = sourceUrl; + this.isRemote = sourceUrl.includes("://"); + } - logger.info(`Load the Pyodide pyodide-lock.json from ${url}`); - const res = await fetch(url, undefined); - const resJson = await res.json(); + private async readJson(filepath: string): Promise { + const url = path.join(this.sourceUrl, filepath); - return resJson.packages; + if (this.isRemote) { + logger.debug(`Fetching ${url}`); + const res = await fetch(url); + if (!res.ok) { + throw new Error( + `Failed to download ${url}: ${res.status} ${res.statusText}` + ); + } + return await res.json(); + } else { + logger.debug(`Reading ${url}`); + const buf = await fsPromises.readFile(url); + return JSON.parse(buf.toString()); + } } - static async getInstance(): Promise { - if (this._instance == null) { - this._instance = new PrebuiltPackagesData(); - this._instance._data = await this.loadPrebuiltPackageData(); + private async loadPrebuiltPackageData(): Promise< + Record + > { + if (this._data != null) { + return this._data; } - return this._instance; + + logger.info(`Load pyodide-lock.json`); + const lockJson = await this.readJson("pyodide-lock.json"); + + this._data = lockJson.packages; + return lockJson.packages; } - public getPackageInfoByName(pkgName: string): PackageInfo { - if (this._data == null) { - throw new Error("The package data is not loaded yet."); - } - const pkgInfo = Object.values(this._data).find( - (pkg) => pkg.name === pkgName - ); + public async getPackageInfoByName(pkgName: string): Promise { + const data = await this.loadPrebuiltPackageData(); + const pkgInfo = Object.values(data).find((pkg) => pkg.name === pkgName); if (pkgInfo == null) { throw new Error(`Package ${pkgName} is not found in the lock file.`); } diff --git a/packages/desktop/bin-src/dump_artifacts/url.ts b/packages/desktop/bin-src/dump_artifacts/url.ts deleted file mode 100644 index 540f92f73..000000000 --- a/packages/desktop/bin-src/dump_artifacts/url.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { version as pyodideVersion } from "pyodide"; - -export function makePyodideUrl(filename: string): string { - return `https://cdn.jsdelivr.net/pyodide/v${pyodideVersion}/full/${filename}`; -}