Skip to content

Commit

Permalink
Add --pyodide-source option and remove manual downloading of the preb…
Browse files Browse the repository at this point in the history 
…uilt packages relying on Pyodide's caching mechanism (#937)

* Add --pyodide-source option and remove manual downloading of the prebuilt packages relying on Pyodide's caching mechanism

* Update README.md

* Refactoring

* Specify indexURL

* Revert "Specify indexURL"

This reverts commit 63223b5.
  • Loading branch information
@whitphx
whitphx authored May 30, 2024
1 parent fa88ed3 commit b6edd77
Show file tree
Hide file tree
Showing 4 changed files with 120 additions and 96 deletions.
13 changes: 13 additions & 0 deletions packages/desktop/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,19 @@ See the [./samples](./samples) directory for sample projects.
To make your app secure, be sure to use the latest version of Electron.
This is [announced](https://www.electronjs.org/docs/latest/tutorial/security#16-use-a-current-version-of-electron) as one of the security best practices in the Electron document too.

## Use a custom Pyodide source

The `dump` command downloads some Pyodide resources such as the prebuilt package wheel files from [the JsDelivr CDN](https://pyodide.org/en/stable/usage/downloading-and-deploying.html#cdn) by default.
If you want to use a different Pyodide source, for example when accessing JsDelivr (`cdn.jsdelivr.net`) is restricted in your environment,
you can specify a URL or a path to the Pyodide source by setting the `--pyodide-source` option of the `dump` command.

For example, if you downloaded a Pyodide package from the [Pyodide releases](https://pyodide.org/en/stable/usage/downloading-and-deploying.html#github-releases) and saved it in `/path/to/pyodide/`, you can specify the URL to the Pyodide package like below.

```sh
npm run dump -- --pyodide-source /path/to/pyodide/
yarn dump --pyodide-source /path/to/pyodide/
```

## Configure the app

### Hide the toolbar, hamburger menu, and the footer
Expand Down
129 changes: 63 additions & 66 deletions packages/desktop/bin-src/dump_artifacts/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,12 @@ import { hideBin } from "yargs/helpers";
import path from "node:path";
import fsPromises from "node:fs/promises";
import fsExtra from "fs-extra";
import { loadPyodide, type PyodideInterface } from "pyodide";
import { makePyodideUrl } from "./url";
import { PrebuiltPackagesData } from "./pyodide_packages";
import {
loadPyodide,
version as pyodideVersion,
type PyodideInterface,
} from "pyodide";
import { PrebuiltPackagesDataReader } from "./pyodide_packages";
import { dumpManifest } from "./manifest";
import { readConfig } from "./config";
import { validateRequirements, parseRequirementsTxt } from "@stlite/common";
Expand Down Expand Up @@ -76,22 +79,30 @@ async function copyBuildDirectory(options: CopyBuildDirectoryOptions) {
await fsExtra.copy(sourceDir, options.copyTo, { errorOnExist: true });
}

interface InspectUsedPrebuiltPackagesOptions {
interface LoadUsedPrebuiltPackagesOptions {
pyodideSource: string;
pyodideRuntimeDir: string;
requirements: string[];
}
/**
* Get the list of the prebuilt packages used by the given requirements.
* These package files (`pyodide/*.whl`) will be vendored in the app executable
* and loaded at runtime to avoid problems such as https://github.com/whitphx/stlite/issues/558
* Load the Pyodide runtime and install the given requirements to load the prebuilt packages used by the requirements.
* Those prebuilt package wheels will be downloaded/copied to a local directory, `pyodideRuntimeDir`.
* Pyodide's caching mechanism available in the Node environment is used here as the wheel file downloader.
* `pyodideRuntimeDir` should be "build/pyodide" so that the downloaded/copied files will be vendored in the app executable.
* This vendoring and runtime-loading mechanism is necessary to avoid problems such as https://github.com/whitphx/stlite/issues/558
*/
async function inspectUsedPrebuiltPackages(
options: InspectUsedPrebuiltPackagesOptions
async function saveUsedPrebuiltPackages(
options: LoadUsedPrebuiltPackagesOptions
): Promise<string[]> {
if (options.requirements.length === 0) {
return [];
}

const pyodide = await loadPyodide();
const pyodide = await loadPyodide({
packageCacheDir: options.pyodideRuntimeDir,
});
// @ts-ignore
pyodide._api.setCdnUrl(options.pyodideSource);

await installPackages(pyodide, {
requirements: options.requirements,
Expand Down Expand Up @@ -148,35 +159,45 @@ async function installPackages(
interface CreateSitePackagesSnapshotOptions {
requirements: string[];
usedPrebuiltPackages: string[];
pyodideRuntimeDir: string;
pyodideSource: string;
saveTo: string;
}
async function createSitePackagesSnapshot(
options: CreateSitePackagesSnapshotOptions
) {
logger.info("Create the site-packages snapshot file...");

const pyodide = await loadPyodide();
const pyodide = await loadPyodide({
packageCacheDir: options.pyodideRuntimeDir,
});
// @ts-ignore
pyodide._api.setCdnUrl(options.pyodideSource);

await ensureLoadPackage(pyodide, "micropip");
const micropip = pyodide.pyimport("micropip");

const prebuiltPackagesData = await PrebuiltPackagesData.getInstance();
const prebuiltPackagesDataReader = new PrebuiltPackagesDataReader(
options.pyodideSource
);

const mockedPackages: string[] = [];
if (options.usedPrebuiltPackages.length > 0) {
logger.info(
"Mocking prebuilt packages so that they will not be included in the site-packages snapshot because these will be installed from the vendored wheel files at runtime..."
);
options.usedPrebuiltPackages.forEach((pkg) => {
const packageInfo = prebuiltPackagesData.getPackageInfoByName(pkg);
for (const pkg of options.usedPrebuiltPackages) {
const packageInfo = await prebuiltPackagesDataReader.getPackageInfoByName(
pkg
);
if (packageInfo == null) {
throw new Error(`Package ${pkg} is not found in the lock file.`);
}

logger.debug(`Mock ${packageInfo.name} ${packageInfo.version}`);
micropip.add_mock_package(packageInfo.name, packageInfo.version);
mockedPackages.push(packageInfo.name);
});
}
}

logger.info(`Install the requirements %j`, options.requirements);
Expand Down Expand Up @@ -215,7 +236,7 @@ async function createSitePackagesSnapshot(
interface CopyAppDirectoryOptions {
cwd: string;
filePathPatterns: string[];
buildAppDirectory: string;
destAppDir: string;
}

async function copyAppDirectory(options: CopyAppDirectoryOptions) {
Expand All @@ -238,7 +259,7 @@ async function copyAppDirectory(options: CopyAppDirectoryOptions) {
await Promise.all(
fileRelPaths.map(async (relPath) => {
const srcPath = path.resolve(options.cwd, relPath);
const destPath = path.resolve(options.buildAppDirectory, relPath);
const destPath = path.resolve(options.destAppDir, relPath);
logger.debug(`Copy ${srcPath} to ${destPath}`);
await fsExtra.copy(srcPath, destPath, {
errorOnExist: true,
Expand All @@ -249,12 +270,12 @@ async function copyAppDirectory(options: CopyAppDirectoryOptions) {
);
}

async function assertAppDirectoryContainsEntrypoint(
appDirectory: string,
async function assertAppDirContainsEntrypoint(
appDir: string,
entrypoint: string
) {
try {
await fsPromises.access(path.resolve(appDirectory, entrypoint));
await fsPromises.access(path.resolve(appDir, entrypoint));
} catch {
throw new Error(
`The entrypoint file "${entrypoint}" is not included in the bundled files.`
Expand All @@ -281,42 +302,6 @@ async function writePrebuiltPackagesTxt(
});
}

interface DownloadPrebuiltPackageWheelsOptions {
packages: string[];
destDir: string;
}
async function downloadPrebuiltPackageWheels(
options: DownloadPrebuiltPackageWheelsOptions
) {
const prebuiltPackagesData = await PrebuiltPackagesData.getInstance();
const usedPrebuiltPackages = options.packages.map((pkgName) =>
prebuiltPackagesData.getPackageInfoByName(pkgName)
);
const usedPrebuiltPackageUrls = usedPrebuiltPackages.map((pkg) =>
makePyodideUrl(pkg.file_name)
);

logger.info("Downloading the used prebuilt packages...");
await Promise.all(
usedPrebuiltPackageUrls.map(async (pkgUrl) => {
const dstPath = path.resolve(
options.destDir,
"./pyodide",
path.basename(pkgUrl)
);
logger.debug(`Download ${pkgUrl} to ${dstPath}`);
const res = await fetch(pkgUrl);
if (!res.ok) {
throw new Error(
`Failed to download ${pkgUrl}: ${res.status} ${res.statusText}`
);
}
const buf = await res.arrayBuffer();
await fsPromises.writeFile(dstPath, Buffer.from(buf));
})
);
}

yargs(hideBin(process.argv))
.command(
"* [appHomeDirSource] [packages..]",
Expand Down Expand Up @@ -355,6 +340,18 @@ yargs(hideBin(process.argv))
alias: "k",
describe: "Keep the existing build directory contents except appHomeDir.",
})
.options("pyodideSource", {
type: "string",
describe:
"The base URL or path of the Pyodide files to download or copy, such as the prebuild package wheels and pyodide-lock.json",
default: `https://cdn.jsdelivr.net/pyodide/v${pyodideVersion}/full/`,
coerce: (urlOrPath) => {
if (!urlOrPath.endsWith("/")) {
urlOrPath += "/";
}
return urlOrPath;
},
})
.options("logLevel", {
type: "string",
default: "info",
Expand Down Expand Up @@ -406,27 +403,31 @@ yargs(hideBin(process.argv))
]);
logger.info("Validated dependency list: %j", dependencies);

const usedPrebuiltPackages = await inspectUsedPrebuiltPackages({
await copyBuildDirectory({ copyTo: destDir, keepOld: args.keepOldBuild });

const usedPrebuiltPackages = await saveUsedPrebuiltPackages({
pyodideSource: args.pyodideSource,
pyodideRuntimeDir: path.resolve(destDir, "./pyodide"),
requirements: dependencies,
});
logger.info(
"The prebuilt packages loaded for the given requirements: %j",
usedPrebuiltPackages
);

await copyBuildDirectory({ copyTo: destDir, keepOld: args.keepOldBuild });

const buildAppDirectory = path.resolve(destDir, "./app_files"); // This path will be loaded in the `readStreamlitAppDirectory` handler in electron/main.ts.
const destAppDir = path.resolve(destDir, "./app_files"); // This path will be loaded in the `readStreamlitAppDirectory` handler in electron/main.ts.
await copyAppDirectory({
cwd: projectDir,
filePathPatterns: config.files,
buildAppDirectory,
destAppDir,
});
assertAppDirectoryContainsEntrypoint(buildAppDirectory, config.entrypoint);
assertAppDirContainsEntrypoint(destAppDir, config.entrypoint);

await createSitePackagesSnapshot({
requirements: dependencies,
usedPrebuiltPackages,
pyodideSource: args.pyodideSource,
pyodideRuntimeDir: path.resolve(destDir, "./pyodide"),
saveTo: path.resolve(destDir, "./site-packages-snapshot.tar.gz"), // This path will be loaded in the `readSitePackagesSnapshot` handler in electron/main.ts.
});
// These prebuilt packages will be vendored in the build artifact by `downloadPrebuiltPackageWheels()`
Expand All @@ -439,10 +440,6 @@ yargs(hideBin(process.argv))
path.resolve(destDir, "./prebuilt-packages.txt"), // This path will be loaded in the `readRequirements` handler in electron/main.ts.
usedPrebuiltPackages
);
await downloadPrebuiltPackageWheels({
packages: usedPrebuiltPackages,
destDir,
});
await dumpManifest({
packageJsonStliteDesktopField: packageJson.stlite?.desktop,
manifestFilePath: path.resolve(destDir, "./stlite-manifest.json"),
Expand Down
69 changes: 44 additions & 25 deletions packages/desktop/bin-src/dump_artifacts/pyodide_packages.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { makePyodideUrl } from "./url";
import path from "node:path";
import fsPromises from "node:fs/promises";
import { logger } from "./logger";

interface PackageInfo {
Expand All @@ -7,39 +8,57 @@ interface PackageInfo {
file_name: string;
depends: string[];
}
export class PrebuiltPackagesData {
private static _instance: PrebuiltPackagesData;
export class PrebuiltPackagesDataReader {
private sourceUrl: string;
private isRemote: boolean;
private _data: Record<string, PackageInfo> | null = null;

private constructor() {}

private static async loadPrebuiltPackageData(): Promise<
Record<string, PackageInfo>
> {
const url = makePyodideUrl("pyodide-lock.json");
constructor(sourceUrl: string) {
// These path logics are based on https://github.com/pyodide/pyodide/blob/0.25.1/src/js/compat.ts#L122
if (sourceUrl.startsWith("file://")) {
// handle file:// with filesystem operations rather than with fetch.
sourceUrl = sourceUrl.slice("file://".length);
}
this.sourceUrl = sourceUrl;
this.isRemote = sourceUrl.includes("://");
}

logger.info(`Load the Pyodide pyodide-lock.json from ${url}`);
const res = await fetch(url, undefined);
const resJson = await res.json();
private async readJson(filepath: string): Promise<any> {
const url = path.join(this.sourceUrl, filepath);

return resJson.packages;
if (this.isRemote) {
logger.debug(`Fetching ${url}`);
const res = await fetch(url);
if (!res.ok) {
throw new Error(
`Failed to download ${url}: ${res.status} ${res.statusText}`
);
}
return await res.json();
} else {
logger.debug(`Reading ${url}`);
const buf = await fsPromises.readFile(url);
return JSON.parse(buf.toString());
}
}

static async getInstance(): Promise<PrebuiltPackagesData> {
if (this._instance == null) {
this._instance = new PrebuiltPackagesData();
this._instance._data = await this.loadPrebuiltPackageData();
private async loadPrebuiltPackageData(): Promise<
Record<string, PackageInfo>
> {
if (this._data != null) {
return this._data;
}
return this._instance;

logger.info(`Load pyodide-lock.json`);
const lockJson = await this.readJson("pyodide-lock.json");

this._data = lockJson.packages;
return lockJson.packages;
}

public getPackageInfoByName(pkgName: string): PackageInfo {
if (this._data == null) {
throw new Error("The package data is not loaded yet.");
}
const pkgInfo = Object.values(this._data).find(
(pkg) => pkg.name === pkgName
);
public async getPackageInfoByName(pkgName: string): Promise<PackageInfo> {
const data = await this.loadPrebuiltPackageData();
const pkgInfo = Object.values(data).find((pkg) => pkg.name === pkgName);
if (pkgInfo == null) {
throw new Error(`Package ${pkgName} is not found in the lock file.`);
}
Expand Down
5 changes: 0 additions & 5 deletions packages/desktop/bin-src/dump_artifacts/url.ts
View file

This file was deleted.

0 comments on commit b6edd77

Please sign in to comment.