From 5e94533b85e1e843f659dbaeb35899fdabfd4909 Mon Sep 17 00:00:00 2001 From: Tom Jakubowski Date: Thu, 24 Oct 2024 17:17:18 -0700 Subject: [PATCH] Generate sdist by hand This updates the perspective-python build to create an sdist tarball by hand, not going through `maturin sdist`. This bypasses some issues we have seen with Maturin mispackaging the perspective Cargo workspace in the sdist. Maturin is still used as the `build-backend` to build a wheel from the sdist. CI still uses `maturin build` to build wheels. Local development builds still use `maturin develop`. The sdist now includes the .data directory so that Maturin will place it in the wheel correctly. Building an sdist with PSP_BUILD_SDIST=1 will now error if it appears the jupyterlab plugin was not built into the .data directory. Most of the work is in generating a PKG-INFO file. Its output matches what's in the 3.1.2 sdist on pypi, minus a correction to fix the `Home-page` field miscapitalized by Maturin and some unimportant whitespace changes in `Require-Dist`. Signed-off-by: Tom Jakubowski --- .cargo/config.toml | 3 + .gitignore | 1 + packages/perspective-jupyterlab/build.mjs | 7 +- pnpm-lock.yaml | 100 ++++++++++++++++++++++ rust/perspective-python/Cargo.toml | 7 +- rust/perspective-python/build.mjs | 97 ++++++++++++++++++--- rust/perspective-python/package.json | 4 +- 7 files changed, 201 insertions(+), 18 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 486fe209d8..932b619b2a 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -14,6 +14,9 @@ rustflags = ["-C", "target-feature=+crt-static", "--cfg=web_sys_unstable_apis"] [target.x86_64-pc-windows-msvc] rustflags = ["-C", "target-feature=+crt-static", "--cfg=web_sys_unstable_apis"] +[patch.crates-io] +perspective-client = { path = "rust/perspective-client" } +perspective-server = { path = "rust/perspective-server" } [future-incompat-report] frequency = 'never' diff --git a/.gitignore b/.gitignore index 0a1e58ad72..7daa0674bc 100644 --- a/.gitignore +++ b/.gitignore @@ -246,6 +246,7 @@ rust/perspective-server/cmake .pyodide-*/ rust/perspective-python/*.data +rust/perspective-python/PKG-INFO # rust/perspective-python/perspective.data/data/share/jupyter/labextensions/@finos/perspective-jupyterlab/package.json rust/perspective-viewer/docs/exprtk.md rust/perspective-server/docs/lib_gen.md diff --git a/packages/perspective-jupyterlab/build.mjs b/packages/perspective-jupyterlab/build.mjs index 6bafccf6f6..6f30594ed1 100644 --- a/packages/perspective-jupyterlab/build.mjs +++ b/packages/perspective-jupyterlab/build.mjs @@ -138,11 +138,8 @@ async function build_all() { x.replace("-", "").replace(".", "") ); - const psp_dir = `perspective_python-${version}.data`; - await cpy( - ["dist/cjs/**/*"], - `../../rust/perspective-python/${psp_dir}/data/share/jupyter/labextensions/@finos/perspective-jupyterlab` - ); + const labext_dest = `../../rust/perspective-python/perspective_python-${version}.data/data/share/jupyter/labextensions/@finos/perspective-jupyterlab`; + await cpy(["dist/cjs/**/*"], labext_dest); } build_all(); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2e55bf093f..b7455ff024 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -829,6 +829,12 @@ importers: cpy: specifier: ^9.0.1 version: 9.0.1 + glob: + specifier: ^11 + version: 11.0.0 + tar: + specifier: ^7.4.3 + version: 7.4.3 rust/perspective-viewer: dependencies: @@ -2281,6 +2287,10 @@ packages: resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} + '@isaacs/fs-minipass@4.0.1': + resolution: {integrity: sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==} + engines: {node: '>=18.0.0'} + '@jest/schemas@29.6.3': resolution: {integrity: sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -3674,6 +3684,10 @@ packages: resolution: {integrity: sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==} engines: {node: '>=10'} + chownr@3.0.0: + resolution: {integrity: sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==} + engines: {node: '>=18'} + chroma-js@1.4.1: resolution: {integrity: sha512-jTwQiT859RTFN/vIf7s+Vl/Z2LcMrvMv3WUFmd/4u76AdlFC0NTNgqEEFPcRiHmAswPsMiQEDZLM8vX8qXpZNQ==} @@ -5083,6 +5097,10 @@ packages: glob-to-regexp@0.4.1: resolution: {integrity: sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==} + glob@10.4.5: + resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==} + hasBin: true + glob@11.0.0: resolution: {integrity: sha512-9UiX/Bl6J2yaBbxKoEBRm4Cipxgok8kQYcOPEhScPwebu2I0HoQOuYdIO6S3hLuWoZgpDpwQZMzTFxgpkyT76g==} engines: {node: 20 || >=22} @@ -5718,6 +5736,9 @@ packages: isomorphic.js@0.2.5: resolution: {integrity: sha512-PIeMbHqMt4DnUP3MA/Flc0HElYjMXArsw1qwJZcm9sqR8mq3l8NYizFMty0pWwE/tzIGH3EKK5+jes5mAr85yw==} + jackspeak@3.4.3: + resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} + jackspeak@4.0.1: resolution: {integrity: sha512-cub8rahkh0Q/bw1+GxP7aeSe29hHHn2V4m29nnDlvCdlgU+3UGxkZp7Z53jLUdpX3jdTO0nJZUDl3xvbWc2Xog==} engines: {node: 20 || >=22} @@ -5992,6 +6013,9 @@ packages: resolution: {integrity: sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + lru-cache@10.4.3: + resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + lru-cache@11.0.1: resolution: {integrity: sha512-CgeuL5uom6j/ZVrg7G/+1IXqRY8JXX4Hghfy5YE0EhoYQWvndP1kufu58cmZLNIDKnRhZrXfdS9urVWx98AipQ==} engines: {node: 20 || >=22} @@ -6336,6 +6360,10 @@ packages: resolution: {integrity: sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==} engines: {node: '>=10'} + minimatch@9.0.5: + resolution: {integrity: sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==} + engines: {node: '>=16 || 14 >=14.17'} + minimist-options@4.1.0: resolution: {integrity: sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==} engines: {node: '>= 6'} @@ -6359,6 +6387,10 @@ packages: resolution: {integrity: sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==} engines: {node: '>= 8'} + minizlib@3.0.1: + resolution: {integrity: sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==} + engines: {node: '>= 18'} + mitt@3.0.1: resolution: {integrity: sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==} @@ -6710,6 +6742,10 @@ packages: path-parse@1.0.7: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} + path-scurry@1.11.1: + resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==} + engines: {node: '>=16 || 14 >=14.18'} + path-scurry@2.0.0: resolution: {integrity: sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg==} engines: {node: 20 || >=22} @@ -7570,6 +7606,10 @@ packages: deprecated: Rimraf versions prior to v4 are no longer supported hasBin: true + rimraf@5.0.10: + resolution: {integrity: sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==} + hasBin: true + rimraf@6.0.1: resolution: {integrity: sha512-9dkvaxAsk/xNXSJzMgFqqMCuFgt2+KsOFek3TMLfo8NCPfWpBmqwyNn5Y+NX56QUYfCtsyhF3ayiboEoUmJk/A==} engines: {node: 20 || >=22} @@ -8064,6 +8104,10 @@ packages: resolution: {integrity: sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==} engines: {node: '>=10'} + tar@7.4.3: + resolution: {integrity: sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==} + engines: {node: '>=18'} + terser-webpack-plugin@5.3.10: resolution: {integrity: sha512-BKFPWlPDndPs+NGGCr1U59t0XScL5317Y0UReNrHaw9/FwhPENlq6bfgs+4yPfyP51vqC1bQ4rp1EfXW5ZSH9w==} engines: {node: '>= 10.13.0'} @@ -8682,6 +8726,10 @@ packages: yallist@4.0.0: resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + yallist@5.0.0: + resolution: {integrity: sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==} + engines: {node: '>=18'} + yaml@1.10.2: resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==} engines: {node: '>= 6'} @@ -10905,6 +10953,10 @@ snapshots: wrap-ansi: 8.1.0 wrap-ansi-cjs: wrap-ansi@7.0.0 + '@isaacs/fs-minipass@4.0.1': + dependencies: + minipass: 7.1.2 + '@jest/schemas@29.6.3': dependencies: '@sinclair/typebox': 0.27.8 @@ -13018,6 +13070,8 @@ snapshots: chownr@2.0.0: {} + chownr@3.0.0: {} + chroma-js@1.4.1: {} chrome-trace-event@1.0.4: {} @@ -14654,6 +14708,15 @@ snapshots: glob-to-regexp@0.4.1: {} + glob@10.4.5: + dependencies: + foreground-child: 3.3.0 + jackspeak: 3.4.3 + minimatch: 9.0.5 + minipass: 7.1.2 + package-json-from-dist: 1.0.0 + path-scurry: 1.11.1 + glob@11.0.0: dependencies: foreground-child: 3.3.0 @@ -15375,6 +15438,12 @@ snapshots: isomorphic.js@0.2.5: {} + jackspeak@3.4.3: + dependencies: + '@isaacs/cliui': 8.0.2 + optionalDependencies: + '@pkgjs/parseargs': 0.11.0 + jackspeak@4.0.1: dependencies: '@isaacs/cliui': 8.0.2 @@ -15651,6 +15720,8 @@ snapshots: lowercase-keys@3.0.0: {} + lru-cache@10.4.3: {} + lru-cache@11.0.1: {} lru-cache@5.1.1: @@ -16276,6 +16347,10 @@ snapshots: dependencies: brace-expansion: 2.0.1 + minimatch@9.0.5: + dependencies: + brace-expansion: 2.0.1 + minimist-options@4.1.0: dependencies: arrify: 1.0.1 @@ -16297,6 +16372,11 @@ snapshots: minipass: 3.3.6 yallist: 4.0.0 + minizlib@3.0.1: + dependencies: + minipass: 7.1.2 + rimraf: 5.0.10 + mitt@3.0.1: {} mkdirp@0.5.6: @@ -16637,6 +16717,11 @@ snapshots: path-parse@1.0.7: {} + path-scurry@1.11.1: + dependencies: + lru-cache: 10.4.3 + minipass: 7.1.2 + path-scurry@2.0.0: dependencies: lru-cache: 11.0.1 @@ -17659,6 +17744,10 @@ snapshots: dependencies: glob: 7.2.3 + rimraf@5.0.10: + dependencies: + glob: 10.4.5 + rimraf@6.0.1: dependencies: glob: 11.0.0 @@ -18233,6 +18322,15 @@ snapshots: mkdirp: 1.0.4 yallist: 4.0.0 + tar@7.4.3: + dependencies: + '@isaacs/fs-minipass': 4.0.1 + chownr: 3.0.0 + minipass: 7.1.2 + minizlib: 3.0.1 + mkdirp: 3.0.1 + yallist: 5.0.0 + terser-webpack-plugin@5.3.10(esbuild@0.14.54)(webpack@5.94.0(esbuild@0.14.54)(webpack-cli@4.10.0(webpack@5.94.0))): dependencies: '@jridgewell/trace-mapping': 0.3.25 @@ -19003,6 +19101,8 @@ snapshots: yallist@4.0.0: {} + yallist@5.0.0: {} + yaml@1.10.2: {} yargs-parser@20.2.9: {} diff --git a/rust/perspective-python/Cargo.toml b/rust/perspective-python/Cargo.toml index 12ccacabc0..f2c3063cb3 100644 --- a/rust/perspective-python/Cargo.toml +++ b/rust/perspective-python/Cargo.toml @@ -52,9 +52,12 @@ pyo3-build-config = "0.20.2" python-config-rs = "0.1.2" [dependencies] +# NOTE: when building from the git repo, these perspective-* dependencies are +# overridden with path dependencies in .cargo/config.toml. This is done to +# support the sdist, which doesn't include these packages. +perspective-client = { version = "3.1.2" } +perspective-server = { version = "3.1.2" } async-lock = "2.5.0" -perspective-client = { version = "3.1.2", path = "../perspective-client" } -perspective-server = { version = "3.1.2", path = "../perspective-server" } pollster = "0.3.0" extend = "1.1.2" futures = "0.3.28" diff --git a/rust/perspective-python/build.mjs b/rust/perspective-python/build.mjs index 89812d1cc5..637ee41836 100644 --- a/rust/perspective-python/build.mjs +++ b/rust/perspective-python/build.mjs @@ -14,6 +14,9 @@ import * as fs from "node:fs"; import sh from "../../tools/perspective-scripts/sh.mjs"; import * as url from "url"; import * as TOML from "@iarna/toml"; +import * as tar from "tar"; +import * as glob from "glob"; +import * as path from "path"; const __dirname = url.fileURLToPath(new URL(".", import.meta.url)).slice(0, -1); const pkg = JSON.parse( @@ -111,22 +114,96 @@ if (build_wheel) { cmd.sh(`maturin build ${flags} --features=${features.join(",")} ${target}`); } -const old = fs.readFileSync("./pyproject.toml"); - if (build_sdist) { - const toml = TOML.parse(old); - console.log(toml); - delete toml.tool.maturin["data"]; - fs.writeFileSync("./pyproject.toml", TOML.stringify(toml)); - cmd.sh(`maturin sdist`); + // `maturin sdist` has some issues with Cargo workspaces, so we assemble the sdist by hand here + // Note that the resulting sdist is _not_ a Cargo workspace, it is rooted in this package. + const cargo_toml = fs.readFileSync("./Cargo.toml"); + const pyproject_toml = fs.readFileSync("./pyproject.toml"); + const cargo = TOML.parse(cargo_toml); + const pyproject = TOML.parse(pyproject_toml); + + const version = cargo["package"]["version"]; + const data_dir = `perspective_python-${version}.data`; + const testfile = path.join( + data_dir, + "data/share/jupyter/labextensions/@finos/perspective-jupyterlab/package.json" + ); + if (!fs.existsSync(testfile)) { + throw new Error( + "labextension is not present in data directory, please build `perspective-jupyterlab`" + ); + } + const readme_md = fs.readFileSync("./README.md"); + const pkg_info = generatePkgInfo(pyproject, cargo, readme_md); + fs.writeFileSync("./PKG-INFO", pkg_info); + const include_paths = Array.from(cargo["package"]["include"]).concat([ + data_dir, + "./PKG-INFO", + ]); + const files = glob.globSync(include_paths); + const wheel_dir = `../target/wheels`; + fs.mkdirSync(wheel_dir, { recursive: true }); + await tar.create( + { + gzip: true, + file: path.join(wheel_dir, `perspective_python-${version}.tar.gz`), + prefix: `perspective_python-${version}`, + }, + files + ); } if (!build_wheel && !build_sdist) { cmd.sh(`maturin develop ${flags}`); } -cmd.runSync(); +if (!cmd.isEmpty()) { + cmd.runSync(); +} -if (build_sdist) { - fs.writeFileSync("./pyproject.toml", old); +// Generates version 2.3 according to https://packaging.python.org/en/latest/specifications/core-metadata/ +// Takes parsed pyproject.toml, Cargo.toml, and contents of README.md. +function generatePkgInfo(pyproject, cargo, readme_md) { + const project = pyproject["project"]; + const field = (name, value) => { + if (typeof value !== "string") { + throw new Error( + `PKG-INFO value for field ${name} was not a string:\n${value}` + ); + } + return `${name}: ${value}`; + }; + const lines = []; + const addField = (key, value) => lines.push(field(key, value)); + addField("Metadata-Version", "2.3"); + addField("Name", project.name); + addField("Version", cargo.package.version); + for (const c of project["classifiers"]) { + addField("Classifier", c); + } + for (const [extra, deps] of Object.entries( + project["optional-dependencies"] + )) { + for (const dep of deps) { + addField("Requires-Dist", `${dep} ; extra == '${extra}'`); + } + } + for (const extra of Object.keys(project["optional-dependencies"])) { + addField("Provides-Extra", extra); + } + addField("Summary", cargo.package.description); + addField("Home-page", cargo.package.homepage); + addField("Author", cargo.package.authors[0]); + addField("Author-email", cargo.package.authors[0]); + addField("License", cargo.package.license); + addField("Requires-Python", project["requires-python"]); + addField( + "Description-Content-Type", + "text/markdown; charset=UTF-8; variant=GFM" + ); + addField("Project-URL", `Source Code, ${cargo.package.repository}`); + lines.push(""); + lines.push(readme_md); + + return lines.join("\n"); } diff --git a/rust/perspective-python/package.json b/rust/perspective-python/package.json index 356ee1b829..95685a4e09 100644 --- a/rust/perspective-python/package.json +++ b/rust/perspective-python/package.json @@ -17,7 +17,9 @@ "devDependencies": { "@finos/perspective-scripts": "workspace:*", "@iarna/toml": "^1.0.0-rc.1", - "cpy": "^9.0.1" + "glob": "^11", + "cpy": "^9.0.1", + "tar": "^7.4.3" }, "dependencies": { "@finos/perspective-jupyterlab": "workspace:*"