Improvements to Emscripten/Pyodide CI

.github/workflows/test.yml

@@ -153,6 +153,23 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      - run: |
+          PYODIDE_VERSION=0.21.0-alpha.2
+
+          cd tests
+          npm i pyodide@0.21.0-alpha.2 prettier
+          cd node_modules/pyodide/
+          node ../prettier/bin-prettier.js -w pyodide.asm.js
+          EMSCRIPTEN_VERSION=$(node -p "require('./repodata.json').info.platform.split('_').slice(1).join('.')")
+          PYTHON_VERSION=3.10.2
+
+          echo "PYODIDE_VERSION=$PYODIDE_VERSION" >> $GITHUB_ENV
+          echo "EMSCRIPTEN_VERSION=$EMSCRIPTEN_VERSION" >> $GITHUB_ENV
+          echo "PYTHON_VERSION=$PYTHON_VERSION" >> $GITHUB_ENV
+          echo "ORIG_PATH=$PATH" >> $GITHUB_ENV
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
@@ -162,15 +179,12 @@ jobs:
           override: true
       - uses: mymindstorm/setup-emsdk@v11
         with:
-          version: 3.1.13
+          version: ${{env.EMSCRIPTEN_VERSION}}
           actions-cache-folder: emsdk-cache
       - uses: actions/setup-python@v2
         id: setup-python
         with:
-          python-version: "3.10"
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 18
+          python-version: ${{env.PYTHON_VERSION}}
       - run: pip install nox
       - uses: actions/cache@v3
         with:
@@ -179,7 +193,9 @@ jobs:
           key: ${{ hashFiles('tests/*.js') }} - ${{ hashFiles('noxfile.py') }} - ${{ steps.setup-python.outputs.python-path }}
       - uses: Swatinem/rust-cache@v1
       - name: Run tests
-        run: nox -s test-emscripten
+        run: |
+          export PATH=$ORIG_PATH:$PATH
+          nox -s test-emscripten
 
   test-alpine:
     name: Test Alpine Linux

.gitignore

@@ -14,4 +14,4 @@ tags
 test-crates/wheels/
 test-crates/targets/
 test-crates/venvs/
-tests/pyodide/
+node_modules

noxfile.py

@@ -1,41 +1,19 @@
-import sys
 from pathlib import Path
 
 import nox
-
-
-def download_pyodide(session: nox.Session, pyodide_dir: Path) -> None:
-    pyodide_dir.mkdir()
-
-    PYODIDE_DEV = "https://pyodide-cdn2.iodide.io/dev/full/"
-    pyodide_files = [
-        "pyodide.js",
-        "repodata.json",
-        "pyodide.asm.js",
-        "pyodide.asm.data",
-        "pyodide.asm.wasm",
-        "pyodide_py.tar",
-    ]
-    with session.chdir(pyodide_dir):
-        for file in pyodide_files:
-            session.run("wget", "-q", PYODIDE_DEV + file, external=True)
-        session.run("npm", "i", "node-fetch", external=True)
+import sys
 
 
 @nox.session(name="test-emscripten")
 def test_emscripten(session: nox.Session):
     emscripten_dir = Path("./tests").resolve()
-    pyodide_dir = emscripten_dir / "pyodide"
-    if not pyodide_dir.exists():
-        download_pyodide(session, pyodide_dir)
 
     test_crates = [
         "test-crates/pyo3-pure",
         "test-crates/pyo3-mixed",
     ]
     for crate in test_crates:
         crate = Path(crate).resolve()
-
         ver = sys.version_info
         session.run(
             "cargo",

tests/emscripten_runner.js

@@ -1,36 +1,5 @@
-const http = require("http");
-const fs = require("fs");
 const { opendir } = require("node:fs/promises");
-
-const { loadPyodide } = require("./pyodide/pyodide.js");
-
-const PORT = 8124;
-
-const server = http
-  .createServer(function (request, response) {
-    const filePath = "." + request.url;
-    const contentType = "application/octet-stream";
-    fs.readFile(
-      process.argv[2] + "/target/wheels/" + filePath,
-      function (error, content) {
-        if (error) {
-          if (error.code == "ENOENT") {
-            response.writeHead(404);
-            response.end("Not found");
-            response.end();
-          } else {
-            response.writeHead(500);
-            response.end("error: " + error.code);
-            response.end();
-          }
-        } else {
-          response.writeHead(200, { "Content-Type": contentType });
-          response.end(content, "utf-8");
-        }
-      }
-    );
-  })
-  .listen(PORT);
+const { loadPyodide } = require("pyodide");
 
 async function findWheel(distDir) {
   const dir = await opendir(distDir);
@@ -41,19 +10,16 @@ async function findWheel(distDir) {
   }
 }
 
-const localhost = `http://0.0.0.0:${PORT}`;
 const pkgDir = process.argv[2];
 const distDir = pkgDir + "/target/wheels";
 const testDir = pkgDir + "/tests";
 
 async function main() {
   const wheelName = await findWheel(distDir);
-  const wheelURL = `${localhost}/${wheelName}`;
+  const wheelURL = `file:${distDir}/${wheelName}`;
 
-  let errcode = 1;
   try {
-    pyodide = await loadPyodide({ indexURL: "./pyodide", fullStdLib: false });
-    pyodide._api.setCdnUrl("https://pyodide-cdn2.iodide.io/dev/full/");
+    pyodide = await loadPyodide();
     const FS = pyodide.FS;
     const NODEFS = FS.filesystems.NODEFS;
     FS.mkdir("/test_dir");
@@ -65,11 +31,8 @@ async function main() {
     errcode = pytest.main(pyodide.toPy(["/test_dir", "-vv"]));
   } catch (e) {
     console.error(e);
-    errcode = 1;
-  } finally {
-    server.close();
+    process.exit(1);
   }
-  process.exit(errcode);
 }
 
 main();