Add CI tests in Pyodide

Author: hoodmane

.github/workflows/ci.yml

@@ -3,6 +3,7 @@ on:
   push:
     branches:
       - main
+      - emscripten-ci2
   pull_request:
 
 concurrency:
@@ -350,3 +351,52 @@ jobs:
           CIBW_BUILD_VERBOSITY: 1
         with:
           package-dir: examples/namespace_package
+
+  test-emscripten:
+    name: Test Emscripten
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      - run: |
+          PYODIDE_VERSION=0.21.0-alpha.2
+
+          cd emscripten
+          npm i pyodide@0.21.0-alpha.2 prettier
+          cd node_modules/pyodide/
+          node ../prettier/bin-prettier.js -w pyodide.asm.js
+          EMSCRIPTEN_VERSION=$(node -p "require('./repodata.json').info.platform.split('_').slice(1).join('.')")
+          PYTHON_VERSION=3.10.2
+
+          echo "PYODIDE_VERSION=$PYODIDE_VERSION" >> $GITHUB_ENV
+          echo "EMSCRIPTEN_VERSION=$EMSCRIPTEN_VERSION" >> $GITHUB_ENV
+          echo "PYTHON_VERSION=$PYTHON_VERSION" >> $GITHUB_ENV
+          echo "ORIG_PATH=$PATH" >> $GITHUB_ENV
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+          components: rust-src
+          target: wasm32-unknown-emscripten
+          override: true
+      - uses: mymindstorm/setup-emsdk@v11
+        with:
+          version: ${{env.EMSCRIPTEN_VERSION}}
+          actions-cache-folder: emsdk-cache
+      - uses: actions/setup-python@v2
+        id: setup-python
+        with:
+          python-version: ${{env.PYTHON_VERSION}}
+      - run: pip install nox
+      - uses: actions/cache@v3
+        with:
+          path: |
+            tests/pyodide
+          key: ${{ hashFiles('tests/*.js') }} - ${{ hashFiles('noxfile.py') }} - ${{ steps.setup-python.outputs.python-path }}
+      - uses: Swatinem/rust-cache@v1
+      - name: Test
+        run: |
+          export PATH=$ORIG_PATH:$PATH
+          nox -s test-examples-emscripten

emscripten/.gitignore

@@ -0,0 +1,6 @@
+builddir
+main.*
+!main.c
+pybuilddir.txt
+pyodide
+node_modules

emscripten/_sysconfigdata__emscripten_wasm32-emscripten.py

@@ -0,0 +1,16 @@
+# system configuration generated and used by the sysconfig module
+build_time_vars = {
+    "ABIFLAGS": "",
+    "AR": "/src/emsdk/emsdk/upstream/emscripten/emar",
+    "ARFLAGS": "rcs",
+    "BLDSHARED": "emcc -sSIDE_MODULE=1 -L/src/emscripten/python-lib/",
+    "CC": "emcc -I/src/emscripten/python-include/",
+    "CCSHARED": "",
+    "CFLAGS": "-Wno-unused-result -Wsign-compare -Wunreachable-code -DNDEBUG -g "
+    "-fwrapv -O3 -Wall -O2 -g0 -fPIC",
+    "EXT_SUFFIX": ".cpython-310-wasm32-emscripten.so",
+    "HOST_GNU_TYPE": "wasm32-unknown-emscripten",
+    "LDSHARED": "emcc -sSIDE_MODULE=1",
+    "Py_DEBUG": "0",
+    "py_version_nodot": "310",
+}

emscripten/emcc_wrapper.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+import subprocess
+import sys
+
+
+def update_args(args):
+    # remove -lc. Not sure if it makes a difference but -lc doesn't belong here.
+    # https://github.com/emscripten-core/emscripten/issues/17191
+    for i in reversed(range(len(args))):
+        if args[i] == "c" and args[i - 1] == "-l":
+            del args[i - 1 : i + 1]
+
+    return args
+
+
+def main(args):
+    args = update_args(args)
+    return subprocess.call(["emcc"] + args)
+
+
+if __name__ == "__main__":
+    args = sys.argv[1:]
+    sys.exit(main(args))

emscripten/pyo3_config.ini

@@ -0,0 +1,7 @@
+implementation=CPython
+version=3.10
+shared=true
+abi3=false
+lib_name=python3.10
+pointer_width=32
+suppress_build_script_link_lines=false

emscripten/runner.js

@@ -0,0 +1,38 @@
+const { opendir } = require("node:fs/promises");
+const { loadPyodide } = require("pyodide");
+
+async function findWheel(distDir) {
+  const dir = await opendir(distDir);
+  for await (const dirent of dir) {
+    if (dirent.name.endsWith("whl")) {
+      return dirent.name;
+    }
+  }
+}
+
+const pkgDir = process.argv[2];
+const distDir = pkgDir + "/dist";
+const testDir = pkgDir + "/tests";
+
+async function main() {
+  const wheelName = await findWheel(distDir);
+  const wheelURL = `file:${distDir}/${wheelName}`;
+
+  try {
+    pyodide = await loadPyodide();
+    const FS = pyodide.FS;
+    const NODEFS = FS.filesystems.NODEFS;
+    FS.mkdir("/test_dir");
+    FS.mount(NODEFS, { root: testDir }, "/test_dir");
+    await pyodide.loadPackage(["micropip", "pytest", "tomli"]);
+    const micropip = pyodide.pyimport("micropip");
+    await micropip.install(wheelURL);
+    const pytest = pyodide.pyimport("pytest");
+    errcode = pytest.main(pyodide.toPy(["/test_dir", "-vv"]));
+  } catch (e) {
+    console.error(e);
+    process.exit(1);
+  }
+}
+
+main();

examples/html-py-ever/tests/conftest.py

@@ -0,0 +1,11 @@
+import sys
+import pytest
+
+if sys.platform == "emscripten":
+
+    @pytest.fixture
+    def benchmark():
+        def result(func, *args, **kwargs):
+            return func(*args, **kwargs)
+
+        return result

noxfile.py

@@ -1,10 +1,10 @@
+import nox
 import os
+import sys
 import tarfile
 from glob import glob
 from pathlib import Path
 
-import nox
-
 
 @nox.session(name="test-examples", venv_backend="none")
 def test_examples(session: nox.Session):
@@ -35,3 +35,51 @@ def mypy(session: nox.Session):
 def test(session: nox.Session):
     session.install("pytest", ".")
     session.run("pytest", "setuptools_rust", *session.posargs)
+
+
+@nox.session(name="test-examples-emscripten")
+def test_examples_emscripten(session: nox.Session):
+    session.install(".")
+    emscripten_dir = Path("./emscripten").resolve()
+
+    session.run(
+        "rustup",
+        "component",
+        "add",
+        "rust-src",
+        "--toolchain",
+        "nightly",
+        external=True,
+    )
+    examples_dir = Path("examples")
+    test_crates = [
+        examples_dir / "html-py-ever",
+        examples_dir / "namespace_package",
+    ]
+    for example in test_crates:
+        env = os.environ.copy()
+        env.update(
+            RUSTUP_TOOLCHAIN="nightly",
+            _SETUPTOOLSRUST_BUILD_STD="1",
+            PYTHONPATH=str(emscripten_dir),
+            _PYTHON_SYSCONFIGDATA_NAME="_sysconfigdata__emscripten_wasm32-emscripten",
+            _PYTHON_HOST_PLATFORM="emscripten_3_1_14_wasm32",
+            CARGO_BUILD_TARGET="wasm32-unknown-emscripten",
+            CARGO_UNSTABLE_BUILD_STD="true",
+            CARGO_TARGET_WASM32_UNKNOWN_EMSCRIPTEN_LINKER=str(
+                emscripten_dir / "emcc_wrapper.py"
+            ),
+            PYO3_CONFIG_FILE=str(emscripten_dir / "pyo3_config.ini"),
+            RUSTFLAGS=" ".join(
+                [
+                    "-C relocation-model=pic",
+                    "-C link-arg=-sSIDE_MODULE=2",
+                    "-C link-arg=-sWASM_BIGINT",
+                ]
+            ),
+        )
+        with session.chdir(example):
+            session.run("python", "setup.py", "bdist_wheel", env=env, external=True)
+
+        with session.chdir(emscripten_dir):
+            session.run("node", "runner.js", str(example), external=True)