Add a stub Rust crate

.github/workflows/release-artifacts.yml

@@ -15,7 +15,7 @@ on:
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
-  
+
 permissions:
   contents: write
 
@@ -91,7 +91,34 @@ jobs:
 
   build-sdist:
     name: "Build pypi distribution files"
-    uses: "matrix-org/backend-meta/.github/workflows/packaging.yml@v1"
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+
+      - name: Install tools
+        run: python -m pip install build twine
+
+      - name: Build wheel and sdist
+        run: poetry build
+
+      # This checks that the "long description" fields renders correctly on PyPI.
+      # We don't upload to PyPI, but the two steps below only upload artifacts to GitHub.
+      - name: Run twine check
+        run: python -m twine check dist/*
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: Wheel
+          path: dist/*.whl
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: Sdist
+          path: dist/*.tar.gz
+
 
   # if it's a tag, create a release and attach the artifacts to it
   attach-assets:

.gitignore

@@ -60,3 +60,10 @@ book/
 # complement
 /complement-*
 /master.tar.gz
+
+# rust
+/target/
+/synapse/*.so
+
+# Poetry will create a setup.py, which we don't want to include.
+/setup.py

Cargo.toml

@@ -0,0 +1,5 @@
+# We make the whole Synapse folder a workspace so that we can run `cargo`
+# commands from the root (rather than having to cd into rust/).
+
+[workspace]
+members = ["rust"]

build.py

@@ -0,0 +1,16 @@
+# A build script for poetry that adds the rust extension.
+
+import os
+
+from setuptools_rust import Binding, RustExtension
+
+
+def build(setup_kwargs):
+    original_project_dir = os.path.dirname(os.path.realpath(__file__))
+    cargo_toml_path = os.path.join(original_project_dir, "rust", "Cargo.toml")
+
+    extension = RustExtension(
+        target="synapse.synapse_rust", path=cargo_toml_path, binding=Binding.PyO3
+    )
+    setup_kwargs.setdefault("rust_extensions", []).append(extension)
+    setup_kwargs["zip_safe"] = False

changelog.d/12595.misc

@@ -0,0 +1 @@
+Add a stub Rust crate.

docker/Dockerfile

@@ -104,6 +104,7 @@ RUN \
     rustc \
     zlib1g-dev \
     git \
+    rustc cargo \
     && rm -rf /var/lib/apt/lists/*
 
 # To speed up rebuilds, install all of the dependencies before we copy over

docker/Dockerfile-dhvirtualenv

@@ -28,12 +28,12 @@ FROM ${distro} as builder
 
 RUN apt-get update -qq -o Acquire::Languages=none
 RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
-        -yqq --no-install-recommends \
-        build-essential \
-        ca-certificates \
-        devscripts \
-        equivs \
-        wget
+    -yqq --no-install-recommends \
+    build-essential \
+    ca-certificates \
+    devscripts \
+    equivs \
+    wget
 
 # fetch and unpack the package
 RUN mkdir /dh-virtualenv
@@ -70,20 +70,21 @@ ENV LANG C.UTF-8
 # TODO: it would be nice to do that automatically.
 RUN apt-get update -qq -o Acquire::Languages=none \
     && env DEBIAN_FRONTEND=noninteractive apt-get install \
-        -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
-        build-essential \
-        debhelper \
-        devscripts \
-        libsystemd-dev \
-        lsb-release \
-        pkg-config \
-        python3-dev \
-        python3-pip \
-        python3-setuptools \
-        python3-venv \
-        sqlite3 \
-        libpq-dev \
-        xmlsec1
+    -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
+    build-essential \
+    debhelper \
+    devscripts \
+    libsystemd-dev \
+    lsb-release \
+    pkg-config \
+    python3-dev \
+    python3-pip \
+    python3-setuptools \
+    python3-venv \
+    sqlite3 \
+    libpq-dev \
+    xmlsec1 \
+    rustc cargo
 
 COPY --from=builder /dh-virtualenv_1.2.2-1_all.deb /
 

pyproject.toml

@@ -82,8 +82,13 @@ include = [
     { path = "sytest-blacklist", format = "sdist" },
     { path = "tests", format = "sdist" },
     { path = "UPGRADE.rst", format = "sdist" },
+    { path = "rust/Cargo.toml", format = "sdist" },
+    { path = "rust/Cargo.lock", format = "sdist" },
+    { path = "rust/src/**", format = "sdist" },
 ]
 
+build = "build.py"
+
 [tool.poetry.scripts]
 synapse_homeserver = "synapse.app.homeserver:main"
 synapse_worker = "synapse.app.generic_worker:main"
@@ -284,6 +289,17 @@ twine = "*"
 # Towncrier min version comes from #3425. Rationale unclear.
 towncrier = ">=18.6.0rc1"
 
+# This is for building the rust components during "poetry install", which
+# currently ignores the `build-system.requires` directive (c.f.
+# https://github.com/python-poetry/poetry/issues/6154). Both `pip install` and
+# `poetry build` do the right thing without this explicit dependency.
+#
+# This isn't really a dev-dependency, as `poetry install --no-dev` will fail,
+# but the alternative is to add it to the main list of deps where it isn't
+# needed.
+setuptools_rust = ">=1.3"
+setuptools = ">=62.4"
+
 [build-system]
-requires = ["poetry-core>=1.0.0"]
+requires = ["poetry-core>=1.0.0", "setuptools_rust>=1.3"]
 build-backend = "poetry.core.masonry.api"

rust/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "synapse-rust"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+name = "synapse_rust"
+crate-type = ["cdylib"]
+
+[dependencies]
+pyo3 = { version = "0.16.5", features = ["extension-module", "macros"] }

rust/src/lib.rs

@@ -0,0 +1,17 @@
+use pyo3::prelude::*;
+
+/// Formats the sum of two numbers as string.
+#[pyfunction]
+#[pyo3(text_signature = "(a, b, /)")]
+fn sum_as_string(a: usize, b: usize) -> PyResult<String> {
+    Ok((a + b).to_string())
+}
+
+/// A Python module implemented in Rust. The name of this function must match
+/// the `lib.name` setting in the `Cargo.toml`, else Python will not be able to
+/// import the module.
+#[pymodule]
+fn synapse_rust(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
+    m.add_function(wrap_pyfunction!(sum_as_string, m)?)?;
+    Ok(())
+}

scripts-dev/build_wheels.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -ex
+
+cd $(dirname "$0")/..
+
+docker run --rm -it -v $( pwd ):/io quay.io/pypa/manylinux2014_x86_64 bash /io/scripts-dev/build_wheels_inner.sh

scripts-dev/build_wheels_inner.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Builds wheels when run in a manylinux container
+
+set -ex
+
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+export PATH="$HOME/.cargo/bin:$PATH"
+
+# Compile wheels
+for PYBIN in /opt/python/cp{37,38,39,310}*/bin; do
+    rm -rf /io/build/
+
+    # Until https://github.com/python-poetry/poetry/pull/5401 lands we need to
+    # manually install setuptools_rust
+    rm -rf /tmp/venv
+    "${PYBIN}/python" -m venv /tmp/venv
+    source /tmp/venv/bin/activate
+
+    pip install -U poetry setuptools_rust
+    cd /io
+    poetry build -f wheel
+done
+
+# Bundle external shared libraries into the wheels
+for whl in /io/dist/*{cp37,cp38,cp39,cp310}*.whl; do
+    auditwheel repair "$whl" -w /io/dist/
+done
+
+# Install packages and test
+for PYBIN in /opt/python/cp{37,38,39,310}*/bin; do
+    "${PYBIN}/pip" install synapse -f /io/dist/
+done

stubs/synapse/synapse_rust.pyi

@@ -0,0 +1 @@
+def sum_as_string(a: int, b: int) -> str: ...

tests/test_rust.py

@@ -0,0 +1,9 @@
+from tests import unittest
+
+
+class RustTestCase(unittest.TestCase):
+    def test_basic(self):
+        from synapse.synapse_rust import sum_as_string
+
+        result = sum_as_string(1, 2)
+        self.assertEqual("3", result)