Linux ARM and Python 3.11 Support

.github/workflows/build-binaries.yml

@@ -0,0 +1,72 @@
+name: Build Binaries
+on:
+  push:
+    branches:
+      - main
+      - "releases/*"
+
+jobs:
+  # Compile the binaries and upload artifacts
+  compile-binaries:
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - os: ubuntu-latest
+            package-suffix: linux-amd64
+          - os: macos-latest
+            package-suffix: macos-amd64
+          - os: windows-latest
+            package-suffix: windows-amd64
+          - os: ubuntu-arm
+            package-suffix: linux-aarch64
+            # Need the 8 CPU version that has 12GB of RAM, the 4 CPU version
+            # only has 6 GB.
+            runsOn: buildjet-8vcpu-ubuntu-2204-arm
+    runs-on: ${{ matrix.runsOn || matrix.os }}
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      # actions/setup-python doesn't yet support ARM
+      - if: ${{ !endsWith(matrix.os, '-arm') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - if: ${{ matrix.os == 'ubuntu-arm' }}
+        uses: deadsnakes/action@v2.1.1
+        with:
+          python-version: "3.11"
+
+      # Install Rust locally for non-Linux (Linux uses an internal docker
+      # command to build with cibuildwheel which uses rustup install defined
+      # in pyproject.toml)
+      - if: ${{ runner.os != 'Linux' }}
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+      - if: ${{ runner.os != 'Linux' }}
+        uses: Swatinem/rust-cache@v1
+        with:
+          working-directory: temporalio/bridge
+
+      # Prepare
+      - run: python -m pip install --upgrade wheel poetry poethepoet
+      - run: poetry install --no-root -E opentelemetry
+
+      # Add the source dist only for Linux x64 for now
+      - if: ${{ matrix.package-suffix == 'linux-amd64' }}
+        run: poetry build --format sdist
+
+      # Build and fix the wheel
+      - run: poetry run cibuildwheel --output-dir dist
+      - run: poe fix-wheel
+
+      # Simple test
+      - run: poe test-dist-single
+
+      # Upload dist
+      - uses: actions/upload-artifact@v2
+        with:
+          name: packages-${{ matrix.package-suffix }}
+          path: dist

.github/workflows/ci.yml

@@ -1,5 +1,5 @@
 name: Continuous Integration
-on: # rebuild any PRs and main branch changes
+on:
   pull_request:
   push:
     branches:
@@ -12,15 +12,16 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        # TODO(cretz): 3.10.8 is breaking Windows Rust build
-        python: ["3.7", "3.10.7"]
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        python: ["3.7", "3.11"]
+        os: [ubuntu-latest, macos-latest, windows-latest, ubuntu-arm]
         include:
           - os: ubuntu-latest
-            python: 3.10.7
+            python: "3.11"
             docsTarget: true
             protoCheckTarget: true
-    runs-on: ${{ matrix.os }}
+          - os: ubuntu-arm
+            runsOn: buildjet-4vcpu-ubuntu-2204-arm
+    runs-on: ${{ matrix.runsOn || matrix.os }}
     steps:
       - name: Print build information
         run: "echo head_ref: ${{ github.head_ref }}, ref: ${{ github.ref }}, os: ${{ matrix.os }}, python: ${{ matrix.python }}"
@@ -33,15 +34,23 @@ jobs:
       - uses: Swatinem/rust-cache@v1
         with:
           working-directory: temporalio/bridge
-      - uses: actions/setup-python@v4
+      # actions/setup-python doesn't yet support ARM
+      - if: ${{ !endsWith(matrix.os, '-arm') }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+      - if: ${{ matrix.os == 'ubuntu-arm' }}
+        uses: deadsnakes/action@v2.1.1
         with:
           python-version: ${{ matrix.python }}
       - run: python -m pip install --upgrade wheel poetry poethepoet
       - run: poetry install --no-root -E opentelemetry
       - run: poe lint
       - run: poe build-develop
       - run: poe test -s -o log_cli_level=DEBUG
-      - run: poe test -s -o log_cli_level=DEBUG --workflow-environment time-skipping
+      # Time skipping doesn't yet support ARM
+      - if: ${{ !endsWith(matrix.os, '-arm') }}
+        run: poe test -s -o log_cli_level=DEBUG --workflow-environment time-skipping
 
       # Confirm protos are already generated properly
       - name: Check generated protos
@@ -58,103 +67,3 @@ jobs:
       - name: Deploy prod API docs
         if: ${{ github.ref == 'refs/heads/main' && matrix.docsTarget }}
         run: npx vercel deploy build/apidocs -t ${{ secrets.VERCEL_TOKEN }} --name python --scope temporal --prod --yes
-
-  # Compile the binaries and upload artifacts
-  compile-binaries:
-    strategy:
-      fail-fast: true
-      matrix:
-        include:
-          - os: ubuntu-latest
-            package-suffix: linux-amd64
-            ci-arch: auto
-          - os: macos-latest
-            package-suffix: macos-amd64
-            ci-arch: auto
-          # TODO(cretz): Disabling macOS arm because cibuildwheel is still
-          # generating an x64 wheel name even for arm64
-          #- os: macos-latest
-          #  package-suffix: macos-arm64
-          #  ci-arch: arm64
-            rust-add-target: aarch64-apple-darwin
-          - os: windows-latest
-            package-suffix: windows-amd64
-            ci-arch: auto
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: recursive
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10.7"
-
-      # Install Rust locally for non-Linux (Linux uses an internal docker
-      # command to build with cibuildwheel which uses rustup install defined
-      # in pyproject.toml)
-      - if: ${{ runner.os != 'Linux' }}
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          target: ${{ matrix.rust-add-target }}
-      - if: ${{ runner.os != 'Linux' }}
-        uses: Swatinem/rust-cache@v1
-        with:
-          working-directory: temporalio/bridge
-
-      # Prepare
-      - run: python -m pip install --upgrade wheel poetry poethepoet
-      - run: poetry install --no-root -E opentelemetry
-
-      # Add the source dist only for Linux x64 for now
-      - if: ${{ matrix.package-suffix == 'linux-amd64' }}
-        run: poetry build --format sdist
-
-      # Build and fix the wheel
-      - run: poetry run cibuildwheel --output-dir dist --arch ${{ matrix.ci-arch }}
-      - run: poe fix-wheel
-
-      # Do test only for ci-arch auto (i.e. local machine)
-      - if: ${{ matrix.ci-arch == 'auto' }}
-        run: poe test-dist-single
-
-      # Upload dist
-      - uses: actions/upload-artifact@v2
-        with:
-          name: packages-${{ matrix.package-suffix }}
-          path: dist
-
-  # We separate out Linux aarch64 so we can choose not to run it during PR since
-  # it is so slow in cibuildwheel (uses QEMU emulation). We can put this back in
-  # the above matrix when Linux ARM runners are available.
-  compile-binaries-linux-aarch64:
-    # Skip compiling Linux aarch64 on PR
-    if: ${{ github.event_name != 'pull_request' }}
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: recursive
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10.7"
-
-      # Need QEMU for ARM build on Linux
-      - uses: docker/setup-qemu-action@v1
-        with:
-          image: tonistiigi/binfmt:latest
-          platforms: arm64
-
-      # Prepare
-      - run: python -m pip install --upgrade wheel poetry poethepoet
-      - run: poetry install --no-root -E opentelemetry
-
-      # Build and fix the wheel
-      - run: poetry run cibuildwheel --output-dir dist --arch aarch64
-      - run: poe fix-wheel
-
-      # Upload dist
-      - uses: actions/upload-artifact@v2
-        with:
-          name: packages-linux-aarch64
-          path: dist

pyproject.toml

@@ -35,7 +35,7 @@ typing-extensions = "^4.2.0"
 
 [tool.poetry.dev-dependencies]
 black = "^22.3.0"
-cibuildwheel = "^2.7.0"
+cibuildwheel = "^2.11.0"
 grpcio-tools = "^1.48.0"
 isort = "^5.10.1"
 mypy = "^0.971"
@@ -110,11 +110,6 @@ build-verbosity = "1"
 before-all = "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y && yum install -y openssl-devel"
 environment = { PATH = "$PATH:$HOME/.cargo/bin", CARGO_NET_GIT_FETCH_WITH_CLI = "true" }
 
-[[tool.cibuildwheel.overrides]]
-# We need the aarch64 target for Rust
-before-all = "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable --target aarch64-unknown-linux-gnu -y && yum install -y openssl-devel"
-select = "*_aarch64"
-
 [tool.isort]
 profile = "black"
 skip_gitignore = true

temporalio/worker/workflow_instance.py

@@ -1135,7 +1135,12 @@ def _next_seq(self, type: str) -> int:
         self._curr_seqs[type] = seq
         return seq
 
-    def _register_task(self, task: asyncio.Task, *, name: Optional[str]) -> None:
+    def _register_task(
+        self,
+        task: asyncio.Task,
+        *,
+        name: Optional[str],
+    ) -> None:
         # Name not supported on older Python versions
         if sys.version_info >= (3, 8):
             # Put the workflow info at the end of the task name
@@ -1324,8 +1329,13 @@ def create_task(
         coro: Union[Awaitable[_T], Generator[Any, None, _T]],
         *,
         name: Optional[str] = None,
+        context: Optional[contextvars.Context] = None,
     ) -> asyncio.Task[_T]:
-        task = asyncio.Task(coro, loop=self)
+        # Context only supported on newer Python versions
+        if sys.version_info >= (3, 11):
+            task = asyncio.Task(coro, loop=self, context=context)  # type: ignore
+        else:
+            task = asyncio.Task(coro, loop=self)
         self._register_task(task, name=name)
         return task