Add concurrent Python region tests

Author: Christopher Rowley

AGENTS.md

@@ -1,15 +1,14 @@
 # Repository Instructions
 
 ## Running tests
-- **Julia tests**: Run from the project root with `julia --project -e 'using Pkg; Pkg.test()'`. Expect a warning about the General registry being unreachable in locked-down environments; PyCall tests are marked broken/skipped. With the currently installed Julia 1.13.0, the functional tests pass but Aqua's persistent-task subprocess check may fail because its `done.log` is not created.
+- **Julia tests**: Run from the project root with `julia --project -e 'using Pkg; Pkg.test()'`. Expect a warning about the General registry being unreachable in locked-down environments; PyCall tests are marked broken/skipped. With the currently installed Julia 1.13.1, the functional tests pass but Aqua's persistent-task subprocess check may fail because its `done.log` is not created.
 - **Python tests**:
   - Copy `pysrc/juliacall/juliapkg-dev.json` to `pysrc/juliacall/juliapkg.json` before running (do **not** commit this copy).
   - Execute with `uv run pytest -s --nbval ./pytest` (add `--cov=pysrc` when coverage is needed).
-- Sometimes `juliapkg` requires Julia 1.10–1.11. Check `juliaup status` rather than
-  assuming 1.11.7 is installed: as of 2026-10-10 this environment only has Julia
-  1.13 builds. JuliaPkg selecting Julia 1.13.1 under uv's downloaded Python 3.10
-  fails to load `libjulia-codegen` because the Python process has loaded a
-  `libstdc++.so.6` that lacks `GLIBCXX_3.4.30`.
+- Sometimes `juliapkg` requires Julia 1.10–1.11. Juliaup provides Julia 1.11.9
+  through the `1.11` channel. JuliaPkg selecting Julia 1.13.1 under
+  uv's downloaded Python 3.10 fails to load `libjulia-codegen` because the Python
+  process has loaded a `libstdc++.so.6` that lacks `GLIBCXX_3.4.30`.
 - `julia --project=docs docs/make.jl` requires a valid Git `origin` so Documenter can infer
   source links; it fails during `makedocs` in checkouts without one.
 

pytest/test_all.py

@@ -155,3 +155,45 @@ def test_parallel_call(yld):
         t2 = time() - t0
     # executing the tasks should take about 1 second because they happen in parallel
     assert 0.9 < t2 < 1.5
+
+
+def test_concurrent_callback_roundtrips():
+    """Python threads can yield in Julia and repeatedly call back into Python."""
+    from concurrent.futures import ThreadPoolExecutor
+    from threading import Barrier, get_ident
+    from juliacall import Main as jl
+
+    # Each worker crosses Python -> Julia -> Python -> Julia repeatedly. Checking
+    # the OS thread in the Python callback catches restoring another worker's
+    # borrowed thread state, while the start barrier makes state races likely on
+    # both GIL and free-threaded builds without assuming how Julia schedules them.
+    jl.jl_eval(
+        """
+        function _callback_roundtrip(callback, python_tid, start)
+            total = 0
+            for value in start:(start + 19)
+                yield()
+                total += pyconvert(Int, callback(python_tid, value))
+            end
+            return total
+        end
+        """
+    )
+
+    def callback(expected_tid, value):
+        assert get_ident() == expected_tid
+        return 2 * int(jl.identity(value))
+
+    participants = 2
+    start_barrier = Barrier(participants)
+
+    def worker(start):
+        python_tid = get_ident()
+        start_barrier.wait()
+        return jl._callback_roundtrip(callback, python_tid, start)
+
+    with ThreadPoolExecutor(participants) as pool:
+        results = list(pool.map(worker, range(participants)))
+
+    for start, total in enumerate(results):
+        assert total == 2 * sum(range(start, start + 20))

test/C.jl

@@ -25,20 +25,28 @@ end
     @test PythonCall.C.PyThreadState_GetUnchecked() == C_NULL
 
     # Nested regions reuse the state cached by the owning Julia thread. A break
-    # must relinquish that cache while yielding and restore it on return.
+    # must relinquish that cache while yielding, keep the task on the same Julia
+    # thread, and restore both the cache and the task's original stickiness.
+    test_task = current_task()
+    oldsticky = test_task.sticky
     @pyregion begin
         local state = PythonCall.C.TASK_STATE()
         local thread_state = PythonCall.C.THREAD_STATE()
-        @test thread_state.task === current_task()
+        local tid = threadid()
+        @test test_task.sticky
+        @test thread_state.task === test_task
         @test thread_state.task_state === state
-        @pyregion @test PythonCall.C.current_task_state(current_task()) === state
+        @pyregion @test PythonCall.C.current_task_state(test_task) === state
         @pyregionbreak begin
+            @test test_task.sticky
             @test thread_state.task === nothing
             yield()
+            @test threadid() == tid
         end
-        @test thread_state.task === current_task()
+        @test thread_state.task === test_task
         @test thread_state.task_state === state
     end
+    @test test_task.sticky == oldsticky
     @test PythonCall.C.THREAD_STATE().task === nothing
 
     @test @pyregion begin
@@ -51,6 +59,30 @@ end
         pyconvert(Int, pyint(3)) == 3
     end
 
+    # A task waiting in ordinary Julia code must release its Python state so that
+    # another task on the same Julia thread can use Python. The bounded wait makes
+    # a missing break fail promptly instead of leaving the test suite deadlocked.
+    started = Channel{Nothing}(1)
+    ready = Ref(false)
+    break_status = Ref(:not_started)
+    values = Ref((0, 0))
+    @sync begin
+        @async @pyregion begin
+            put!(started, nothing)
+            first = pyconvert(Int, pybuiltins.sum([1, 2, 3]))
+            break_status[] = @pyregionbreak timedwait(() -> ready[], 2; pollint=0.001)
+            second = pyconvert(Int, pybuiltins.sum([4, 5, 6]))
+            values[] = (first, second)
+        end
+        @async begin
+            take!(started)
+            @test pyconvert(Int, pybuiltins.sum([7, 8, 9])) == 24
+            ready[] = true
+        end
+    end
+    @test break_status[] == :ok
+    @test values[] == (6, 15)
+
     @test_throws ErrorException @pyregion @pyregionbreak error("region exception")
     @test PythonCall.C.PyThreadState_GetUnchecked() == C_NULL
     @test_throws PyException @pyregion pybuiltins.int("not an integer")