CI: `test_on_completed_callback_refcount` failed on Python 3.12

[hongchaodeng]

What happened + What you expected to happen

test_on_completed_callback_refcount failed on Python 3.12

Versions / Dependencies

Python 3.12

Reproduction script

python -m pytest -vvs python/ray/tests/test_asyncio.py -k "test_on_completed_callback_refcount"

Issue Severity

None

[hongchaodeng]

I can reproduce this using only python 3.12 + cython:

# main.py

import asyncio
import sys

from my_module.func import SignalActor

async def main():
    signal = SignalActor()

    def callback():
        raise ValueError("error")

    refcount = sys.getrefcount(callback)
    print(f"{refcount=}")

    fut = asyncio.create_task(signal.wait(callback))

    refcount2 = sys.getrefcount(callback)
    print(f"{refcount2=}")
    assert refcount2 > refcount

    try:
        signal.send(callback)
    except Exception:
        print("send exception")
    finally:
        print("finally 2")

    try:
        await fut
    except Exception:
        print("wait exception")
    finally:
        print("finally 3")


    while True:
        refcount3 = sys.getrefcount(callback)
        print(f"{refcount3=}")
        if refcount3 == refcount:
            print("success")
            return
        else:
            print("sleep")
            await asyncio.sleep(1)


asyncio.run(main())

# my_module/func.pyx 

import asyncio

cimport cpython

class SignalActor:
    def __init__(self):
        self.ready_event = asyncio.Event()

    def send(self, callback):
        self.ready_event.set()

    async def wait(self, callback):
        cpython.Py_INCREF(callback)
        await self.ready_event.wait()
        try:
            callback()
        # except Exception:
        #     print("unblocking this comment will make it work")
        finally:
            print("finally")
            cpython.Py_DECREF(callback)

# setup.py

from setuptools import setup, Extension
from Cython.Build import cythonize

extensions = [
    Extension(
        name="my_module.func",
        sources=["my_module/func.pyx"],
    )
]

setup(
    name="cython",
    ext_modules=cythonize(extensions),
)

Then run:

python setup.py build_ext --inplace
python main.py

This doesn't work on Python 3.12 but works on prior versions. Seems like a regression in Python/Cython.