[Bug] Can crash SDK with non-halting workflow

[dandavison]

The workflow below crashes the SDK. The fix may be to take a copy of this set with list(self._tasks), but I haven't investigated yet:

sdk-python/temporalio/worker/_workflow_instance.py

Line 1774 in f96679b

for task in self._tasks:

wait_until_positive
wait_until_negative
wait_until_positive
wait_until_negative
wait_until_positive
wait_until_negative
Failed running eviction job, not evicting
Traceback (most recent call last):
  File "/Users/dan/src/temporalio/samples-python/.venv/lib/python3.10/site-packages/temporalio/worker/workflow_sandbox/_runner.py", line 172, in _run_code
    exec(code, self.globals_and_locals, self.globals_and_locals)
  File "<string>", line 2, in <module>
  File "/Users/dan/src/temporalio/samples-python/.venv/lib/python3.10/site-packages/temporalio/worker/workflow_sandbox/_in_sandbox.py", line 81, in activate
    return self.instance.activate(act)
  File "/Users/dan/src/temporalio/samples-python/.venv/lib/python3.10/site-packages/temporalio/worker/_workflow_instance.py", line 361, in activate
    + f"Stack traces below:\n\n{self._stack_trace()}"
  File "/Users/dan/src/temporalio/samples-python/.venv/lib/python3.10/site-packages/temporalio/worker/_workflow_instance.py", line 1774, in _stack_trace
    for task in self._tasks:
RuntimeError: Set changed size during iteration

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/dan/src/temporalio/samples-python/.venv/lib/python3.10/site-packages/temporalio/worker/_workflow.py", line 249, in _handle_activation
    completion = await asyncio.wait_for(
  File "/Users/dan/.pyenv/versions/3.10.13/lib/python3.10/asyncio/tasks.py", line 445, in wait_for
    return fut.result()
  File "/Users/dan/.pyenv/versions/3.10.13/lib/python3.10/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/Users/dan/src/temporalio/samples-python/.venv/lib/python3.10/site-packages/temporalio/worker/workflow_sandbox/_runner.py", line 157, in activate
    self._run_code(
  File "/Users/dan/src/temporalio/samples-python/.venv/lib/python3.10/site-packages/temporalio/worker/workflow_sandbox/_runner.py", line 176, in _run_code
    del self.globals_and_locals[k]
KeyError: '__temporal_importer'

import asyncio

from temporalio import common, workflow
from temporalio.client import Client
from temporalio.worker import Worker


@workflow.defn
class StackTraceBugRepro:

    def __init__(self) -> None:
        self.val = 1

    async def wait_until_positive(self):
        while True:
            print("wait_until_positive")
            await workflow.wait_condition(lambda: self.val > 0)
            self.val = -self.val

    async def wait_until_negative(self):
        while True:
            print("wait_until_negative")
            await workflow.wait_condition(lambda: self.val < 0)
            self.val = -self.val

    @workflow.run
    async def run(self):
        await asyncio.gather(self.wait_until_negative(), self.wait_until_positive())


async def main():
    client = await Client.connect("localhost:7233")
    async with Worker(
        client,
        task_queue="tq",
        workflows=[StackTraceBugRepro],
    ):
        await client.execute_workflow(
            StackTraceBugRepro.run,
            id="wid",
            task_queue="tq",
            id_reuse_policy=common.WorkflowIDReusePolicy.TERMINATE_IF_RUNNING,
        )


if __name__ == "__main__":
    asyncio.run(main())