feat(python): More robust handling of async database calls

py-polars/polars/io/database/_executor.py

@@ -253,17 +253,42 @@ def _from_rows(
 
     @staticmethod
     def _run_async(co: Coroutine) -> Any:  # type: ignore[type-arg]
-        """Consolidate async event loop acquisition and coroutine/func execution."""
+        """Run asynchronous code as if it was synchronous."""
         import asyncio
 
+        can_nest_asyncio = close_loop = False
         try:
             with warnings.catch_warnings():
                 warnings.simplefilter("ignore", DeprecationWarning)
                 loop = asyncio.get_event_loop()
         except RuntimeError:
+            # if we create the loop, we can clean it up
             loop = asyncio.new_event_loop()
             asyncio.set_event_loop(loop)
-        return loop.run_until_complete(co)
+            close_loop = True
+
+        try:
+            import nest_asyncio
+
+            nest_asyncio.apply()
+            can_nest_asyncio = True
+        except ModuleNotFoundError:
+            pass
+
+        try:
+            return loop.run_until_complete(co)
+        except RuntimeError as err:
+            if not can_nest_asyncio and "event loop is already running" in str(err):
+                msg = (
+                    "Synchronising this query requires nesting asyncio calls."
+                    "\n\nPlease run: pip install nest_asyncio"
+                )
+                raise RuntimeError(msg) from err
+            else:
+                raise
+        finally:
+            if close_loop:
+                loop.close()
 
     @staticmethod
     def _inject_type_overrides(

py-polars/polars/meta/versions.py

@@ -28,6 +28,7 @@ def show_versions() -> None:
     gevent:               24.2.1
     hvplot:               0.9.2
     matplotlib:           3.8.3
+    nest_asyncio:         1.6.0
     numpy:                1.26.4
     openpyxl:             3.1.2
     pandas:               2.2.1
@@ -70,6 +71,7 @@ def _get_dependency_info() -> dict[str, str]:
         "gevent",
         "hvplot",
         "matplotlib",
+        "nest_asyncio",
         "numpy",
         "openpyxl",
         "pandas",

py-polars/pyproject.toml

@@ -40,6 +40,7 @@ Changelog = "https://github.com/pola-rs/polars/releases"
 [project.optional-dependencies]
 # NOTE: keep this list in sync with show_versions() and requirements-dev.txt
 adbc = ["adbc_driver_manager", "adbc_driver_sqlite"]
+async = ["nest_asyncio"]
 cloudpickle = ["cloudpickle"]
 connectorx = ["connectorx >= 0.3.2"]
 deltalake = ["deltalake >= 0.14.0"]
@@ -60,7 +61,7 @@ timezone = ["backports.zoneinfo; python_version < '3.9'", "tzdata; platform_syst
 xlsx2csv = ["xlsx2csv >= 0.8.0"]
 xlsxwriter = ["xlsxwriter"]
 all = [
-  "polars[adbc,cloudpickle,connectorx,deltalake,fastexcel,fsspec,gevent,numpy,pandas,plot,pyarrow,pydantic,pyiceberg,sqlalchemy,timezone,xlsx2csv,xlsxwriter]",
+  "polars[adbc,async,cloudpickle,connectorx,deltalake,fastexcel,fsspec,gevent,numpy,pandas,plot,pyarrow,pydantic,pyiceberg,sqlalchemy,timezone,xlsx2csv,xlsxwriter]",
 ]
 
 [tool.maturin]
@@ -94,6 +95,7 @@ module = [
   "kuzu",
   "matplotlib.*",
   "moto.server",
+  "nest_asyncio",
   "openpyxl",
   "polars.polars",
   "pyarrow.*",