Implementation of Python's 'with' statement as py::with

docs/advanced/pycpp/utilities.rst

@@ -21,6 +21,48 @@ expected in Python:
     auto args = py::make_tuple("unpacked", true);
     py::print("->", *args, "end"_a="<-"); // -> unpacked True <-
 
+Executing code in a Python `with` statement
+===========================================
+
+Where in C++, the scope of local variables and their destructor is used to manage
+resources in the RAII idiom, Python has a ``with`` statement and context managers
+for such situations. At the start and end of the ``with`` statement, the context
+manager's ``__enter__`` and ``__exit__`` methods are called, respectively. To more
+easily use context managers in a C++ context, pybind11 provides a utility function
+`py::with <with>`, that matches the semantics of a Python ``with``-statement (see
+`PEP 343 <https://www.python.org/dev/peps/pep-0343/>`_):
+
+.. code-block:: cpp
+
+	auto io = py::module::import("io");
+	py::with(io.attr("open")("tmp.out", "w"), [](py::object &&f) {
+        for (int i = 0; i < 10; ++i) {
+            f.attr("write")(i);
+            f.attr("write")("\n");
+        }
+	});
+
+This code snippet corresponds to the following in Python:
+
+.. code-block:: python
+
+    import io
+    with io.open("tmp.out", "w") as f:
+        for i in range(10):
+            f.write(i)
+            f.write("\n")
+
+The `py::object` parameter of the lambda function can be omitted if the object resulting
+from the context manager (i.e., the ``as VAR`` part in the ``with`` statement) is not of use.
+
+Optionally, an extra `py::with_exception_policy <with_exception_policy>` argument can be
+passed to `py::with <with>`. If the value ``py::with_exception_policy::translate`` is selected,
+pybind11 will try to translate any C++ exception inside the ``with`` statement and pass the
+Python exception as argument into the ``__exit__`` method of the context manager (cfr. PEP 343).
+If ``py::with_exception_policy::translate`` is passed and an exception gets thrown, pybind11
+will not try to translate it, `__exit__` will be called as if no exception was thrown,
+and the original exception will be cascaded down to the caller.
+
 .. _ostream_redirect:
 
 Capturing standard output from ostream

include/pybind11/detail/internals.h

@@ -288,4 +288,32 @@ T &get_or_create_shared_data(const std::string &name) {
     return *ptr;
 }
 
+/// Translates a std::exception_ptr into a Python exception, using the registered translators.
+/// Returns nullptr if the exception was successfully translated, or the leftover untranslated
+/// C++ exception resulting from the potential partial translation.
+inline std::exception_ptr translate_exception(std::exception_ptr last_exception) {
+    /* Give each registered exception translator a chance to translate it
+       to a Python exception in reverse order of registration.
+
+       A translator may choose to do one of the following:
+
+       - catch the exception and call PyErr_SetString or PyErr_SetObject
+         to set a standard (or custom) Python exception, or
+       - do nothing and let the exception fall through to the next translator, or
+       - delegate translation to the next translator by throwing a new type of exception. */
+
+    auto &registered_exception_translators = detail::get_internals().registered_exception_translators;
+    for (auto& translator : registered_exception_translators) {
+        try {
+            translator(last_exception);
+        } catch (...) {
+            last_exception = std::current_exception();
+            continue;
+        }
+        return nullptr;
+    }
+    // Return the leftover exception that is not handled by any registered translator
+    return last_exception;
+}
+
 NAMESPACE_END(PYBIND11_NAMESPACE)

include/pybind11/pybind11.h

@@ -664,29 +664,10 @@ class cpp_function : public function {
             e.restore();
             return nullptr;
         } catch (...) {
-            /* When an exception is caught, give each registered exception
-               translator a chance to translate it to a Python exception
-               in reverse order of registration.
-
-               A translator may choose to do one of the following:
-
-                - catch the exception and call PyErr_SetString or PyErr_SetObject
-                  to set a standard (or custom) Python exception, or
-                - do nothing and let the exception fall through to the next translator, or
-                - delegate translation to the next translator by throwing a new type of exception. */
-
-            auto last_exception = std::current_exception();
-            auto &registered_exception_translators = get_internals().registered_exception_translators;
-            for (auto& translator : registered_exception_translators) {
-                try {
-                    translator(last_exception);
-                } catch (...) {
-                    last_exception = std::current_exception();
-                    continue;
-                }
-                return nullptr;
-            }
-            PyErr_SetString(PyExc_SystemError, "Exception escaped from default exception translator!");
+            // When an exception is caught, try to translate it into Python exception.
+            auto untranslated = translate_exception(std::current_exception());
+            if (untranslated)
+                PyErr_SetString(PyExc_SystemError, "Exception escaped from default exception translator!");
             return nullptr;
         }
 
@@ -1841,6 +1822,112 @@ void print(Args &&...args) {
     detail::print(c.args(), c.kwargs());
 }
 
+NAMESPACE_BEGIN(detail)
+
+template <typename T>
+struct dependent_false { static constexpr bool value = false; };
+
+template <typename Block, typename Signature = remove_cv_t<function_signature_t<Block>>, typename SFINAE = void>
+struct with_block_call_traits {
+    static void call(Block &&, object &&) {
+        static_assert(dependent_false<Block>::value,
+                      "The inner block function passed to pybind11::with should either take no arguments, "
+                      "or a single argument convertible from a pybind11::object& or pybind11::object&&, "
+                      "and should return void.");
+    }
+};
+
+template <typename Block>
+struct with_block_call_traits<Block, void()> {
+public:
+    static void call(Block &&block, object &&) {
+        std::forward<Block>(block)();
+    }
+};
+
+template <typename Block, typename Arg>
+struct with_block_call_traits<Block, void(Arg),
+                            enable_if_t<std::is_convertible<object&, Arg>::value ||
+                                        std::is_convertible<object&&, Arg>::value>> {
+private:
+    static void call_impl(Block &&block, object &&obj, std::true_type) {
+        std::forward<Block>(block)(std::move(obj));
+    }
+
+    static void call_impl(Block &&block, object &&obj, std::false_type) {
+        std::forward<Block>(block)(obj);
+    }
+
+public:
+    static void call(Block &&block, object &&obj) {
+        call_impl(std::forward<Block>(block), std::move(obj), std::is_convertible<object&&, Arg>());
+    }
+};
+
+template<typename Block>
+void call_with_block(Block &&block, object &&obj) {
+    with_block_call_traits<Block>::call(std::forward<Block>(block), std::move(obj));
+}
+
+NAMESPACE_END(detail)
+
+enum class with_exception_policy {
+    cascade,
+    translate
+};
+
+// PEP 343 specification: https://www.python.org/dev/peps/pep-0343/#specification-the-with-statement
+template <typename Block>
+void with(const object &mgr, Block &&block, with_exception_policy policy = with_exception_policy::translate) {
+    object exit = mgr.attr("__exit__");
+    object value = mgr.attr("__enter__")();
+    bool exc = true;
+
+    std::exception_ptr original_exception = nullptr;
+    try {
+        try {
+            detail::call_with_block(std::forward<Block>(block), std::move(value));
+        }
+        catch (const error_already_set &) {
+            exc = false;
+            // If already a Python error, catch in the outer try-catch
+            original_exception = std::current_exception();
+            throw;
+        }
+        catch (...) {
+            exc = false;
+            // Else, try our best to translate the error into a Python error before calling mrg.__exit__
+            original_exception = std::current_exception();
+            if (policy == with_exception_policy::translate) {
+                auto untranslated = translate_exception(std::current_exception());
+                if (untranslated)
+                    std::rethrow_exception(untranslated);
+                else
+                    throw error_already_set();
+            }
+            else {
+                throw;
+            }
+        }
+    }
+    catch (const error_already_set &e) {
+        // A Python error
+        auto exit_result = exit(e.type() ? e.type() : none(),
+                                e.value() ? e.value() : none(),
+                                e.trace() ? e.trace() : none());
+        if (!bool_(std::move(exit_result)))
+            std::rethrow_exception(original_exception);
+    }
+    catch (...) {
+        // Not a Python error
+        exit(none(), none(), none());
+        std::rethrow_exception(original_exception);
+    }
+
+    if (exc)
+        exit(none(), none(), none());
+}
+
 #if defined(WITH_THREAD) && !defined(PYPY_VERSION)
 
 /* The functions below essentially reproduce the PyGILState_* API using a RAII

tests/CMakeLists.txt

@@ -60,6 +60,7 @@ set(PYBIND11_TEST_FILES
   test_stl_binders.cpp
   test_tagbased_polymorphic.cpp
   test_virtual_functions.cpp
+  test_with.cpp
 )
 
 # Invoking cmake with something like:

tests/test_with.cpp

@@ -0,0 +1,136 @@
+/*
+    tests/test_with.cpp -- Usage of pybind11 with statement
+
+    Copyright (c) 2018 Yannick Jadoul
+
+    All rights reserved. Use of this source code is governed by a
+    BSD-style license that can be found in the LICENSE file.
+*/
+
+#include "pybind11_tests.h"
+
+#include <pybind11/eval.h>
+
+class CppContextManager {
+public:
+    CppContextManager(std::string value, bool swallow_exceptions=false)
+        : m_value(std::move(value)), m_swallow_exceptions(swallow_exceptions) {}
+
+    std::string enter() {
+        ++m_entered;
+        return m_value;
+    }
+
+    bool exit(py::args args) {
+        ++m_exited;
+        m_exit_args = args;
+        return m_swallow_exceptions;
+    }
+
+    std::string m_value;
+    bool m_swallow_exceptions;
+    unsigned int m_entered = 0;
+    unsigned int m_exited = 0;
+    py::object m_exit_args = py::none();
+};
+
+class NewCppException {};
+
+
+TEST_SUBMODULE(with_, m) {
+    py::class_<CppContextManager>(m, "CppContextManager")
+        .def(py::init<std::string, bool>(), py::arg("value"), py::arg("swallow_exceptions") = false)
+        .def("__enter__", &CppContextManager::enter)
+        .def("__exit__", &CppContextManager::exit)
+        .def_readonly("value", &CppContextManager::m_value)
+        .def_readonly("entered", &CppContextManager::m_entered)
+        .def_readonly("exited", &CppContextManager::m_exited)
+        .def_readonly("exit_args", &CppContextManager::m_exit_args);
+
+    py::enum_<py::with_exception_policy>(m, "WithExceptionPolicy")
+        .value("Translate", py::with_exception_policy::translate)
+        .value("Cascade", py::with_exception_policy::cascade);
+
+    m.def("no_args", [](const py::object &mgr) {
+        py::object value;
+        py::with(mgr, [&value]() {
+            value = py::none();
+        });
+        return value;
+    });
+
+    m.def("lvalue_arg", [](const py::object &mgr) {
+        py::object value;
+        py::with(mgr, [&value](py::object v) {
+            value = v;
+        });
+        return value;
+    });
+
+    m.def("lvalue_ref_arg", [](const py::object &mgr) {
+        py::object value;
+        py::with(mgr, [&value](py::object &v) {
+            value = v;
+        });
+        return value;
+    });
+
+    m.def("lvalue_const_ref_arg", [](const py::object &mgr) {
+        py::object value;
+        py::with(mgr, [&value](const py::object &v) {
+            value = v;
+        });
+        return value;
+    });
+
+    m.def("rvalue_ref_arg", [](const py::object &mgr) {
+        py::object value;
+        py::with(mgr, [&value](py::object &&v) {
+            value = v;
+        });
+        return value;
+    });
+
+    m.def("python_exception", [](const py::object &mgr, py::with_exception_policy exception_policy) {
+        py::object value;
+        py::with(mgr, [&value](py::object v) {
+            value = v;
+            py::exec("raise RuntimeError('This is a test. Please stay calm.')");
+        }, exception_policy);
+        return value;
+    });
+
+    m.def("cpp_exception", [](const py::object &mgr, py::with_exception_policy exception_policy) {
+        py::object value;
+        py::with(mgr, [&value](py::object v) {
+            value = v;
+            throw std::runtime_error("This is a test. Please stay calm.");
+        }, exception_policy);
+        return value;
+    });
+
+    m.def("catch_cpp_exception", [](const py::object &mgr) {
+        try {
+            py::with(mgr, []() {
+                throw NewCppException();
+            });
+        }
+        catch (const py::error_already_set &) {
+            return "error_already_set";
+        }
+        catch (const NewCppException &) {
+            return "original_exception";
+        }
+        catch (...) {
+            return "another_exception";
+        }
+        return "no_exception";
+    });
+
+    m.def("SHOULD_NOT_COMPILE_UNCOMMENTED", [](const py::object &mgr) {
+        // py::with(mgr, [](int) {});
+        // py::with(mgr, [](py::object, int) {});
+        // py::with(mgr, [](py::object) { return "something"; });
+        (void) mgr;
+    });
+}