[v2.9] Backport some recent commits

.github/workflows/ci.yml

@@ -30,8 +30,8 @@ jobs:
         - '3.6'
         - '3.9'
         - '3.10'
-        - 'pypy-3.7-v7.3.7'
-        - 'pypy-3.8-v7.3.7'
+        - 'pypy-3.7'
+        - 'pypy-3.8'
 
         # Items in here will either be added to the build matrix (if not
         # present), or add new keys to an existing matrix element if all the

include/pybind11/cast.h

@@ -85,11 +85,15 @@ protected:
                                                                                                   \
 public:                                                                                           \
     static constexpr auto name = py_name;                                                         \
-    template <typename T_, enable_if_t<std::is_same<type, remove_cv_t<T_>>::value, int> = 0>      \
-    static handle cast(T_ *src, return_value_policy policy, handle parent) {                      \
+    template <typename T_,                                                                        \
+              ::pybind11::detail::enable_if_t<                                                    \
+                  std::is_same<type, ::pybind11::detail::remove_cv_t<T_>>::value,                 \
+                  int> = 0>                                                                       \
+    static ::pybind11::handle cast(                                                               \
+        T_ *src, ::pybind11::return_value_policy policy, ::pybind11::handle parent) {             \
         if (!src)                                                                                 \
-            return none().release();                                                              \
-        if (policy == return_value_policy::take_ownership) {                                      \
+            return ::pybind11::none().release();                                                  \
+        if (policy == ::pybind11::return_value_policy::take_ownership) {                          \
             auto h = cast(std::move(*src), policy, parent);                                       \
             delete src;                                                                           \
             return h;                                                                             \
@@ -100,7 +104,7 @@ public:
     operator type &() { return value; }                /* NOLINT(bugprone-macro-parentheses) */   \
     operator type &&() && { return std::move(value); } /* NOLINT(bugprone-macro-parentheses) */   \
     template <typename T_>                                                                        \
-    using cast_op_type = pybind11::detail::movable_cast_op_type<T_>
+    using cast_op_type = ::pybind11::detail::movable_cast_op_type<T_>
 
 template <typename CharT>
 using is_std_char_type = any_of<std::is_same<CharT, char>, /* std::string */

include/pybind11/detail/class.h

@@ -101,14 +101,14 @@ inline PyTypeObject *make_static_property_type() {
 inline PyTypeObject *make_static_property_type() {
     auto d = dict();
     PyObject *result = PyRun_String(R"(\
-        class pybind11_static_property(property):
-            def __get__(self, obj, cls):
-                return property.__get__(self, cls, cls)
-
-            def __set__(self, obj, value):
-                cls = obj if isinstance(obj, type) else type(obj)
-                property.__set__(self, cls, value)
-        )",
+class pybind11_static_property(property):
+    def __get__(self, obj, cls):
+        return property.__get__(self, cls, cls)
+
+    def __set__(self, obj, value):
+        cls = obj if isinstance(obj, type) else type(obj)
+        property.__set__(self, cls, value)
+)",
                                     Py_file_input,
                                     d.ptr(),
                                     d.ptr());

include/pybind11/detail/type_caster_base.h

@@ -510,9 +510,10 @@ PYBIND11_NOINLINE std::string error_string() {
         }
 
         PyFrameObject *frame = trace->tb_frame;
+        Py_XINCREF(frame);
         errorString += "\n\nAt:\n";
         while (frame) {
-#    if PY_VERSION_HEX >= 0x03090000
+#    if PY_VERSION_HEX >= 0x030900B1
             PyCodeObject *f_code = PyFrame_GetCode(frame);
 #    else
             PyCodeObject *f_code = frame->f_code;
@@ -522,8 +523,15 @@ PYBIND11_NOINLINE std::string error_string() {
             errorString += "  " + handle(f_code->co_filename).cast<std::string>() + "("
                            + std::to_string(lineno)
                            + "): " + handle(f_code->co_name).cast<std::string>() + "\n";
-            frame = frame->f_back;
             Py_DECREF(f_code);
+#    if PY_VERSION_HEX >= 0x030900B1
+            auto *b_frame = PyFrame_GetBack(frame);
+#    else
+            auto *b_frame = frame->f_back;
+            Py_XINCREF(b_frame);
+#    endif
+            Py_DECREF(frame);
+            frame = b_frame;
         }
     }
 #endif

include/pybind11/embed.h

@@ -247,6 +247,10 @@ inline void finalize_interpreter() {
     if (builtins.contains(id) && isinstance<capsule>(builtins[id])) {
         internals_ptr_ptr = capsule(builtins[id]);
     }
+    // Local internals contains data managed by the current interpreter, so we must clear them to
+    // avoid undefined behaviors when initializing another interpreter
+    detail::get_local_internals().registered_types_cpp.clear();
+    detail::get_local_internals().registered_exception_translators.clear();
 
     Py_Finalize();
 

include/pybind11/eval.h

@@ -82,7 +82,7 @@ template <eval_mode mode = eval_expr, size_t N>
 object eval(const char (&s)[N], object global = globals(), object local = object()) {
     /* Support raw string literals by removing common leading whitespace */
     auto expr = (s[0] == '\n') ? str(module_::import("textwrap").attr("dedent")(s)) : str(s);
-    return eval<mode>(expr, global, local);
+    return eval<mode>(expr, std::move(global), std::move(local));
 }
 
 inline void exec(const str &expr, object global = globals(), object local = object()) {
@@ -91,7 +91,7 @@ inline void exec(const str &expr, object global = globals(), object local = obje
 
 template <size_t N>
 void exec(const char (&s)[N], object global = globals(), object local = object()) {
-    eval<eval_statements>(s, global, local);
+    eval<eval_statements>(s, std::move(global), std::move(local));
 }
 
 #if defined(PYPY_VERSION) && PY_VERSION_HEX >= 0x03000000

include/pybind11/pybind11.h

@@ -334,7 +334,7 @@ class cpp_function : public function {
                             const std::type_info *const *types,
                             size_t args) {
         // Do NOT receive `unique_rec` by value. If this function fails to move out the unique_ptr,
-        // we do not want this to destuct the pointer. `initialize` (the caller) still relies on
+        // we do not want this to destruct the pointer. `initialize` (the caller) still relies on
         // the pointee being alive after this call. Only move out if a `capsule` is going to keep
         // it alive.
         auto *rec = unique_rec.get();
@@ -2671,19 +2671,19 @@ get_type_override(const void *this_ptr, const type_info *this_type, const char *
 
     /* Don't call dispatch code if invoked from overridden function.
        Unfortunately this doesn't work on PyPy. */
-#if !defined(PYPY_VERSION) && PY_VERSION_HEX < 0x030B0000
-    // TODO: Remove PyPy workaround for Python 3.11.
-    // Current API fails on 3.11 since co_varnames can be null.
+#if !defined(PYPY_VERSION)
 #    if PY_VERSION_HEX >= 0x03090000
     PyFrameObject *frame = PyThreadState_GetFrame(PyThreadState_Get());
     if (frame != nullptr) {
         PyCodeObject *f_code = PyFrame_GetCode(frame);
         // f_code is guaranteed to not be NULL
         if ((std::string) str(f_code->co_name) == name && f_code->co_argcount > 0) {
             PyObject *locals = PyEval_GetLocals();
-            if (locals != nullptr && f_code->co_varnames != nullptr) {
-                PyObject *self_caller
-                    = dict_getitem(locals, PyTuple_GET_ITEM(f_code->co_varnames, 0));
+            if (locals != nullptr) {
+                PyObject *co_varnames = PyObject_GetAttrString((PyObject *) f_code, "co_varnames");
+                PyObject *self_arg = PyTuple_GET_ITEM(co_varnames, 0);
+                Py_DECREF(co_varnames);
+                PyObject *self_caller = dict_getitem(locals, self_arg);
                 if (self_caller == self.ptr()) {
                     Py_DECREF(f_code);
                     Py_DECREF(frame);
@@ -2729,9 +2729,9 @@ get_type_override(const void *this_ptr, const type_info *this_type, const char *
                        d.ptr());
     if (result == nullptr)
         throw error_already_set();
+    Py_DECREF(result);
     if (d["self"].is_none())
         return function();
-    Py_DECREF(result);
 #endif
 
     return override;

noxfile.py

@@ -1,8 +1,9 @@
 import nox
 
+nox.needs_version = ">=2022.1.7"
 nox.options.sessions = ["lint", "tests", "tests_packaging"]
 
-PYTHON_VERISONS = ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
+PYTHON_VERSIONS = ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
 
 
 @nox.session(reuse_venv=True)
@@ -14,7 +15,7 @@ def lint(session: nox.Session) -> None:
     session.run("pre-commit", "run", "-a")
 
 
-@nox.session(python=PYTHON_VERISONS)
+@nox.session(python=PYTHON_VERSIONS)
 def tests(session: nox.Session) -> None:
     """
     Run the tests (requires a compiler).
@@ -24,14 +25,12 @@ def tests(session: nox.Session) -> None:
     session.install("-r", "tests/requirements.txt")
     session.run(
         "cmake",
-        "-S",
-        ".",
-        "-B",
-        tmpdir,
+        "-S.",
+        f"-B{tmpdir}",
         "-DPYBIND11_WERROR=ON",
         "-DDOWNLOAD_CATCH=ON",
         "-DDOWNLOAD_EIGEN=ON",
-        *session.posargs
+        *session.posargs,
     )
     session.run("cmake", "--build", tmpdir)
     session.run("cmake", "--build", tmpdir, "--config=Release", "--target", "check")

tests/test_custom_type_casters.cpp

@@ -20,6 +20,7 @@ class ArgInspector2 {
     std::string arg = "(default arg inspector 2)";
 };
 class ArgAlwaysConverts {};
+
 namespace pybind11 {
 namespace detail {
 template <>
@@ -105,6 +106,34 @@ struct type_caster<DestructionTester> {
 } // namespace detail
 } // namespace pybind11
 
+// Define type caster outside of `pybind11::detail` and then alias it.
+namespace other_lib {
+struct MyType {};
+// Corrupt `py` shorthand alias for surrounding context.
+namespace py {}
+// Corrupt unqualified relative `pybind11` namespace.
+namespace pybind11 {}
+// Correct alias.
+namespace py_ = ::pybind11;
+// Define caster. This is effectively no-op, we only ensure it compiles and we
+// don't have any symbol collision when using macro mixin.
+struct my_caster {
+    PYBIND11_TYPE_CASTER(MyType, py_::detail::const_name("MyType"));
+    bool load(py_::handle, bool) { return true; }
+
+    static py_::handle cast(const MyType &, py_::return_value_policy, py_::handle) {
+        return py_::bool_(true).release();
+    }
+};
+} // namespace other_lib
+// Effectively "alias" it into correct namespace (via inheritance).
+namespace pybind11 {
+namespace detail {
+template <>
+struct type_caster<other_lib::MyType> : public other_lib::my_caster {};
+} // namespace detail
+} // namespace pybind11
+
 TEST_SUBMODULE(custom_type_casters, m) {
     // test_custom_type_casters
 
@@ -175,4 +204,6 @@ TEST_SUBMODULE(custom_type_casters, m) {
     m.def("destruction_tester_cstats",
           &ConstructorStats::get<DestructionTester>,
           py::return_value_policy::reference);
+
+    m.def("other_lib_type", [](other_lib::MyType x) { return x; });
 }

tests/test_custom_type_casters.py

@@ -115,3 +115,7 @@ def test_custom_caster_destruction():
 
     # Make sure we still only have the original object (from ..._no_destroy()) alive:
     assert cstats.alive() == 1
+
+
+def test_custom_caster_other_lib():
+    assert m.other_lib_type(True)

tests/test_embed/test_interpreter.cpp

@@ -380,3 +380,21 @@ TEST_CASE("sys.argv gets initialized properly") {
     }
     py::initialize_interpreter();
 }
+
+TEST_CASE("make_iterator can be called before then after finalizing an interpreter") {
+    // Reproduction of issue #2101 (https://github.com/pybind/pybind11/issues/2101)
+    py::finalize_interpreter();
+
+    std::vector<int> container;
+    {
+        pybind11::scoped_interpreter g;
+        auto iter = pybind11::make_iterator(container.begin(), container.end());
+    }
+
+    REQUIRE_NOTHROW([&]() {
+        pybind11::scoped_interpreter g;
+        auto iter = pybind11::make_iterator(container.begin(), container.end());
+    }());
+
+    py::initialize_interpreter();
+}

tests/test_sequences_and_iterators.cpp

@@ -452,7 +452,7 @@ TEST_SUBMODULE(sequences_and_iterators, m) {
             [](IntPairs &self) { return py::make_value_iterator(self, py::call_guard<int>()); },
             py::keep_alive<0, 1>());
 
-    // test_iterater_referencing
+    // test_iterator_referencing
     py::class_<NonCopyableInt>(m, "NonCopyableInt")
         .def(py::init<int>())
         .def("set", &NonCopyableInt::set)

tools/FindCatch.cmake

@@ -9,6 +9,8 @@
 #  CATCH_INCLUDE_DIR      - path to catch.hpp
 #  CATCH_VERSION          - version number
 
+option(DOWNLOAD_CATCH "Download catch2 if not found")
+
 if(NOT Catch_FIND_VERSION)
   message(FATAL_ERROR "A version number must be specified.")
 elseif(Catch_FIND_REQUIRED)

tools/FindPythonLibsNew.cmake

@@ -112,11 +112,24 @@ endif()
 # VERSION. VERSION will typically be like "2.7" on unix, and "27" on windows.
 execute_process(
   COMMAND
-    "${PYTHON_EXECUTABLE}" "-c" "from distutils import sysconfig as s;import sys;import struct;
+    "${PYTHON_EXECUTABLE}" "-c" "
+import sys;import struct;
+import sysconfig as s
+USE_SYSCONFIG = sys.version_info >= (3, 10)
+if not USE_SYSCONFIG:
+    from distutils import sysconfig as ds
 print('.'.join(str(v) for v in sys.version_info));
 print(sys.prefix);
-print(s.get_python_inc(plat_specific=True));
-print(s.get_python_lib(plat_specific=True));
+if USE_SYSCONFIG:
+    scheme = s.get_default_scheme()
+    if scheme == 'posix_local':
+        # Debian's default scheme installs to /usr/local/ but we want to find headers in /usr/
+        scheme = 'posix_prefix'
+    print(s.get_path('platinclude', scheme))
+    print(s.get_path('platlib'))
+else:
+    print(ds.get_python_inc(plat_specific=True));
+    print(ds.get_python_lib(plat_specific=True));
 print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO'));
 print(hasattr(sys, 'gettotalrefcount')+0);
 print(struct.calcsize('@P'));

tools/pybind11Config.cmake.in

@@ -88,7 +88,7 @@ you can either use the basic targets, or use the FindPython tools:
   set_target_properties(MyModule2 PROPERTIES
                                   INTERPROCEDURAL_OPTIMIZATION ON
                                   CXX_VISIBILITY_PRESET ON
-                                  VISIBLITY_INLINES_HIDDEN ON)
+                                  VISIBILITY_INLINES_HIDDEN ON)
 
 If you build targets yourself, you may be interested in stripping the output
 for reduced size; this is the one other feature that the helper function gives you.