Add C++ interface for the Python interpreter

CMakeLists.txt

@@ -46,6 +46,7 @@ set(PYBIND11_HEADERS
   include/pybind11/descr.h
   include/pybind11/options.h
   include/pybind11/eigen.h
+  include/pybind11/embed.h
   include/pybind11/eval.h
   include/pybind11/functional.h
   include/pybind11/numpy.h

docs/reference.rst

@@ -58,6 +58,17 @@ Passing extra arguments to ``def`` or ``class_``
 .. doxygengroup:: annotations
     :members:
 
+Embedding the interpreter
+=========================
+
+.. doxygendefine:: PYBIND11_EMBEDDED_MODULE
+
+.. doxygenfunction:: initialize_interpreter
+
+.. doxygenfunction:: finalize_interpreter
+
+.. doxygenclass:: scoped_interpreter
+
 Python build-in functions
 =========================
 

include/pybind11/embed.h

@@ -0,0 +1,175 @@
+/*
+    pybind11/embed.h: Support for embedding the interpreter
+
+    Copyright (c) 2017 Wenzel Jakob <wenzel.jakob@epfl.ch>
+
+    All rights reserved. Use of this source code is governed by a
+    BSD-style license that can be found in the LICENSE file.
+*/
+
+#pragma once
+
+#include "pybind11.h"
+#include "eval.h"
+
+#if defined(PYPY_VERSION)
+#  error Embedding the interpreter is not supported with PyPy
+#endif
+
+#if PY_MAJOR_VERSION >= 3
+#  define PYBIND11_EMBEDDED_MODULE_IMPL(name)            \
+      extern "C" PyObject *pybind11_init_impl_##name() { \
+          return pybind11_init_wrapper_##name();         \
+      }
+#else
+#  define PYBIND11_EMBEDDED_MODULE_IMPL(name)            \
+      extern "C" void pybind11_init_impl_##name() {      \
+          pybind11_init_wrapper_##name();                \
+      }
+#endif
+
+/** \rst
+    Add a new module to the table of builtins for the interpreter. Must be
+    defined in global scope. The first macro parameter is the name of the
+    module (without quotes). The second parameter is the variable which will
+    be used as the interface to add functions and classes to the module.
+
+    .. code-block:: cpp
+
+        PYBIND11_EMBEDDED_MODULE(example, m) {
+            // ... initialize functions and classes here
+            m.def("foo", []() {
+                return "Hello, World!";
+            });
+        }
+ \endrst */
+#define PYBIND11_EMBEDDED_MODULE(name, variable)                              \
+    static void pybind11_init_##name(pybind11::module &);                     \
+    static PyObject *pybind11_init_wrapper_##name() {                         \
+        auto m = pybind11::module(#name);                                     \
+        try {                                                                 \
+            pybind11_init_##name(m);                                          \
+            return m.ptr();                                                   \
+        } catch (pybind11::error_already_set &e) {                            \
+            e.clear();                                                        \
+            PyErr_SetString(PyExc_ImportError, e.what());                     \
+            return nullptr;                                                   \
+        } catch (const std::exception &e) {                                   \
+            PyErr_SetString(PyExc_ImportError, e.what());                     \
+            return nullptr;                                                   \
+        }                                                                     \
+    }                                                                         \
+    PYBIND11_EMBEDDED_MODULE_IMPL(name)                                       \
+    pybind11::detail::embedded_module name(#name, pybind11_init_impl_##name); \
+    void pybind11_init_##name(pybind11::module &variable)
+
+
+NAMESPACE_BEGIN(pybind11)
+NAMESPACE_BEGIN(detail)
+
+/// Python 2.7/3.x compatible version of `PyImport_AppendInittab` and error checks.
+struct embedded_module {
+#if PY_MAJOR_VERSION >= 3
+    using init_t = PyObject *(*)();
+#else
+    using init_t = void (*)();
+#endif
+    embedded_module(const char *name, init_t init) {
+        if (Py_IsInitialized())
+            pybind11_fail("Can't add new modules after the interpreter has been initialized");
+
+        auto result = PyImport_AppendInittab(name, init);
+        if (result == -1)
+            pybind11_fail("Insufficient memory to add a new module");
+    }
+};
+
+NAMESPACE_END(detail)
+
+/** \rst
+    Initialize the Python interpreter. No other pybind11 or CPython API functions can be
+    called before this is done; with the exception of `PYBIND11_EMBEDDED_MODULE`. The
+    optional parameter can be used to skip the registration of signal handlers (see the
+    Python documentation for details). Calling this function again after the interpreter
+    has already been initialized is a fatal error.
+ \endrst */
+inline void initialize_interpreter(bool init_signal_handlers = true) {
+    if (Py_IsInitialized())
+        pybind11_fail("The interpreter is already running");
+
+    Py_InitializeEx(init_signal_handlers ? 1 : 0);
+
+    // Make .py files in the working directory available by default
+    auto sys_path = reinterpret_borrow<list>(module::import("sys").attr("path"));
+    sys_path.append(".");
+}
+
+/** \rst
+    Shut down the Python interpreter. No pybind11 or CPython API functions can be called
+    after this. In addition, pybind11 objects must not outlive the interpreter:
+
+    .. code-block:: cpp
+
+        { // BAD
+            py::initialize_interpreter();
+            auto hello = py::str("Hello, World!");
+            py::finalize_interpreter();
+        } // <-- BOOM, hello's destructor is called after interpreter shutdown
+
+        { // GOOD
+            py::initialize_interpreter();
+            { // scoped
+                auto hello = py::str("Hello, World!");
+            } // <-- OK, hello is cleaned up properly
+            py::finalize_interpreter();
+        }
+
+        { // BETTER
+            py::scoped_interpreter guard{};
+            auto hello = py::str("Hello, World!");
+        }
+
+    .. warning::
+
+        Python cannot unload binary extension modules. If `initialize_interpreter` is
+        called again to restart the interpreter, the initializers of those modules will
+        be executed for a second time and they will fail. This is a known CPython issue.
+        See the Python documentation for details.
+
+ \endrst */
+inline void finalize_interpreter() { Py_Finalize(); }
+
+/** \rst
+    Scope guard version of `initialize_interpreter` and `finalize_interpreter`.
+    This a move-only guard and only a single instance can exist.
+
+    .. code-block:: cpp
+
+        #include <pybind11/embed.h>
+
+        int main() {
+            py::scoped_interpreter guard{};
+            py::print(Hello, World!);
+        } // <-- interpreter shutdown
+ \endrst */
+class scoped_interpreter {
+public:
+    scoped_interpreter(bool init_signal_handlers = true) {
+        initialize_interpreter(init_signal_handlers);
+    }
+
+    scoped_interpreter(const scoped_interpreter &) = delete;
+    scoped_interpreter(scoped_interpreter &&other) noexcept { other.is_valid = false; }
+    scoped_interpreter &operator=(const scoped_interpreter &) = delete;
+    scoped_interpreter &operator=(scoped_interpreter &&) = delete;
+
+    ~scoped_interpreter() {
+        if (is_valid)
+            finalize_interpreter();
+    }
+
+private:
+    bool is_valid = true;
+};
+
+NAMESPACE_END(pybind11)

include/pybind11/pybind11.h

@@ -798,6 +798,10 @@ public:
     }
 };
 
+/// \ingroup python_builtins
+/// Return a dictionary representing the global symbol table, i.e. ``__main__.__dict__``.
+inline dict globals() { return module::import("__main__").attr("__dict__").cast<dict>(); }
+
 NAMESPACE_BEGIN(detail)
 /// Generic support for creating new Python heap types
 class generic_type : public object {

setup.py

@@ -21,6 +21,7 @@ else:
         'include/pybind11/complex.h',
         'include/pybind11/descr.h',
         'include/pybind11/eigen.h',
+        'include/pybind11/embed.h',
         'include/pybind11/eval.h',
         'include/pybind11/functional.h',
         'include/pybind11/numpy.h',

tests/test_cmake_build/embed.cpp

@@ -1,13 +1,8 @@
-#include <pybind11/pybind11.h>
-#include <pybind11/eval.h>
+#include <pybind11/embed.h>
 namespace py = pybind11;
 
-PyObject *make_module() {
-    py::module m("test_cmake_build");
-
+PYBIND11_EMBEDDED_MODULE(test_cmake_build, m) {
     m.def("add", [](int i, int j) { return i + j; });
-
-    return m.ptr();
 }
 
 int main(int argc, char *argv[]) {
@@ -15,16 +10,12 @@ int main(int argc, char *argv[]) {
         throw std::runtime_error("Expected test.py file as the first argument");
     auto test_py_file = argv[1];
 
-    PyImport_AppendInittab("test_cmake_build", &make_module);
-    Py_Initialize();
-    {
+    py::scoped_interpreter guard{};
+
     auto m = py::module::import("test_cmake_build");
     if (m.attr("add")(1, 2).cast<int>() != 3)
         throw std::runtime_error("embed.cpp failed");
 
-        auto globals = py::module::import("__main__").attr("__dict__");
     py::module::import("sys").attr("argv") = py::make_tuple("test.py", "embed.cpp");
-        py::eval_file(test_py_file, globals);
-    }
-    Py_Finalize();
+    py::eval_file(test_py_file, py::globals());
 }

tests/test_embed/catch.cpp

@@ -1,5 +1,16 @@
-// Catch provides the `int main()` function here. This is a standalone
+// The Catch implementation is compiled here. This is a standalone
 // translation unit to avoid recompiling it for every test change.
 
-#define CATCH_CONFIG_MAIN
+#include <pybind11/embed.h>
+
+#define CATCH_CONFIG_RUNNER
 #include <catch.hpp>
+
+namespace py = pybind11;
+
+int main(int argc, const char *argv[]) {
+    py::scoped_interpreter guard{};
+    auto result = Catch::Session().run(argc, argv);
+
+    return result < 0xff ? result : 0xff;
+}

tests/test_embed/test_interpreter.cpp

@@ -1,6 +1,4 @@
-#include <pybind11/pybind11.h>
-#include <pybind11/eval.h>
-
+#include <pybind11/embed.h>
 #include <catch.hpp>
 
 namespace py = pybind11;
@@ -24,41 +22,62 @@ class PyWidget final : public Widget {
     int the_answer() const override { PYBIND11_OVERLOAD_PURE(int, Widget, the_answer); }
 };
 
-PyObject *make_embedded_module() {
-    py::module m("widget_module");
-
+PYBIND11_EMBEDDED_MODULE(widget_module, m) {
     py::class_<Widget, PyWidget>(m, "Widget")
         .def(py::init<std::string>())
         .def_property_readonly("the_message", &Widget::the_message);
-
-    return m.ptr();
 }
 
-py::object import_file(const std::string &module, const std::string &path, py::object globals) {
-    auto locals = py::dict("module_name"_a=module, "path"_a=path);
-    py::eval<py::eval_statements>(
-        "import imp\n"
-        "with open(path) as file:\n"
-        "    new_module = imp.load_module(module_name, file, path, ('py', 'U', imp.PY_SOURCE))",
-        globals, locals
-    );
-    return locals["new_module"];
+PYBIND11_EMBEDDED_MODULE(throw_exception, ) {
+    throw std::runtime_error("C++ Error");
+}
+
+PYBIND11_EMBEDDED_MODULE(throw_error_already_set, ) {
+    auto d = py::dict();
+    d["missing"].cast<py::object>();
 }
 
 TEST_CASE("Pass classes and data between modules defined in C++ and Python") {
-    PyImport_AppendInittab("widget_module", &make_embedded_module);
-    Py_Initialize();
-    {
-        auto globals = py::module::import("__main__").attr("__dict__");
-        auto module = import_file("widget", "test_interpreter.py", globals);
+    auto module = py::module::import("test_interpreter");
     REQUIRE(py::hasattr(module, "DerivedWidget"));
 
-        auto py_widget = module.attr("DerivedWidget")("Hello, World!");
+    auto locals = py::dict("hello"_a="Hello, World!", "x"_a=5, **module.attr("__dict__"));
+    py::exec(R"(
+        widget = DerivedWidget("{} - {}".format(hello, x))
+        message = widget.the_message
+    )", py::globals(), locals);
+    REQUIRE(locals["message"].cast<std::string>() == "Hello, World! - 5");
+
+    auto py_widget = module.attr("DerivedWidget")("The question");
     auto message = py_widget.attr("the_message");
-        REQUIRE(message.cast<std::string>() == "Hello, World!");
+    REQUIRE(message.cast<std::string>() == "The question");
 
     const auto &cpp_widget = py_widget.cast<const Widget &>();
     REQUIRE(cpp_widget.the_answer() == 42);
-    }
-    Py_Finalize();
+}
+
+TEST_CASE("Import error handling") {
+    REQUIRE_NOTHROW(py::module::import("widget_module"));
+    REQUIRE_THROWS_WITH(py::module::import("throw_exception"),
+                        "ImportError: C++ Error");
+    REQUIRE_THROWS_WITH(py::module::import("throw_error_already_set"),
+                        Catch::Contains("ImportError: KeyError"));
+}
+
+TEST_CASE("There can be only one interpreter") {
+    static_assert(std::is_move_constructible<py::scoped_interpreter>::value, "");
+    static_assert(!std::is_move_assignable<py::scoped_interpreter>::value, "");
+    static_assert(!std::is_copy_constructible<py::scoped_interpreter>::value, "");
+    static_assert(!std::is_copy_assignable<py::scoped_interpreter>::value, "");
+
+    REQUIRE_THROWS_WITH(py::initialize_interpreter(), "The interpreter is already running");
+    REQUIRE_THROWS_WITH(py::scoped_interpreter(), "The interpreter is already running");
+
+    py::finalize_interpreter();
+    REQUIRE_NOTHROW(py::scoped_interpreter());
+    {
+        auto pyi1 = py::scoped_interpreter();
+        auto pyi2 = std::move(pyi1);
+    }
+    py::initialize_interpreter();
 }