diff --git a/CMakeLists.txt b/CMakeLists.txt index bd483d3e6..2179e3aa9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,6 +46,7 @@ set(PYBIND11_HEADERS include/pybind11/descr.h include/pybind11/options.h include/pybind11/eigen.h + include/pybind11/embed.h include/pybind11/eval.h include/pybind11/functional.h include/pybind11/numpy.h diff --git a/docs/reference.rst b/docs/reference.rst index 3d211f7e9..f375791c1 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -58,6 +58,17 @@ Passing extra arguments to ``def`` or ``class_`` .. doxygengroup:: annotations :members: +Embedding the interpreter +========================= + +.. doxygendefine:: PYBIND11_EMBEDDED_MODULE + +.. doxygenfunction:: initialize_interpreter + +.. doxygenfunction:: finalize_interpreter + +.. doxygenclass:: scoped_interpreter + Python build-in functions ========================= diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h new file mode 100644 index 000000000..a92465209 --- /dev/null +++ b/include/pybind11/embed.h @@ -0,0 +1,175 @@ +/* + pybind11/embed.h: Support for embedding the interpreter + + Copyright (c) 2017 Wenzel Jakob + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "pybind11.h" +#include "eval.h" + +#if defined(PYPY_VERSION) +# error Embedding the interpreter is not supported with PyPy +#endif + +#if PY_MAJOR_VERSION >= 3 +# define PYBIND11_EMBEDDED_MODULE_IMPL(name) \ + extern "C" PyObject *pybind11_init_impl_##name() { \ + return pybind11_init_wrapper_##name(); \ + } +#else +# define PYBIND11_EMBEDDED_MODULE_IMPL(name) \ + extern "C" void pybind11_init_impl_##name() { \ + pybind11_init_wrapper_##name(); \ + } +#endif + +/** \rst + Add a new module to the table of builtins for the interpreter. Must be + defined in global scope. The first macro parameter is the name of the + module (without quotes). The second parameter is the variable which will + be used as the interface to add functions and classes to the module. + + .. code-block:: cpp + + PYBIND11_EMBEDDED_MODULE(example, m) { + // ... initialize functions and classes here + m.def("foo", []() { + return "Hello, World!"; + }); + } + \endrst */ +#define PYBIND11_EMBEDDED_MODULE(name, variable) \ + static void pybind11_init_##name(pybind11::module &); \ + static PyObject *pybind11_init_wrapper_##name() { \ + auto m = pybind11::module(#name); \ + try { \ + pybind11_init_##name(m); \ + return m.ptr(); \ + } catch (pybind11::error_already_set &e) { \ + e.clear(); \ + PyErr_SetString(PyExc_ImportError, e.what()); \ + return nullptr; \ + } catch (const std::exception &e) { \ + PyErr_SetString(PyExc_ImportError, e.what()); \ + return nullptr; \ + } \ + } \ + PYBIND11_EMBEDDED_MODULE_IMPL(name) \ + pybind11::detail::embedded_module name(#name, pybind11_init_impl_##name); \ + void pybind11_init_##name(pybind11::module &variable) + + +NAMESPACE_BEGIN(pybind11) +NAMESPACE_BEGIN(detail) + +/// Python 2.7/3.x compatible version of `PyImport_AppendInittab` and error checks. +struct embedded_module { +#if PY_MAJOR_VERSION >= 3 + using init_t = PyObject *(*)(); +#else + using init_t = void (*)(); +#endif + embedded_module(const char *name, init_t init) { + if (Py_IsInitialized()) + pybind11_fail("Can't add new modules after the interpreter has been initialized"); + + auto result = PyImport_AppendInittab(name, init); + if (result == -1) + pybind11_fail("Insufficient memory to add a new module"); + } +}; + +NAMESPACE_END(detail) + +/** \rst + Initialize the Python interpreter. No other pybind11 or CPython API functions can be + called before this is done; with the exception of `PYBIND11_EMBEDDED_MODULE`. The + optional parameter can be used to skip the registration of signal handlers (see the + Python documentation for details). Calling this function again after the interpreter + has already been initialized is a fatal error. + \endrst */ +inline void initialize_interpreter(bool init_signal_handlers = true) { + if (Py_IsInitialized()) + pybind11_fail("The interpreter is already running"); + + Py_InitializeEx(init_signal_handlers ? 1 : 0); + + // Make .py files in the working directory available by default + auto sys_path = reinterpret_borrow(module::import("sys").attr("path")); + sys_path.append("."); +} + +/** \rst + Shut down the Python interpreter. No pybind11 or CPython API functions can be called + after this. In addition, pybind11 objects must not outlive the interpreter: + + .. code-block:: cpp + + { // BAD + py::initialize_interpreter(); + auto hello = py::str("Hello, World!"); + py::finalize_interpreter(); + } // <-- BOOM, hello's destructor is called after interpreter shutdown + + { // GOOD + py::initialize_interpreter(); + { // scoped + auto hello = py::str("Hello, World!"); + } // <-- OK, hello is cleaned up properly + py::finalize_interpreter(); + } + + { // BETTER + py::scoped_interpreter guard{}; + auto hello = py::str("Hello, World!"); + } + + .. warning:: + + Python cannot unload binary extension modules. If `initialize_interpreter` is + called again to restart the interpreter, the initializers of those modules will + be executed for a second time and they will fail. This is a known CPython issue. + See the Python documentation for details. + + \endrst */ +inline void finalize_interpreter() { Py_Finalize(); } + +/** \rst + Scope guard version of `initialize_interpreter` and `finalize_interpreter`. + This a move-only guard and only a single instance can exist. + + .. code-block:: cpp + + #include + + int main() { + py::scoped_interpreter guard{}; + py::print(Hello, World!); + } // <-- interpreter shutdown + \endrst */ +class scoped_interpreter { +public: + scoped_interpreter(bool init_signal_handlers = true) { + initialize_interpreter(init_signal_handlers); + } + + scoped_interpreter(const scoped_interpreter &) = delete; + scoped_interpreter(scoped_interpreter &&other) noexcept { other.is_valid = false; } + scoped_interpreter &operator=(const scoped_interpreter &) = delete; + scoped_interpreter &operator=(scoped_interpreter &&) = delete; + + ~scoped_interpreter() { + if (is_valid) + finalize_interpreter(); + } + +private: + bool is_valid = true; +}; + +NAMESPACE_END(pybind11) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 4954dcf4d..45cd8bfb6 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -798,6 +798,10 @@ public: } }; +/// \ingroup python_builtins +/// Return a dictionary representing the global symbol table, i.e. ``__main__.__dict__``. +inline dict globals() { return module::import("__main__").attr("__dict__").cast(); } + NAMESPACE_BEGIN(detail) /// Generic support for creating new Python heap types class generic_type : public object { diff --git a/setup.py b/setup.py index 517818e72..f1eac9c8d 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,7 @@ else: 'include/pybind11/complex.h', 'include/pybind11/descr.h', 'include/pybind11/eigen.h', + 'include/pybind11/embed.h', 'include/pybind11/eval.h', 'include/pybind11/functional.h', 'include/pybind11/numpy.h', diff --git a/tests/test_cmake_build/embed.cpp b/tests/test_cmake_build/embed.cpp index 44900c8b5..b9581d2fd 100644 --- a/tests/test_cmake_build/embed.cpp +++ b/tests/test_cmake_build/embed.cpp @@ -1,13 +1,8 @@ -#include -#include +#include namespace py = pybind11; -PyObject *make_module() { - py::module m("test_cmake_build"); - +PYBIND11_EMBEDDED_MODULE(test_cmake_build, m) { m.def("add", [](int i, int j) { return i + j; }); - - return m.ptr(); } int main(int argc, char *argv[]) { @@ -15,16 +10,12 @@ int main(int argc, char *argv[]) { throw std::runtime_error("Expected test.py file as the first argument"); auto test_py_file = argv[1]; - PyImport_AppendInittab("test_cmake_build", &make_module); - Py_Initialize(); - { - auto m = py::module::import("test_cmake_build"); - if (m.attr("add")(1, 2).cast() != 3) - throw std::runtime_error("embed.cpp failed"); + py::scoped_interpreter guard{}; - auto globals = py::module::import("__main__").attr("__dict__"); - py::module::import("sys").attr("argv") = py::make_tuple("test.py", "embed.cpp"); - py::eval_file(test_py_file, globals); - } - Py_Finalize(); + auto m = py::module::import("test_cmake_build"); + if (m.attr("add")(1, 2).cast() != 3) + throw std::runtime_error("embed.cpp failed"); + + py::module::import("sys").attr("argv") = py::make_tuple("test.py", "embed.cpp"); + py::eval_file(test_py_file, py::globals()); } diff --git a/tests/test_embed/catch.cpp b/tests/test_embed/catch.cpp index f79fe17ab..cface485d 100644 --- a/tests/test_embed/catch.cpp +++ b/tests/test_embed/catch.cpp @@ -1,5 +1,16 @@ -// Catch provides the `int main()` function here. This is a standalone +// The Catch implementation is compiled here. This is a standalone // translation unit to avoid recompiling it for every test change. -#define CATCH_CONFIG_MAIN +#include + +#define CATCH_CONFIG_RUNNER #include + +namespace py = pybind11; + +int main(int argc, const char *argv[]) { + py::scoped_interpreter guard{}; + auto result = Catch::Session().run(argc, argv); + + return result < 0xff ? result : 0xff; +} diff --git a/tests/test_embed/test_interpreter.cpp b/tests/test_embed/test_interpreter.cpp index 97af7eb60..1069ccac6 100644 --- a/tests/test_embed/test_interpreter.cpp +++ b/tests/test_embed/test_interpreter.cpp @@ -1,6 +1,4 @@ -#include -#include - +#include #include namespace py = pybind11; @@ -24,41 +22,62 @@ class PyWidget final : public Widget { int the_answer() const override { PYBIND11_OVERLOAD_PURE(int, Widget, the_answer); } }; -PyObject *make_embedded_module() { - py::module m("widget_module"); - +PYBIND11_EMBEDDED_MODULE(widget_module, m) { py::class_(m, "Widget") .def(py::init()) .def_property_readonly("the_message", &Widget::the_message); - - return m.ptr(); } -py::object import_file(const std::string &module, const std::string &path, py::object globals) { - auto locals = py::dict("module_name"_a=module, "path"_a=path); - py::eval( - "import imp\n" - "with open(path) as file:\n" - " new_module = imp.load_module(module_name, file, path, ('py', 'U', imp.PY_SOURCE))", - globals, locals - ); - return locals["new_module"]; +PYBIND11_EMBEDDED_MODULE(throw_exception, ) { + throw std::runtime_error("C++ Error"); +} + +PYBIND11_EMBEDDED_MODULE(throw_error_already_set, ) { + auto d = py::dict(); + d["missing"].cast(); } TEST_CASE("Pass classes and data between modules defined in C++ and Python") { - PyImport_AppendInittab("widget_module", &make_embedded_module); - Py_Initialize(); - { - auto globals = py::module::import("__main__").attr("__dict__"); - auto module = import_file("widget", "test_interpreter.py", globals); - REQUIRE(py::hasattr(module, "DerivedWidget")); + auto module = py::module::import("test_interpreter"); + REQUIRE(py::hasattr(module, "DerivedWidget")); - auto py_widget = module.attr("DerivedWidget")("Hello, World!"); - auto message = py_widget.attr("the_message"); - REQUIRE(message.cast() == "Hello, World!"); + auto locals = py::dict("hello"_a="Hello, World!", "x"_a=5, **module.attr("__dict__")); + py::exec(R"( + widget = DerivedWidget("{} - {}".format(hello, x)) + message = widget.the_message + )", py::globals(), locals); + REQUIRE(locals["message"].cast() == "Hello, World! - 5"); - const auto &cpp_widget = py_widget.cast(); - REQUIRE(cpp_widget.the_answer() == 42); - } - Py_Finalize(); + auto py_widget = module.attr("DerivedWidget")("The question"); + auto message = py_widget.attr("the_message"); + REQUIRE(message.cast() == "The question"); + + const auto &cpp_widget = py_widget.cast(); + REQUIRE(cpp_widget.the_answer() == 42); +} + +TEST_CASE("Import error handling") { + REQUIRE_NOTHROW(py::module::import("widget_module")); + REQUIRE_THROWS_WITH(py::module::import("throw_exception"), + "ImportError: C++ Error"); + REQUIRE_THROWS_WITH(py::module::import("throw_error_already_set"), + Catch::Contains("ImportError: KeyError")); +} + +TEST_CASE("There can be only one interpreter") { + static_assert(std::is_move_constructible::value, ""); + static_assert(!std::is_move_assignable::value, ""); + static_assert(!std::is_copy_constructible::value, ""); + static_assert(!std::is_copy_assignable::value, ""); + + REQUIRE_THROWS_WITH(py::initialize_interpreter(), "The interpreter is already running"); + REQUIRE_THROWS_WITH(py::scoped_interpreter(), "The interpreter is already running"); + + py::finalize_interpreter(); + REQUIRE_NOTHROW(py::scoped_interpreter()); + { + auto pyi1 = py::scoped_interpreter(); + auto pyi2 = std::move(pyi1); + } + py::initialize_interpreter(); }