scoped_interpreter. overloaded constructor: PyConfig param (#4330)

* scoped_interpreter overloaded ctor: PyConfig param

* style: pre-commit fixes

* refact: some logics extracted into funcs (precheck_interpreter, _initialize_interpreter); config_guard

* style: pre-commit fixes

* refact: scoped_config, some funcs hidden in detail ns

* refact: macro PYBIND11_PYCONFIG_SUPPORT_PY_VERSION + undef

* feat: PYBIND11_PYCONFIG_SUPPORT_PY_VERSION set to 3.8

* tests: Custom PyConfig

* ci: python 3.6 -> 3.8

* ci: reverted py 38 back  to 36; refact: initialize_interpreter overloads

* style: pre-commit fixes

* fix: readability-implicit-bool-conversion

* refact: each initialize_interpreter overloads in pybind11 ns

* Move `initialize_interpreter_pre_pyconfig()` into the `detail` namespace.

Move the `PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX` define down to where it is used for the first time, and check if it is defined already, so that it is possible to customize from the compilation command line, just in case there is some unforeseen issue for Python 3.8, 3.9, 3.10.

* tests: Add program dir to path, Custom PyConfig with argv

* refact: clang-formatted

* tests: Add-program-dir-to-path covers both scoped_interpreter overloads

* tests: Add-program-dir-to-path fixed

* tests: Add-program-dir-to-path py_version dependant validation

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Ralf W. Grosse-Kunstleve <rwgk@google.com>

include/pybind11/embed.h

@@ -86,37 +86,22 @@ inline wchar_t *widen_chars(const char *safe_arg) {
     return widened_arg;
 }
 
-PYBIND11_NAMESPACE_END(detail)
+inline void precheck_interpreter() {
-
-/** \rst
-    Initialize the Python interpreter. No other pybind11 or CPython API functions can be
-    called before this is done; with the exception of `PYBIND11_EMBEDDED_MODULE`. The
-    optional `init_signal_handlers` parameter can be used to skip the registration of
-    signal handlers (see the `Python documentation`_ for details). Calling this function
-    again after the interpreter has already been initialized is a fatal error.
-
-    If initializing the Python interpreter fails, then the program is terminated.  (This
-    is controlled by the CPython runtime and is an exception to pybind11's normal behavior
-    of throwing exceptions on errors.)
-
-    The remaining optional parameters, `argc`, `argv`, and `add_program_dir_to_path` are
-    used to populate ``sys.argv`` and ``sys.path``.
-    See the |PySys_SetArgvEx documentation|_ for details.
-
-    .. _Python documentation: https://docs.python.org/3/c-api/init.html#c.Py_InitializeEx
-    .. |PySys_SetArgvEx documentation| replace:: ``PySys_SetArgvEx`` documentation
-    .. _PySys_SetArgvEx documentation: https://docs.python.org/3/c-api/init.html#c.PySys_SetArgvEx
- \endrst */
-inline void initialize_interpreter(bool init_signal_handlers = true,
-                                   int argc = 0,
-                                   const char *const *argv = nullptr,
-                                   bool add_program_dir_to_path = true) {
     if (Py_IsInitialized() != 0) {
         pybind11_fail("The interpreter is already running");
     }
+}
 
-#if PY_VERSION_HEX < 0x030B0000
+#if !defined(PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX)
+#    define PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX (0x03080000)
+#endif
 
+#if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
+inline void initialize_interpreter_pre_pyconfig(bool init_signal_handlers,
+                                                int argc,
+                                                const char *const *argv,
+                                                bool add_program_dir_to_path) {
+    detail::precheck_interpreter();
     Py_InitializeEx(init_signal_handlers ? 1 : 0);
 #    if defined(WITH_THREAD) && PY_VERSION_HEX < 0x03070000
     PyEval_InitThreads();
@@ -150,26 +135,30 @@ inline void initialize_interpreter(bool init_signal_handlers = true,
     auto *pysys_argv = widened_argv.get();
 
     PySys_SetArgvEx(argc, pysys_argv, static_cast<int>(add_program_dir_to_path));
-#else
+}
-    PyConfig config;
+#endif
-    PyConfig_InitIsolatedConfig(&config);
-    config.isolated = 0;
-    config.use_environment = 1;
-    config.install_signal_handlers = init_signal_handlers ? 1 : 0;
 
-    PyStatus status = PyConfig_SetBytesArgv(&config, argc, const_cast<char *const *>(argv));
+PYBIND11_NAMESPACE_END(detail)
-    if (PyStatus_Exception(status)) {
+
+#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
+inline void initialize_interpreter(PyConfig *config,
+                                   int argc = 0,
+                                   const char *const *argv = nullptr,
+                                   bool add_program_dir_to_path = true) {
+    detail::precheck_interpreter();
+    PyStatus status = PyConfig_SetBytesArgv(config, argc, const_cast<char *const *>(argv));
+    if (PyStatus_Exception(status) != 0) {
         // A failure here indicates a character-encoding failure or the python
         // interpreter out of memory. Give up.
-        PyConfig_Clear(&config);
+        PyConfig_Clear(config);
-        throw std::runtime_error(PyStatus_IsError(status) ? status.err_msg
+        throw std::runtime_error(PyStatus_IsError(status) != 0 ? status.err_msg
-                                                          : "Failed to prepare CPython");
+                                                               : "Failed to prepare CPython");
     }
-    status = Py_InitializeFromConfig(&config);
+    status = Py_InitializeFromConfig(config);
-    PyConfig_Clear(&config);
+    if (PyStatus_Exception(status) != 0) {
-    if (PyStatus_Exception(status)) {
+        PyConfig_Clear(config);
-        throw std::runtime_error(PyStatus_IsError(status) ? status.err_msg
+        throw std::runtime_error(PyStatus_IsError(status) != 0 ? status.err_msg
-                                                          : "Failed to init CPython");
+                                                               : "Failed to init CPython");
     }
     if (add_program_dir_to_path) {
         PyRun_SimpleString("import sys, os.path; "
@@ -177,6 +166,43 @@ inline void initialize_interpreter(bool init_signal_handlers = true,
                            "os.path.abspath(os.path.dirname(sys.argv[0])) "
                            "if sys.argv and os.path.exists(sys.argv[0]) else '')");
     }
+    PyConfig_Clear(config);
+}
+#endif
+
+/** \rst
+    Initialize the Python interpreter. No other pybind11 or CPython API functions can be
+    called before this is done; with the exception of `PYBIND11_EMBEDDED_MODULE`. The
+    optional `init_signal_handlers` parameter can be used to skip the registration of
+    signal handlers (see the `Python documentation`_ for details). Calling this function
+    again after the interpreter has already been initialized is a fatal error.
+
+    If initializing the Python interpreter fails, then the program is terminated.  (This
+    is controlled by the CPython runtime and is an exception to pybind11's normal behavior
+    of throwing exceptions on errors.)
+
+    The remaining optional parameters, `argc`, `argv`, and `add_program_dir_to_path` are
+    used to populate ``sys.argv`` and ``sys.path``.
+    See the |PySys_SetArgvEx documentation|_ for details.
+
+    .. _Python documentation: https://docs.python.org/3/c-api/init.html#c.Py_InitializeEx
+    .. |PySys_SetArgvEx documentation| replace:: ``PySys_SetArgvEx`` documentation
+    .. _PySys_SetArgvEx documentation: https://docs.python.org/3/c-api/init.html#c.PySys_SetArgvEx
+ \endrst */
+inline void initialize_interpreter(bool init_signal_handlers = true,
+                                   int argc = 0,
+                                   const char *const *argv = nullptr,
+                                   bool add_program_dir_to_path = true) {
+#if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
+    detail::initialize_interpreter_pre_pyconfig(
+        init_signal_handlers, argc, argv, add_program_dir_to_path);
+#else
+    PyConfig config;
+    PyConfig_InitIsolatedConfig(&config);
+    config.isolated = 0;
+    config.use_environment = 1;
+    config.install_signal_handlers = init_signal_handlers ? 1 : 0;
+    initialize_interpreter(&config, argc, argv, add_program_dir_to_path);
 #endif
 }
 
@@ -264,6 +290,15 @@ public:
         initialize_interpreter(init_signal_handlers, argc, argv, add_program_dir_to_path);
     }
 
+#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
+    explicit scoped_interpreter(PyConfig *config,
+                                int argc = 0,
+                                const char *const *argv = nullptr,
+                                bool add_program_dir_to_path = true) {
+        initialize_interpreter(config, argc, argv, add_program_dir_to_path);
+    }
+#endif
+
     scoped_interpreter(const scoped_interpreter &) = delete;
     scoped_interpreter(scoped_interpreter &&other) noexcept { other.is_valid = false; }
     scoped_interpreter &operator=(const scoped_interpreter &) = delete;

tests/test_embed/test_interpreter.cpp

@@ -166,6 +166,72 @@ TEST_CASE("There can be only one interpreter") {
     py::initialize_interpreter();
 }
 
+#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
+TEST_CASE("Custom PyConfig") {
+    py::finalize_interpreter();
+    PyConfig config;
+    PyConfig_InitPythonConfig(&config);
+    REQUIRE_NOTHROW(py::scoped_interpreter{&config});
+    {
+        py::scoped_interpreter p{&config};
+        REQUIRE(py::module_::import("widget_module").attr("add")(1, 41).cast<int>() == 42);
+    }
+    py::initialize_interpreter();
+}
+
+TEST_CASE("Custom PyConfig with argv") {
+    py::finalize_interpreter();
+    {
+        PyConfig config;
+        PyConfig_InitIsolatedConfig(&config);
+        char *argv[] = {strdup("a.out")};
+        py::scoped_interpreter argv_scope{&config, 1, argv};
+        std::free(argv[0]);
+        auto module = py::module::import("test_interpreter");
+        auto py_widget = module.attr("DerivedWidget")("The question");
+        const auto &cpp_widget = py_widget.cast<const Widget &>();
+        REQUIRE(cpp_widget.argv0() == "a.out");
+    }
+    py::initialize_interpreter();
+}
+#endif
+
+TEST_CASE("Add program dir to path") {
+    static auto get_sys_path_size = []() -> size_t {
+        auto sys_path = py::module::import("sys").attr("path");
+        return py::len(sys_path);
+    };
+    static auto validate_path_len = [](size_t default_len) {
+#if PY_VERSION_HEX < 0x030A0000
+        // It seems a value remains in sys.path
+        // left by the previous call of scoped_interpreter ctor.
+        REQUIRE(get_sys_path_size() > default_len);
+#else
+        REQUIRE(get_sys_path_size() == default_len + 1);
+#endif
+    };
+    py::finalize_interpreter();
+
+    size_t sys_path_default_size = 0;
+    {
+        py::scoped_interpreter scoped_interp{true, 0, nullptr, false};
+        sys_path_default_size = get_sys_path_size();
+    }
+    {
+        py::scoped_interpreter scoped_interp{}; // expected to append some to sys.path
+        validate_path_len(sys_path_default_size);
+    }
+#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX
+    {
+        PyConfig config;
+        PyConfig_InitPythonConfig(&config);
+        py::scoped_interpreter scoped_interp{&config}; // expected to append some to sys.path
+        validate_path_len(sys_path_default_size);
+    }
+#endif
+    py::initialize_interpreter();
+}
+
 bool has_pybind11_internals_builtin() {
     auto builtins = py::handle(PyEval_GetBuiltins());
     return builtins.contains(PYBIND11_INTERNALS_ID);