Replace Boost.python with pybind11

.github/ci-hpc-config.yml

@@ -30,12 +30,12 @@ gnu-14.2.0:
         -DENABLE_ALL_TESTS=ON
         -DBoost_ROOT=/usr/local/apps/boost/1.87.0/GNU/14.2
         -DBoost_INCLUDE_DIR=/usr/local/apps/boost/1.87.0/GNU/14.2/include
-        -DBoost_PYTHON310_LIBRARY_RELEASE=/usr/local/apps/boost/1.87.0/GNU/14.2/lib/libboost_python312.so
         -DPython3_ROOT_DIR=/usr/local/apps/python3/3.12.9-01
         -DPython3_EXECUTABLE=/usr/local/apps/python3/3.12.9-01/bin/python3
         -DPython3_INCLUDE_DIRS=/usr/local/apps/python3/3.12.9-01/include/python3.12
         -DPython3_LIBRARIES=/usr/local/apps/python3/3.12.9-01/lib64/libpython3.12.so
         -DPython3_LIBRARY_DIRS=/usr/local/apps/python3/3.12.9-01/lib64
+        -DCMAKE_PREFIX_PATH=/usr/local/apps/python3/3.12.11-01/lib/python3.12/site-packages/pybind11/share/cmake/pybind11
     ctest_options:
       # disable s_http* tests
       - -L nightly -E '(s_http)'
@@ -54,15 +54,15 @@ gnu-12.2.0:
     cmake_options:
       - >-
         -DCMAKE_VERBOSE_MAKEFILE=ON
-        -DENABLE_ALL_TESTS=ON 
-        -DBoost_ROOT=/usr/local/apps/boost/1.81.0/GNU/12.2 
-        -DBoost_INCLUDE_DIR=/usr/local/apps/boost/1.81.0/GNU/12.2/include 
-        -DBoost_PYTHON310_LIBRARY_RELEASE=/usr/local/apps/boost/1.81.0/GNU/12.2/lib/libboost_python310.so 
-        -DPython3_ROOT_DIR=/usr/local/apps/python3/3.10.10-01 
-        -DPython3_EXECUTABLE=/usr/local/apps/python3/3.10.10-01/bin/python3 
-        -DPython3_INCLUDE_DIRS=/usr/local/apps/python3/3.10.10-01/include/python3.10 
-        -DPython3_LIBRARIES=/usr/local/apps/python3/3.10.10-01/lib64/libpython3.10.so 
+        -DENABLE_ALL_TESTS=ON
+        -DBoost_ROOT=/usr/local/apps/boost/1.81.0/GNU/12.2
+        -DBoost_INCLUDE_DIR=/usr/local/apps/boost/1.81.0/GNU/12.2/include
+        -DPython3_ROOT_DIR=/usr/local/apps/python3/3.10.10-01
+        -DPython3_EXECUTABLE=/usr/local/apps/python3/3.10.10-01/bin/python3
+        -DPython3_INCLUDE_DIRS=/usr/local/apps/python3/3.10.10-01/include/python3.10
+        -DPython3_LIBRARIES=/usr/local/apps/python3/3.10.10-01/lib64/libpython3.10.so
         -DPython3_LIBRARY_DIRS=/usr/local/apps/python3/3.10.10-01/lib64
+        -DCMAKE_PREFIX_PATH=/usr/local/apps/python3/3.10.10-01/lib/python3.10/site-packages/pybind11/share/cmake/pybind11
     ctest_options:
       # disable s_http* tests
       - -L nightly -E '(s_http)'
@@ -82,14 +82,14 @@ gnu-8.5.0:
       - >-
         -DCMAKE_VERBOSE_MAKEFILE=ON
         -DENABLE_ALL_TESTS=ON
-        -DBoost_ROOT=/usr/local/apps/boost/1.84.0/GNU/8.5 
-        -DBoost_INCLUDE_DIR=/usr/local/apps/boost/1.84.0/GNU/8.5/include 
-        -DBoost_PYTHON310_LIBRARY_RELEASE=/usr/local/apps/boost/1.84.0/GNU/8.5/lib/libboost_python311.so 
+        -DBoost_ROOT=/usr/local/apps/boost/1.84.0/GNU/8.5
+        -DBoost_INCLUDE_DIR=/usr/local/apps/boost/1.84.0/GNU/8.5/include
         -DPython3_ROOT_DIR=/usr/local/apps/python3/3.11.8-01
-        -DPython3_EXECUTABLE=/usr/local/apps/python3/3.11.8-01/bin/python3 
+        -DPython3_EXECUTABLE=/usr/local/apps/python3/3.11.8-01/bin/python3
         -DPython3_INCLUDE_DIRS=/usr/local/apps/python3/3.11.8-01/include/python3.11
-        -DPython3_LIBRARIES=/usr/local/apps/python3/3.11.8-01/lib64/libpython3.11.so 
+        -DPython3_LIBRARIES=/usr/local/apps/python3/3.11.8-01/lib64/libpython3.11.so
         -DPython3_LIBRARY_DIRS=/usr/local/apps/python3/3.11.8-01/lib64
+        -DCMAKE_PREFIX_PATH=/usr/local/apps/python3/3.11.8-01/lib/python3.11/site-packages/pybind11/share/cmake/pybind11
     ctest_options:
       # disable s_http* tests
       - -L nightly -E '(s_http)'
@@ -112,12 +112,12 @@ intel-2025.0.1:
         -DCMAKE_CXX_COMPILER=icpx
         -DBoost_ROOT=/usr/local/apps/boost/1.87.0/INTEL/2025.0
         -DBoost_INCLUDE_DIR=/usr/local/apps/boost/1.87.0/INTEL/2025.0/include
-        -DBoost_PYTHON310_LIBRARY_RELEASE=/usr/local/apps/boost/1.87.0/INTEL/2025.0/lib/libboost_python312.so
         -DPython3_ROOT_DIR=/usr/local/apps/python3/3.12.9-01
         -DPython3_EXECUTABLE=/usr/local/apps/python3/3.12.9-01/bin/python3
         -DPython3_INCLUDE_DIRS=/usr/local/apps/python3/3.12.9-01/include/python3.12
         -DPython3_LIBRARIES=/usr/local/apps/python3/3.12.9-01/lib64/libpython3.12.so
         -DPython3_LIBRARY_DIRS=/usr/local/apps/python3/3.12.9-01/lib64
+        -DCMAKE_PREFIX_PATH=/usr/local/apps/python3/3.12.9-01/lib/python3.12/site-packages/pybind11/share/cmake/pybind11
     ctest_options:
       # disable s_http* tests
       - -L nightly -E '(s_http)'
@@ -140,12 +140,12 @@ intel-2021.4.0:
         -DCMAKE_CXX_COMPILER=icpx
         -DBoost_ROOT=/usr/local/apps/boost/1.81.0/INTEL/2021.4
         -DBoost_INCLUDE_DIR=/usr/local/apps/boost/1.81.0/INTEL/2021.4/include
-        -DBoost_PYTHON310_LIBRARY_RELEASE=/usr/local/apps/boost/1.81.0/INTEL/2021.4/lib/libboost_python310.so
         -DPython3_ROOT_DIR=/usr/local/apps/python3/3.10.10-01
         -DPython3_EXECUTABLE=/usr/local/apps/python3/3.10.10-01/bin/python3
         -DPython3_INCLUDE_DIRS=/usr/local/apps/python3/3.10.10-01/include/python3.10
         -DPython3_LIBRARIES=/usr/local/apps/python3/3.10.10-01/lib64/libpython3.10.so
         -DPython3_LIBRARY_DIRS=/usr/local/apps/python3/3.10.10-01/lib64
+        -DCMAKE_PREFIX_PATH=/usr/local/apps/python3/3.10.10-01/lib/python3.10/site-packages/pybind11/share/cmake/pybind11
     ctest_options:
       - -L nightly -E py3_
 
@@ -166,12 +166,12 @@ nvidia-24.11:
         -DENABLE_ALL_TESTS=ON
         -DBoost_ROOT=/usr/local/apps/boost/1.87.0/NVIDIA/24.11
         -DBoost_INCLUDE_DIR=/usr/local/apps/boost/1.87.0/NVIDIA/24.11/include
-        -DBoost_PYTHON310_LIBRARY_RELEASE=/usr/local/apps/boost/1.87.0/NVIDIA/24.11/lib/libboost_python312.so
         -DPython3_ROOT_DIR=/usr/local/apps/python3/3.12.9-01
         -DPython3_EXECUTABLE=/usr/local/apps/python3/3.12.9-01/bin/python3
         -DPython3_INCLUDE_DIRS=/usr/local/apps/python3/3.12.9-01/include/python3.12
         -DPython3_LIBRARIES=/usr/local/apps/python3/3.12.9-01/lib64/libpython3.12.so
         -DPython3_LIBRARY_DIRS=/usr/local/apps/python3/3.12.9-01/lib64
+        -DCMAKE_PREFIX_PATH=/usr/local/apps/python3/3.12.9-01/lib/python3.12/site-packages/pybind11/share/cmake/pybind11
     ctest_options:
       # disable all tests
       - --version
@@ -193,12 +193,12 @@ nvidia-22.11:
         -DENABLE_ALL_TESTS=ON
         -DBoost_ROOT=/usr/local/apps/boost/1.81.0/NVIDIA/22.11
         -DBoost_INCLUDE_DIR=/usr/local/apps/boost/1.81.0/NVIDIA/22.11/include
-        -DBoost_PYTHON310_LIBRARY_RELEASE=/usr/local/apps/boost/1.81.0/NVIDIA/22.11/lib/libboost_python310.so
         -DPython3_ROOT_DIR=/usr/local/apps/python3/3.10.10-01
         -DPython3_EXECUTABLE=/usr/local/apps/python3/3.10.10-01/bin/python3
         -DPython3_INCLUDE_DIRS=/usr/local/apps/python3/3.10.10-01/include/python3.10
         -DPython3_LIBRARIES=/usr/local/apps/python3/3.10.10-01/lib64/libpython3.10.so
         -DPython3_LIBRARY_DIRS=/usr/local/apps/python3/3.10.10-01/lib64
+        -DCMAKE_PREFIX_PATH=/usr/local/apps/python3/3.10.10-01/lib/python3.10/site-packages/pybind11/share/cmake/pybind11
     ctest_options: 
       # disable all tests
       - --version
@@ -220,12 +220,12 @@ aocc-4.0.0:
         -DENABLE_ALL_TESTS=ON
         -DBoost_ROOT=/usr/local/apps/boost/1.81.0/AMD/4.0
         -DBoost_INCLUDE_DIR=/usr/local/apps/boost/1.81.0/AMD/4.0/include
-        -DBoost_PYTHON310_LIBRARY_RELEASE=/usr/local/apps/boost/1.81.0/AMD/4.0/lib/libboost_python310.so
         -DPython3_ROOT_DIR=/usr/local/apps/python3/3.10.10-01
         -DPython3_EXECUTABLE=/usr/local/apps/python3/3.10.10-01/bin/python3
         -DPython3_INCLUDE_DIRS=/usr/local/apps/python3/3.10.10-01/include/python3.10
         -DPython3_LIBRARIES=/usr/local/apps/python3/3.10.10-01/lib64/libpython3.10.so
         -DPython3_LIBRARY_DIRS=/usr/local/apps/python3/3.10.10-01/lib64
+        -DCMAKE_PREFIX_PATH=/usr/local/apps/python3/3.10.10-01/lib/python3.10/site-packages/pybind11/share/cmake/pybind11
     ctest_options:
       # disable all tests
       - --version

cmake/Dependencies.cmake

@@ -123,6 +123,25 @@ if (ENABLE_PYTHON)
 endif()
 
 
+# =========================================================================================
+# pybind11
+# =========================================================================================
+if (ENABLE_PYTHON)
+
+  ecbuild_info( "Locating pybind11" )
+
+  find_package(pybind11 2.10.3 REQUIRED)
+
+  ecbuild_info( "pybind11 details:" )
+  ecbuild_info( " * pybind11_FOUND        : ${pybind11_FOUND}" )
+  ecbuild_info( " * pybind11_INCLUDE_DIRS : ${pybind11_INCLUDE_DIRS}" )
+  ecbuild_info( " * pybind11_VERSION      : ${pybind11_VERSION}" )
+  ecbuild_info( " * pybind11_VERSION_MAJOR: ${pybind11_VERSION_MAJOR}" )
+  ecbuild_info( " * pybind11_VERSION_MINOR: ${pybind11_VERSION_MINOR}" )
+  ecbuild_info( " * pybind11_VERSION_PATCH: ${pybind11_VERSION_PATCH}" )
+
+endif()
+
 # =========================================================================================
 # Boost
 # =========================================================================================
@@ -174,21 +193,6 @@ if ( Boost_MINOR_VERSION GREATER_EQUAL 86 )
   list(APPEND _boost_needed_libs process)
 endif()
 
-if (ENABLE_PYTHON)
-  # The following is used to find Boost.python library, as the library name changes with python version
-  if ( Boost_MINOR_VERSION GREATER 66 )
-    # cmake 3.15
-    # see: https://gitlab.kitware.com/cmake/cmake/issues/19656
-    # INTERFACE_LIBRARY targets may only have whitelisted properties.
-    set(_python_base_version "${Python3_VERSION_MAJOR}${Python3_VERSION_MINOR}")
-  else()
-    set(_python_base_version "${Python3_VERSION_MAJOR}")
-  endif()
-  set(ECFLOW_BOOST_PYTHON_COMPONENT "python${_python_base_version}")
-
-  list(APPEND _boost_needed_libs ${ECFLOW_BOOST_PYTHON_COMPONENT})
-endif()
-
 if(HAVE_TESTS) # HAVE_TESTS is defined if ecbuild ENABLE_TESTS is set, (by default this is set)
   list(APPEND _boost_needed_libs unit_test_framework test_exec_monitor )
 endif()

libs/attribute/src/ecflow/attribute/CronAttr.hpp

@@ -77,14 +77,10 @@ class CronAttr {
     const TimeSeries& time() const { return timeSeries_; }
     const TimeSeries& time_series() const { return timeSeries_; }
     bool last_day_of_month() const { return last_day_of_month_; }
-    std::vector<int>::const_iterator week_days_begin() const { return weekDays_.begin(); }
-    std::vector<int>::const_iterator week_days_end() const { return weekDays_.end(); }
-    std::vector<int>::const_iterator last_week_days_of_month_begin() const { return last_week_days_of_month_.begin(); }
-    std::vector<int>::const_iterator last_week_days_end_of_month_end() const { return last_week_days_of_month_.end(); }
-    std::vector<int>::const_iterator days_of_month_begin() const { return daysOfMonth_.begin(); }
-    std::vector<int>::const_iterator days_of_month_end() const { return daysOfMonth_.end(); }
-    std::vector<int>::const_iterator months_begin() const { return months_.begin(); }
-    std::vector<int>::const_iterator months_end() const { return months_.end(); }
+    const std::vector<int>& week_days() const { return weekDays_; }
+    const std::vector<int>& last_week_days_of_month() const { return last_week_days_of_month_; }
+    const std::vector<int>& days_of_month() const { return daysOfMonth_; }
+    const std::vector<int>& months() const { return months_; }
 
     std::string name() const; // for display/gui
     std::string toString() const;

libs/client/src/ecflow/client/ClientInvoker.hpp

@@ -458,6 +458,8 @@ class ClientInvoker {
      */
     bool is_not_retrying(const ClientToServerCmd& cmd) const;
 
+    auto& changed_node_paths() { return server_reply_.changed_nodes(); }
+
 private:
     /**
      * @return 1 when command is selected; 0 if no command is selected (e.g. --help)

libs/pyext/CMakeLists.txt

@@ -55,6 +55,7 @@ set(srcs
   ../src/ecflow/python/ClientDoc.hpp
   ../src/ecflow/python/DefsDoc.hpp
   ../src/ecflow/python/Edit.hpp
+  ../src/ecflow/python/ExportCollections.hpp
   ../src/ecflow/python/GlossaryDoc.hpp
   ../src/ecflow/python/NodeAttrDoc.hpp
   ../src/ecflow/python/NodeUtil.hpp
@@ -67,6 +68,7 @@ set(srcs
   ../src/ecflow/python/EcfExt.cpp
   ../src/ecflow/python/Edit.cpp
   ../src/ecflow/python/ExportClient.cpp
+  ../src/ecflow/python/ExportCollections.cpp
   ../src/ecflow/python/ExportCore.cpp
   ../src/ecflow/python/ExportDefs.cpp
   ../src/ecflow/python/ExportNode.cpp
@@ -127,6 +129,6 @@ set(s_tests
   s_TestSslSetup
 )
 
-if (Python3_FOUND AND (Boost_PYTHON3_FOUND OR Boost_PYTHON${Python3_VERSION_MAJOR}${Python3_VERSION_MINOR}_FOUND))
+if (Python3_FOUND AND pybind11_FOUND)
   add_subdirectory( python3 )
 endif()

libs/pyext/python3/CMakeLists.txt

@@ -32,7 +32,7 @@ ecbuild_add_library(
     ecflow_all
     libsimulator
     Python3::Module
-    Boost::${ECFLOW_BOOST_PYTHON_COMPONENT}
+    pybind11::headers
     $<$<BOOL:${OPENSSL_FOUND}>:OpenSSL::SSL>
   CXXFLAGS
     $<$<CXX_COMPILER_ID:Clang>:-Wno-macro-redefined>

libs/pyext/src/ecflow/python/EcfExt.cpp

@@ -8,31 +8,35 @@
  * nor does it submit to any jurisdiction.
  */
 
+#include "ecflow/core/Version.hpp"
 #include "ecflow/python/PythonBinding.hpp"
 
-void export_Core();
-void export_NodeAttr();
-void export_Node();
-void export_Task();
-void export_SuiteAndFamily();
-void export_Defs();
-void export_Client();
+void export_Collections(py::module& m);
+void export_Core(py::module& m);
+void export_NodeAttr(py::module& m);
+void export_Node(py::module& m);
+void export_Task(py::module& m);
+void export_SuiteAndFamily(py::module& m);
+void export_Defs(py::module& m);
+void export_Client(py::module& m);
 
-// See: http://wiki.python.org/moin/boost.python/HowTo#boost.function_objects
-BOOST_PYTHON_MODULE(ecflow) {
-    py::docstring_options doc_options(true, // show the docstrings from here
-                                      true, // show Python signatures.
-                                      false // Don't mention the C++ method signatures in the generated docstrings
-    );
-    py::scope().attr("__doc__") =
-        "The ecflow module provides the python bindings/api for creating definition structure "
-        "and communicating with the server.";
+PYBIND11_MODULE(ecflow, m) {
+    py::options options;
+    options.enable_user_defined_docstrings(); // show the docstrings from here
+    options.enable_function_signatures();     // show Python signatures.
+    options.enable_enum_members_docstring();
 
-    export_Core();
-    export_NodeAttr();
-    export_Node();
-    export_Task();
-    export_SuiteAndFamily();
-    export_Defs();
-    export_Client();
+    m.doc() = "The ecflow module provides the python bindings/api for creating definition structure "
+              "and communicating with the server.";
+
+    m.attr("__version__") = ecf::Version::base();
+
+    export_Collections(m);
+    export_Core(m);
+    export_NodeAttr(m);
+    export_Node(m);
+    export_Task(m);
+    export_SuiteAndFamily(m);
+    export_Defs(m);
+    export_Client(m);
 }

libs/pyext/src/ecflow/python/Edit.cpp

@@ -17,23 +17,12 @@
 Edit::Edit(const py::dict& dict) {
     pyutil_dict_to_str_vec(dict, vec_);
 }
-Edit::Edit(const py::dict& dict, const py::dict& dict2) {
-    pyutil_dict_to_str_vec(dict, vec_);
-    pyutil_dict_to_str_vec(dict2, vec_);
+
+Edit::Edit(const py::kwargs& kwargs) {
+    pyutil_dict_to_str_vec(kwargs, vec_);
 }
 
-py::object Edit::init(py::tuple args, py::dict kw) {
-    // cout << "Edit::init args: " << len(args) << " kwargs " << len(kw) << "\n";
-    //  args[0] is Edit(i.e self)
-    for (int i = 1; i < len(args); ++i) {
-        if (py::extract<py::dict>(args[i]).check()) {
-            py::dict d = py::extract<py::dict>(args[i]);
-            return args[0].attr("__init__")(d, kw); // calls -> .def(init<dict,dict>() -> Edit(dict,dict)
-        }
-        else {
-            throw std::runtime_error("Edit::Edit: only accepts dictionary and key word arguments");
-        }
-    }
-    py::tuple rest(args.slice(1, py::_));
-    return args[0].attr("__init__")(kw); // calls -> .def(init<dict>() -> Edit(const py::dict& dict)
+Edit::Edit(const py::dict& dict, const py::kwargs& kwargs) {
+    pyutil_dict_to_str_vec(dict, vec_);
+    pyutil_dict_to_str_vec(kwargs, vec_);
 }

libs/pyext/src/ecflow/python/Edit.hpp

@@ -19,10 +19,12 @@
 class Edit {
 public:
     explicit Edit(const py::dict& dict);
-    Edit(const py::dict& dict, const py::dict& dict2);
+    explicit Edit(const py::kwargs& kw);
+    Edit(const py::dict& dict, const py::kwargs& kw);
+
     const std::vector<Variable>& variables() const { return vec_; }
+
     static std::string to_string() { return "edit"; }
-    static py::object init(py::tuple args, py::dict kw);
 
 private:
     std::vector<Variable> vec_;