From 71aa4b015743335fd8928853cab1890f3ff26dee Mon Sep 17 00:00:00 2001 From: Marcos Bento Date: Sat, 19 Jul 2025 07:04:23 +0100 Subject: [PATCH 01/14] Replace Boost.python with Pybind11 --- cmake/Dependencies.cmake | 34 +- .../src/ecflow/attribute/CronAttr.hpp | 12 +- .../src/ecflow/client/ClientInvoker.hpp | 2 + libs/pyext/CMakeLists.txt | 4 +- libs/pyext/python3/CMakeLists.txt | 2 +- libs/pyext/script.py | 11 - libs/pyext/src/ecflow/python/EcfExt.cpp | 47 +- libs/pyext/src/ecflow/python/Edit.cpp | 23 +- libs/pyext/src/ecflow/python/Edit.hpp | 6 +- libs/pyext/src/ecflow/python/ExportClient.cpp | 202 ++-- .../src/ecflow/python/ExportCollections.cpp | 29 + .../src/ecflow/python/ExportCollections.hpp | 29 + libs/pyext/src/ecflow/python/ExportCore.cpp | 250 ++--- libs/pyext/src/ecflow/python/ExportDefs.cpp | 157 ++- libs/pyext/src/ecflow/python/ExportNode.cpp | 273 ++--- .../src/ecflow/python/ExportNodeAttr.cpp | 973 +++++++++--------- .../ecflow/python/ExportSuiteAndFamily.cpp | 78 +- libs/pyext/src/ecflow/python/ExportTask.cpp | 55 +- libs/pyext/src/ecflow/python/NodeAttrDoc.cpp | 45 + libs/pyext/src/ecflow/python/NodeAttrDoc.hpp | 5 + libs/pyext/src/ecflow/python/NodeUtil.cpp | 254 +++-- libs/pyext/src/ecflow/python/NodeUtil.hpp | 19 +- .../pyext/src/ecflow/python/PythonBinding.hpp | 34 +- libs/pyext/src/ecflow/python/PythonUtil.cpp | 75 +- libs/pyext/src/ecflow/python/Trigger.cpp | 17 +- libs/pyext/test/py_u_TestAddDelete.py | 29 +- libs/pyext/test/py_u_TestAddDeleteFunc.py | 26 +- libs/pyext/test/py_u_TestCopy.py | 25 +- 28 files changed, 1420 insertions(+), 1296 deletions(-) delete mode 100644 libs/pyext/script.py create mode 100644 libs/pyext/src/ecflow/python/ExportCollections.cpp create mode 100644 libs/pyext/src/ecflow/python/ExportCollections.hpp diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index 1374efcfc..5719d81d7 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -123,6 +123,25 @@ if (ENABLE_PYTHON) endif() +# ========================================================================================= +# pybind11 +# ========================================================================================= +if (ENABLE_PYTHON) + + ecbuild_info( "Locating pybind11" ) + + find_package(pybind11 REQUIRED) + + ecbuild_info( "pybind11 details:" ) + ecbuild_info( " * pybind11_FOUND : ${pybind11_FOUND}" ) + ecbuild_info( " * pybind11_INCLUDE_DIRS : ${pybind11_INCLUDE_DIRS}" ) + ecbuild_info( " * pybind11_VERSION : ${pybind11_VERSION}" ) + ecbuild_info( " * pybind11_VERSION_MAJOR: ${pybind11_VERSION_MAJOR}" ) + ecbuild_info( " * pybind11_VERSION_MINOR: ${pybind11_VERSION_MINOR}" ) + ecbuild_info( " * pybind11_VERSION_PATCH: ${pybind11_VERSION_PATCH}" ) + +endif() + # ========================================================================================= # Boost # ========================================================================================= @@ -174,21 +193,6 @@ if ( Boost_MINOR_VERSION GREATER_EQUAL 86 ) list(APPEND _boost_needed_libs process) endif() -if (ENABLE_PYTHON) - # The following is used to find Boost.python library, as the library name changes with python version - if ( Boost_MINOR_VERSION GREATER 66 ) - # cmake 3.15 - # see: https://gitlab.kitware.com/cmake/cmake/issues/19656 - # INTERFACE_LIBRARY targets may only have whitelisted properties. - set(_python_base_version "${Python3_VERSION_MAJOR}${Python3_VERSION_MINOR}") - else() - set(_python_base_version "${Python3_VERSION_MAJOR}") - endif() - set(ECFLOW_BOOST_PYTHON_COMPONENT "python${_python_base_version}") - - list(APPEND _boost_needed_libs ${ECFLOW_BOOST_PYTHON_COMPONENT}) -endif() - if(HAVE_TESTS) # HAVE_TESTS is defined if ecbuild ENABLE_TESTS is set, (by default this is set) list(APPEND _boost_needed_libs unit_test_framework test_exec_monitor ) endif() diff --git a/libs/attribute/src/ecflow/attribute/CronAttr.hpp b/libs/attribute/src/ecflow/attribute/CronAttr.hpp index ff1ff71be..f5d115d5e 100644 --- a/libs/attribute/src/ecflow/attribute/CronAttr.hpp +++ b/libs/attribute/src/ecflow/attribute/CronAttr.hpp @@ -77,14 +77,10 @@ class CronAttr { const TimeSeries& time() const { return timeSeries_; } const TimeSeries& time_series() const { return timeSeries_; } bool last_day_of_month() const { return last_day_of_month_; } - std::vector::const_iterator week_days_begin() const { return weekDays_.begin(); } - std::vector::const_iterator week_days_end() const { return weekDays_.end(); } - std::vector::const_iterator last_week_days_of_month_begin() const { return last_week_days_of_month_.begin(); } - std::vector::const_iterator last_week_days_end_of_month_end() const { return last_week_days_of_month_.end(); } - std::vector::const_iterator days_of_month_begin() const { return daysOfMonth_.begin(); } - std::vector::const_iterator days_of_month_end() const { return daysOfMonth_.end(); } - std::vector::const_iterator months_begin() const { return months_.begin(); } - std::vector::const_iterator months_end() const { return months_.end(); } + const std::vector& week_days() const { return weekDays_; } + const std::vector& last_week_days_of_month() const { return last_week_days_of_month_; } + const std::vector& days_of_month() const { return daysOfMonth_; } + const std::vector& months() const { return months_; } std::string name() const; // for display/gui std::string toString() const; diff --git a/libs/client/src/ecflow/client/ClientInvoker.hpp b/libs/client/src/ecflow/client/ClientInvoker.hpp index 74ad2ec21..c02628539 100644 --- a/libs/client/src/ecflow/client/ClientInvoker.hpp +++ b/libs/client/src/ecflow/client/ClientInvoker.hpp @@ -458,6 +458,8 @@ class ClientInvoker { */ bool is_not_retrying(const ClientToServerCmd& cmd) const; + auto& changed_node_paths() { return server_reply_.changed_nodes(); } + private: /** * @return 1 when command is selected; 0 if no command is selected (e.g. --help) diff --git a/libs/pyext/CMakeLists.txt b/libs/pyext/CMakeLists.txt index d47781a0b..2a57dd6b4 100644 --- a/libs/pyext/CMakeLists.txt +++ b/libs/pyext/CMakeLists.txt @@ -55,6 +55,7 @@ set(srcs ../src/ecflow/python/ClientDoc.hpp ../src/ecflow/python/DefsDoc.hpp ../src/ecflow/python/Edit.hpp + ../src/ecflow/python/ExportCollections.hpp ../src/ecflow/python/GlossaryDoc.hpp ../src/ecflow/python/NodeAttrDoc.hpp ../src/ecflow/python/NodeUtil.hpp @@ -67,6 +68,7 @@ set(srcs ../src/ecflow/python/EcfExt.cpp ../src/ecflow/python/Edit.cpp ../src/ecflow/python/ExportClient.cpp + ../src/ecflow/python/ExportCollections.cpp ../src/ecflow/python/ExportCore.cpp ../src/ecflow/python/ExportDefs.cpp ../src/ecflow/python/ExportNode.cpp @@ -127,6 +129,6 @@ set(s_tests s_TestSslSetup ) -if (Python3_FOUND AND (Boost_PYTHON3_FOUND OR Boost_PYTHON${Python3_VERSION_MAJOR}${Python3_VERSION_MINOR}_FOUND)) +if (Python3_FOUND AND pybind11_FOUND) add_subdirectory( python3 ) endif() diff --git a/libs/pyext/python3/CMakeLists.txt b/libs/pyext/python3/CMakeLists.txt index a4a5bb4c0..497a16698 100644 --- a/libs/pyext/python3/CMakeLists.txt +++ b/libs/pyext/python3/CMakeLists.txt @@ -32,7 +32,7 @@ ecbuild_add_library( ecflow_all libsimulator Python3::Module - Boost::${ECFLOW_BOOST_PYTHON_COMPONENT} + pybind11::headers $<$:OpenSSL::SSL> CXXFLAGS $<$:-Wno-macro-redefined> diff --git a/libs/pyext/script.py b/libs/pyext/script.py deleted file mode 100644 index ee6fbbe5b..000000000 --- a/libs/pyext/script.py +++ /dev/null @@ -1,11 +0,0 @@ -## Copyright 2009- ECMWF. -## This software is licensed under the terms of the Apache Licence version 2.0 -## which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. -## In applying this licence, ECMWF does not waive the privileges and immunities -## granted to it by virtue of its status as an intergovernmental organisation -## nor does it submit to any jurisdiction. - -# -# -print 'Hello World !' -number = 42 diff --git a/libs/pyext/src/ecflow/python/EcfExt.cpp b/libs/pyext/src/ecflow/python/EcfExt.cpp index 46d36fd0d..34a9fee19 100644 --- a/libs/pyext/src/ecflow/python/EcfExt.cpp +++ b/libs/pyext/src/ecflow/python/EcfExt.cpp @@ -10,29 +10,30 @@ #include "ecflow/python/PythonBinding.hpp" -void export_Core(); -void export_NodeAttr(); -void export_Node(); -void export_Task(); -void export_SuiteAndFamily(); -void export_Defs(); -void export_Client(); +void export_Collections(py::module& m); +void export_Core(py::module& m); +void export_NodeAttr(py::module& m); +void export_Node(py::module& m); +void export_Task(py::module& m); +void export_SuiteAndFamily(py::module& m); +void export_Defs(py::module& m); +void export_Client(py::module& m); -// See: http://wiki.python.org/moin/boost.python/HowTo#boost.function_objects -BOOST_PYTHON_MODULE(ecflow) { - py::docstring_options doc_options(true, // show the docstrings from here - true, // show Python signatures. - false // Don't mention the C++ method signatures in the generated docstrings - ); - py::scope().attr("__doc__") = - "The ecflow module provides the python bindings/api for creating definition structure " - "and communicating with the server."; +PYBIND11_MODULE(ecflow, m) { + py::options options; + options.enable_user_defined_docstrings(); // show the docstrings from here + options.enable_function_signatures(); // show Python signatures. + options.enable_enum_members_docstring(); - export_Core(); - export_NodeAttr(); - export_Node(); - export_Task(); - export_SuiteAndFamily(); - export_Defs(); - export_Client(); + m.doc() = "The ecflow module provides the python bindings/api for creating definition structure " + "and communicating with the server."; + + export_Collections(m); + export_Core(m); + export_NodeAttr(m); + export_Node(m); + export_Task(m); + export_SuiteAndFamily(m); + export_Defs(m); + export_Client(m); } diff --git a/libs/pyext/src/ecflow/python/Edit.cpp b/libs/pyext/src/ecflow/python/Edit.cpp index ce95e16a6..842103d24 100644 --- a/libs/pyext/src/ecflow/python/Edit.cpp +++ b/libs/pyext/src/ecflow/python/Edit.cpp @@ -17,23 +17,12 @@ Edit::Edit(const py::dict& dict) { pyutil_dict_to_str_vec(dict, vec_); } -Edit::Edit(const py::dict& dict, const py::dict& dict2) { - pyutil_dict_to_str_vec(dict, vec_); - pyutil_dict_to_str_vec(dict2, vec_); + +Edit::Edit(const py::kwargs& kwargs) { + pyutil_dict_to_str_vec(kwargs, vec_); } -py::object Edit::init(py::tuple args, py::dict kw) { - // cout << "Edit::init args: " << len(args) << " kwargs " << len(kw) << "\n"; - // args[0] is Edit(i.e self) - for (int i = 1; i < len(args); ++i) { - if (py::extract(args[i]).check()) { - py::dict d = py::extract(args[i]); - return args[0].attr("__init__")(d, kw); // calls -> .def(init() -> Edit(dict,dict) - } - else { - throw std::runtime_error("Edit::Edit: only accepts dictionary and key word arguments"); - } - } - py::tuple rest(args.slice(1, py::_)); - return args[0].attr("__init__")(kw); // calls -> .def(init() -> Edit(const py::dict& dict) +Edit::Edit(const py::dict& dict, const py::kwargs& kwargs) { + pyutil_dict_to_str_vec(dict, vec_); + pyutil_dict_to_str_vec(kwargs, vec_); } diff --git a/libs/pyext/src/ecflow/python/Edit.hpp b/libs/pyext/src/ecflow/python/Edit.hpp index 2b1ff3d6f..54aece0bd 100644 --- a/libs/pyext/src/ecflow/python/Edit.hpp +++ b/libs/pyext/src/ecflow/python/Edit.hpp @@ -19,10 +19,12 @@ class Edit { public: explicit Edit(const py::dict& dict); - Edit(const py::dict& dict, const py::dict& dict2); + explicit Edit(const py::kwargs& kw); + Edit(const py::dict& dict, const py::kwargs& kw); + const std::vector& variables() const { return vec_; } + static std::string to_string() { return "edit"; } - static py::object init(py::tuple args, py::dict kw); private: std::vector vec_; diff --git a/libs/pyext/src/ecflow/python/ExportClient.cpp b/libs/pyext/src/ecflow/python/ExportClient.cpp index 20e4857cd..85c7c1ef8 100644 --- a/libs/pyext/src/ecflow/python/ExportClient.cpp +++ b/libs/pyext/src/ecflow/python/ExportClient.cpp @@ -25,8 +25,6 @@ #include "ecflow/python/PythonBinding.hpp" #include "ecflow/python/PythonUtil.hpp" -// See: http://wiki.python.org/moin/boost.python/HowTo#boost.function_objects - void set_host_port(ClientInvoker* self, const std::string& host, int port) { self->set_host_port(host, ecf::convert_to(port)); } @@ -55,8 +53,6 @@ const std::string& query1(ClientInvoker* self, const std::string& query_type, co return self->get_string(); } -// const std::string& get_log(ClientInvoker* self) { self->getLog(); return self->get_string();} - const std::string& get_log(ClientInvoker* self, int lastLines) { self->getLog(lastLines); return self->get_string(); @@ -94,16 +90,16 @@ int edit_script_submit(ClientInvoker* self, namespace /* __ANONYMOUS__ */ { -py::object convert_to_pyobject(const std::string& s, bool as_bytes) { - py::object result; +py::object convert_string_to_pyobject(const std::string& s, bool as_bytes) { + PyObject* content = nullptr; if (as_bytes) { - result = py::object(py::handle<>(PyBytes_FromObject( - PyMemoryView_FromMemory(const_cast(s.data()), static_cast(s.size()), PyBUF_READ)))); + content = PyBytes_FromObject( + PyMemoryView_FromMemory(const_cast(s.data()), static_cast(s.size()), PyBUF_READ)); } else { - result = py::object(py::handle<>(PyUnicode_FromStringAndSize(s.data(), static_cast(s.size())))); + content = PyUnicode_FromStringAndSize(s.data(), static_cast(s.size())); } - return result; + return py::reinterpret_steal(content); } } // namespace @@ -116,7 +112,7 @@ py::object get_file(ClientInvoker* self, self->file(absNodePath, file_type, max_lines); const std::string& s = self->get_string(); - return convert_to_pyobject(s, as_bytes); + return convert_string_to_pyobject(s, as_bytes); } /// Set the CLI to enable output to standard out @@ -139,7 +135,6 @@ const std::string& stats(ClientInvoker* self, bool to_stdout = true) { } return self->server_reply().get_string(); } -BOOST_PYTHON_FUNCTION_OVERLOADS(stats_overloads, stats, 1, 2) void stats_reset(ClientInvoker* self) { self->stats_reset(); @@ -510,13 +505,15 @@ void client_invoker_enable_ssl(ClientInvoker* self) { } #endif -void export_Client() { +void export_Client(py::module& m) { + // Need std::shared_ptr, to add support for with( __enter__,__exit__) - py::class_, boost::noncopyable>("Client", ClientDoc::class_client()) - .def("__init__", py::make_constructor(client_invoker_make<>)) - .def("__init__", py::make_constructor(client_invoker_make)) - .def("__init__", py::make_constructor(client_invoker_make)) - .def("__init__", py::make_constructor(client_invoker_make)) + py::class_>(m, "Client", ClientDoc::class_client()) + + .def(py::init(&client_invoker_make<>)) + .def(py::init(&client_invoker_make)) + .def(py::init(&client_invoker_make)) + .def(py::init(&client_invoker_make)) .def("__enter__", &client_enter) // allow with statement .def("__exit__", &client_exit) // allow with statement, remove last handle .def("version", &version, "Returns the current client version") @@ -531,11 +528,11 @@ void export_Client() { .def("set_host_port", &set_host_port) .def("get_host", &ClientInvoker::host, - py::return_value_policy(), + py::return_value_policy::reference, "Return the host, assume set_host_port() has been set, otherwise return localhost") .def("get_port", &ClientInvoker::port, - py::return_value_policy(), + py::return_value_policy::reference, "Return the port, assume set_host_port() has been set. otherwise returns 3141") .def("set_retry_connection_period", set_retry_connection_period, ClientDoc::set_retry_connection_period()) .def("set_connection_attempts", &ClientInvoker::set_connection_attempts, ClientDoc::set_connection_attempts()) @@ -546,17 +543,14 @@ void export_Client() { .def("get_defs", &ClientInvoker::defs, ClientDoc::get_defs()) .def("reset", &ClientInvoker::reset, "reset client definition, and handle number") .def("in_sync", &ClientInvoker::in_sync, ClientDoc::in_sync()) - .def("get_log", &get_log, py::return_value_policy(), ClientDoc::get_log()) - .def("edit_script_edit", - &edit_script_edit, - py::return_value_policy(), - ClientDoc::edit_script_edit()) + .def("get_log", &get_log, py::return_value_policy::reference, ClientDoc::get_log()) + .def("edit_script_edit", &edit_script_edit, py::return_value_policy::reference, ClientDoc::edit_script_edit()) .def("edit_script_preprocess", &edit_script_preprocess, - py::return_value_policy(), + py::return_value_policy::reference, ClientDoc::edit_script_preprocess()) .def("edit_script_submit", &edit_script_submit, ClientDoc::edit_script_submit()) - .def("new_log", &ClientInvoker::new_log, (py::arg("path") = ""), ClientDoc::new_log()) + .def("new_log", &ClientInvoker::new_log, py::arg("path") = "", ClientDoc::new_log()) .def("clear_log", &ClientInvoker::clearLog, ClientDoc::clear_log()) .def("flush_log", &ClientInvoker::flushLog, ClientDoc::flush_log()) .def("log_msg", &ClientInvoker::logMsg, ClientDoc::log_msg()) @@ -566,23 +560,22 @@ void export_Client() { .def("terminate_server", &ClientInvoker::terminateServer, ClientDoc::terminate_server()) .def("wait_for_server_reply", &ClientInvoker::wait_for_server_reply, - (py::arg("time_out") = 60), + py::arg("time_out") = 60, ClientDoc::wait_for_server_reply()) .def("load", &ClientInvoker::loadDefs, - (py::arg("path_to_defs"), - py::arg("force") = false, - py::arg("check_only") = false, - py::arg("print") = false, - py::arg("stats") = false), + py::arg("path_to_defs"), + py::arg("force") = false, + py::arg("check_only") = false, + py::arg("print") = false, + py::arg("stats") = false, ClientDoc::load_defs()) - .def("load", &ClientInvoker::load, (py::arg("defs"), py::arg("force") = false), ClientDoc::load()) + .def("load", &ClientInvoker::load, py::arg("defs"), py::arg("force") = false, ClientDoc::load()) .def("get_server_defs", &ClientInvoker::getDefs, ClientDoc::get_server_defs()) - .def("sync_local", &ClientInvoker::sync_local, (py::arg("sync_suite_clock") = false), ClientDoc::sync()) + .def("sync_local", &ClientInvoker::sync_local, py::arg("sync_suite_clock") = false, ClientDoc::sync()) .def("news_local", &news_local, ClientDoc::news()) - .add_property("changed_node_paths", - py::range(&ClientInvoker::changed_node_paths_begin, &ClientInvoker::changed_node_paths_end), - ClientDoc::changed_node_paths()) + .def_property_readonly( + "changed_node_paths", &ClientInvoker::changed_node_paths, ClientDoc::changed_node_paths()) .def("suites", &suites, ClientDoc::suites()) .def("ch_register", &ch_register, ClientDoc::ch_register()) .def("ch_suites", &ch_suites, ClientDoc::ch_suites()) @@ -598,9 +591,9 @@ void export_Client() { .def("ch_auto_add", &ClientInvoker::ch1_auto_add) .def("checkpt", &ClientInvoker::checkPtDefs, - (py::arg("mode") = ecf::CheckPt::UNDEFINED, - py::arg("check_pt_interval") = 0, - py::arg("check_pt_save_alarm_time") = 0), + py::arg("mode") = ecf::CheckPt::UNDEFINED, + py::arg("check_pt_interval") = 0, + py::arg("check_pt_save_alarm_time") = 0, ClientDoc::checkpt()) .def("restore_from_checkpt", &ClientInvoker::restoreDefsFromCheckPt, ClientDoc::restore_from_checkpt()) .def("reload_wl_file", &ClientInvoker::reloadwsfile, ClientDoc::reload_wl_file()) @@ -609,8 +602,8 @@ void export_Client() { &ClientInvoker::reloadcustompasswdfile, "reload the custom passwd file. ..ecf.custom_passwd. For users using ECF_USER or --user or " "set_user_name()") - .def("requeue", &requeue, (py::arg("abs_node_path"), py::arg("option") = ""), ClientDoc::requeue()) - .def("requeue", &requeues, (py::arg("paths"), py::arg("option") = "")) + .def("requeue", &requeue, py::arg("abs_node_path"), py::arg("option") = "", ClientDoc::requeue()) + .def("requeue", &requeues, py::arg("paths"), py::arg("option") = "") .def("free_trigger_dep", &free_trigger_dep, ClientDoc::free_trigger_dep()) .def("free_trigger_dep", &free_trigger_dep1) .def("free_date_dep", &free_date_dep, ClientDoc::free_date_dep()) @@ -621,37 +614,42 @@ void export_Client() { .def("free_all_dep", &free_all_dep1) .def("ping", &ClientInvoker::pingServer, ClientDoc::ping()) .def("stats", - &stats, - stats_overloads(py::args("to_stdout"), - ClientDoc::stats())[py::return_value_policy()]) + &stats, // This prints to stdout, so we need to use a call guard to redirect output + py::arg("to_stdout") = true, + py::call_guard(), + ClientDoc::stats()) .def("stats_reset", &stats_reset, ClientDoc::stats_reset()) .def("get_file", &get_file, - (py::arg("task"), py::arg("type") = "script", py::arg("max_lines") = "10000", py::arg("as_bytes") = false), + py::arg("task"), + py::arg("type") = "script", + py::arg("max_lines") = "10000", + py::arg("as_bytes") = false, ClientDoc::get_file()) .def("plug", &ClientInvoker::plug, ClientDoc::plug()) - .def("query", &query, py::return_value_policy(), ClientDoc::query()) - .def("query", &query1, py::return_value_policy(), ClientDoc::query()) + .def("query", &query, py::return_value_policy::reference, ClientDoc::query()) + .def("query", &query1, py::return_value_policy::reference, ClientDoc::query()) .def("alter", &alters, - (py::arg("paths"), - py::arg("alter_type"), - py::arg("attribute_type"), - py::arg("name") = "", - py::arg("value") = ""), + py::arg("paths"), + py::arg("alter_type"), + py::arg("attribute_type"), + py::arg("name") = "", + py::arg("value") = "", ClientDoc::alter()) .def("alter", &alter, - (py::arg("abs_node_path"), - py::arg("alter_type"), - py::arg("attribute_type"), - py::arg("name") = "", - py::arg("value") = "")) + py::arg("abs_node_path"), + py::arg("alter_type"), + py::arg("attribute_type"), + py::arg("name") = "", + py::arg("value") = "") .def("sort_attributes", &alter_sort, - (py::arg("abs_node_path"), py::arg("attribute_name"), py::arg("recursive") = true)) - .def( - "sort_attributes", &alter_sorts, (py::arg("paths"), py::arg("attribute_name"), py::arg("recursive") = true)) + py::arg("abs_node_path"), + py::arg("attribute_name"), + py::arg("recursive") = true) + .def("sort_attributes", &alter_sorts, py::arg("paths"), py::arg("attribute_name"), py::arg("recursive") = true) .def("force_event", &force_event, ClientDoc::force_event()) .def("force_event", &force_events) .def("force_state", &force_state, ClientDoc::force_state()) @@ -666,14 +664,15 @@ void export_Client() { .def("group", &ClientInvoker::group, ClientDoc::group()) .def("begin_suite", &ClientInvoker::begin, - (py::arg("suite_name"), py::arg("force") = false), + py::arg("suite_name"), + py::arg("force") = false, ClientDoc::begin_suite()) - .def("begin_all_suites", &ClientInvoker::begin_all_suites, (py::arg("force") = false), ClientDoc::begin_all()) + .def("begin_all_suites", &ClientInvoker::begin_all_suites, py::arg("force") = false, ClientDoc::begin_all()) .def("job_generation", &ClientInvoker::job_gen, ClientDoc::job_gen()) .def("run", &run, ClientDoc::run()) .def("run", &runs) - .def("check", &check, py::return_value_policy(), ClientDoc::check()) - .def("check", &checks, py::return_value_policy()) + .def("check", &check, py::return_value_policy::reference, ClientDoc::check()) + .def("check", &checks, py::return_value_policy::reference) .def("kill", &do_kill, ClientDoc::kill()) .def("kill", &do_kills) .def("status", &the_status, ClientDoc::status()) @@ -688,10 +687,11 @@ void export_Client() { .def("restore", &restores) .def("delete", &ClientInvoker::delete_node, - (py::arg("abs_node_path"), py::arg("force") = false), + py::arg("abs_node_path"), + py::arg("force") = false, ClientDoc::delete_node()) - .def("delete", &delete_node, (py::arg("paths"), py::arg("force") = false)) - .def("delete_all", &ClientInvoker::delete_all, (py::arg("force") = false), ClientDoc::delete_all()) + .def("delete", &delete_node, py::arg("paths"), py::arg("force") = false) + .def("delete_all", &ClientInvoker::delete_all, py::arg("force") = false, ClientDoc::delete_all()) .def("debug_server_on", &ClientInvoker::debug_server_on, "Enable server debug, Will dump to standard out on server host.") @@ -708,7 +708,7 @@ void export_Client() { .def("enable_http", &ClientInvoker::enable_http, "Enable HTTP communication") .def("enable_https", &ClientInvoker::enable_https, "Enable HTTPS communication") - .def("zombie_get", &zombieGet, py::return_value_policy()) + .def("zombie_get", &zombieGet, py::return_value_policy::reference) .def("zombie_fob", &ClientInvoker::zombieFobCli) .def("zombie_fail", &ClientInvoker::zombieFailCli) .def("zombie_adopt", &ClientInvoker::zombieAdoptCli) @@ -739,11 +739,12 @@ void export_Client() { .def("child_init", &ClientInvoker::child_init, "Child command,notify server job has started") .def("child_abort", &ClientInvoker::child_abort, - (py::arg("reason") = ""), + py::arg("reason") = "", "Child command,notify server job has aborted, can provide an optional reason") .def("child_event", &ClientInvoker::child_event, - (py::arg("event_name"), py::arg("value") = true), + py::arg("event_name"), + py::arg("value") = true, "Child command,notify server event occurred, requires the event name") .def("child_meter", &ClientInvoker::child_meter, @@ -754,35 +755,39 @@ void export_Client() { .def("child_wait", &ClientInvoker::child_wait, "Child command,wait for expression to come true") .def("child_queue", &ClientInvoker::child_queue, - (py::arg("queue_name"), py::arg("action"), py::arg("step") = "", py::arg("path_to_node_with_queue") = ""), + py::arg("queue_name"), + py::arg("action"), + py::arg("step") = "", + py::arg("path_to_node_with_queue") = "", "Child command,active:return current step as string, then increment index, requires queue name, and " "optionally path to node with the queue") .def("child_complete", &ClientInvoker::child_complete, "Child command,notify server job has complete"); - py::class_("WhyCmd", - "The why command reports, the reason why a node is not running.\n\n" - "It needs the definition structure and the path to node\n" - "\nConstructor::\n\n" - " WhyCmd(defs, node_path)\n" - " defs_ptr defs : pointer to a definition structure\n" - " string node_path : The node path\n\n" - "\nExceptions:\n\n" - "- raises RuntimeError if the definition is empty\n" - "- raises RuntimeError if the node path is empty\n" - "- raises RuntimeError if the node path cannot be found in the definition\n" - "\nUsage::\n\n" - " try:\n" - " ci = Client()\n" - " ci.sync_local()\n" - " ask = WhyCmd(ci.get_defs(),'/suite/family')\n" - " print(ask.why())\n" - " except RuntimeError, e:\n" - " print(str(e))\n\n", - py::init()) + constexpr const char* whycmd_docs = "The why command reports, the reason why a node is not running.\n\n" + "It needs the definition structure and the path to node\n" + "\nConstructor::\n\n" + " WhyCmd(defs, node_path)\n" + " defs_ptr defs : pointer to a definition structure\n" + " string node_path : The node path\n\n" + "\nExceptions:\n\n" + "- raises RuntimeError if the definition is empty\n" + "- raises RuntimeError if the node path is empty\n" + "- raises RuntimeError if the node path cannot be found in the definition\n" + "\nUsage::\n\n" + " try:\n" + " ci = Client()\n" + " ci.sync_local()\n" + " ask = WhyCmd(ci.get_defs(),'/suite/family')\n" + " print(ask.why())\n" + " except RuntimeError, e:\n" + " print(str(e))\n\n"; + + py::class_(m, "WhyCmd", whycmd_docs) + + .def(py::init()) .def("why", &WhyCmd::why, "returns a '/n' separated string, with reasons why node is not running"); - py::class_( - "UrlCmd", + constexpr const char* urlcmd_docs = "Executes a command ECF_URL_CMD to display a url.\n\n" "It needs the definition structure and the path to node.\n" "\nConstructor::\n\n" @@ -811,7 +816,10 @@ void export_Client() { " url = UrlCmd(ci.get_defs(),'/suite/family/task')\n" " print(url.execute())\n" " except RuntimeError, e:\n" - " print(str(e))\n\n", - py::init()) + " print(str(e))\n\n"; + + py::class_(m, "UrlCmd", urlcmd_docs) + + .def(py::init()) .def("execute", &UrlCmd::execute, "Displays url in the chosen browser"); } diff --git a/libs/pyext/src/ecflow/python/ExportCollections.cpp b/libs/pyext/src/ecflow/python/ExportCollections.cpp new file mode 100644 index 000000000..79c2d06c1 --- /dev/null +++ b/libs/pyext/src/ecflow/python/ExportCollections.cpp @@ -0,0 +1,29 @@ +/* + * Copyright 2009- ECMWF. + * + * This software is licensed under the terms of the Apache Licence version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include "ecflow/python/ExportCollections.hpp" + +void export_Collections(py::module& m) { + // Export the vector of Variable + py::bind_vector>(m, "VariableList"); + + // Export the vector of node_ptr + py::bind_vector>( + m, "NodeVec", "Hold a list of Nodes (i.e `suite`_, `family`_ or `task`_\\ s)"); + + // Export the vector of suite_ptr + py::bind_vector>(m, "SuiteVec"); + + // Export the vector of family_ptr + py::bind_vector>(m, "FamilyVec"); + + // Export the vector of task_ptr + py::bind_vector>(m, "TaskVec"); +} diff --git a/libs/pyext/src/ecflow/python/ExportCollections.hpp b/libs/pyext/src/ecflow/python/ExportCollections.hpp new file mode 100644 index 000000000..ce26e23f2 --- /dev/null +++ b/libs/pyext/src/ecflow/python/ExportCollections.hpp @@ -0,0 +1,29 @@ +/* + * Copyright 2009- ECMWF. + * + * This software is licensed under the terms of the Apache Licence version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#ifndef ecflow_python_ExportCollections_HPP +#define ecflow_python_ExportCollections_HPP + +#include + +#include "ecflow/attribute/Variable.hpp" +#include "ecflow/node/Family.hpp" +#include "ecflow/node/Suite.hpp" +#include "ecflow/node/Task.hpp" +#include "ecflow/python/PythonBinding.hpp" + +PYBIND11_MAKE_OPAQUE(std::vector) + +PYBIND11_MAKE_OPAQUE(std::vector) +PYBIND11_MAKE_OPAQUE(std::vector) +PYBIND11_MAKE_OPAQUE(std::vector) +PYBIND11_MAKE_OPAQUE(std::vector) + +#endif /* ecflow_python_ExportCollections_HPP */ diff --git a/libs/pyext/src/ecflow/python/ExportCore.cpp b/libs/pyext/src/ecflow/python/ExportCore.cpp index ef8cfee07..fcb97e358 100644 --- a/libs/pyext/src/ecflow/python/ExportCore.cpp +++ b/libs/pyext/src/ecflow/python/ExportCore.cpp @@ -21,17 +21,6 @@ #include "ecflow/python/PythonBinding.hpp" #include "ecflow/python/PythonUtil.hpp" -// See: http://wiki.python.org/moin/boost.python/HowTo#boost.function_objects -template -struct pair_to_tuple -{ - using converter = pair_to_tuple; - using ctype = std::pair; - - static PyObject* convert(ctype const& v) { return py::incref(py::make_tuple(v.first, v.second).ptr()); } - static void register_to_python() { py::to_python_converter(); } -}; - bool debug_build() { #ifdef NDEBUG return false; @@ -40,34 +29,29 @@ bool debug_build() { #endif } -void export_Core() { - // For use in test only - py::def("debug_build", debug_build); - - // see: https://github.com/boostorg/python/blob/master/test/raw_ctor.cpp - // Uses a raw constructor approach to support pass arbitrary number arguments on the python side. - // using no_init postpones defining __init__ function until after raw_function for proper overload resolution order, - // since later defs get higher priority. - py::class_("Edit", NodeAttrDoc::variable_doc(), py::no_init) - .def("__init__", raw_function(&Edit::init, 0)) // raw_constructor -> will call -> def(init() ) - .def(py::init()) // - .def(py::init()) // - .def("__str__", &Edit::to_string) // __str__ - ; - - py::class_("File", "Utility class, Used in test only.", py::no_init) - .def("find_server", &ecf::File::find_ecf_server_path, "Provides pathname to the server") - .staticmethod("find_server") - .def("find_client", &ecf::File::find_ecf_client_path, "Provides pathname to the client") - .staticmethod("find_client") - .def("source_dir", &ecf::File::root_source_dir, "Path name to ecflow source directory") - .staticmethod("source_dir") - .def("build_dir", &ecf::File::root_build_dir, "Path name to ecflow build directory") - .staticmethod("build_dir"); - - py::enum_( - "Style", - "Style is used to control printing output for the definition\n\n" +void export_Core(py::module& m) { + // The following is used, in tests only, to detect a debug build + m.def("debug_build", debug_build); + + py::class_(m, "Edit", NodeAttrDoc::variable_doc()) + + .def(py::init()) + .def(py::init()) + .def(py::init()) + .def("__str__", &Edit::to_string); + + constexpr const char* file_docs = "Utility class, Used in test only."; + + py::class_(m, "File", file_docs) + + .def_static("find_server", &ecf::File::find_ecf_server_path, "Provides pathname to the server") + .def_static("find_client", &ecf::File::find_ecf_client_path, "Provides pathname to the client") + .def_static("source_dir", &ecf::File::root_source_dir, "Path name to ecflow source directory") + .def_static("build_dir", &ecf::File::root_build_dir, "Path name to ecflow build directory"); + + constexpr const char* printstyle_type_docs = + "Style is used to control printing output for the definition\n" + "\n" "- DEFS: This style outputs the definition file in a format that is parse-able.\n" " and can be re-loaded back into the server.\n" " Externs are automatically added.\n" @@ -75,7 +59,8 @@ void export_Core() { "- STATE: The output includes additional state information for debug\n" " This excludes the edit history\n" "- MIGRATE: Output includes structure and state, allow migration to future ecflow versions\n" - " This includes edit history. If file is reloaded no checking is done\n\n" + " This includes edit history. If file is reloaded no checking is done\n" + "\n" "The following shows a summary of the features associated with each choice\n" "\n" " ===================== ==== ===== =======\n" @@ -85,55 +70,57 @@ void export_Core() { " Checking on reload Yes Yes No\n" " Edit History No No Yes\n" " Show trigger AST No Yes No\n" - " ===================== ==== ===== =======\n") + " ===================== ==== ===== =======\n"; + + py::enum_(m, "Style", printstyle_type_docs) + .value("NOTHING", PrintStyle::NOTHING) .value("DEFS", PrintStyle::DEFS) .value("STATE", PrintStyle::STATE) .value("MIGRATE", PrintStyle::MIGRATE); - py::class_( - "PrintStyle", + constexpr const char* printstyle_docs = "Singleton used to control the print Style. See :py:class:`ecflow.Style`\n\n" "\nUsage::\n\n" " old_style = PrintStyle.get_style()\n" " PrintStyle.set_style(PrintStyle.STATE)\n" " ...\n" " print(defs) # show the node state\n" - " PrintStyle.set_style(old_style) # reset previous style\n", - py::no_init) - .def("get_style", &PrintStyleHolder::getStyle, "Returns the style, static method") - .staticmethod("get_style") - .def("set_style", &PrintStyleHolder::setStyle, "Set the style, static method") - .staticmethod("set_style"); - - py::enum_( - "CheckPt", + " PrintStyle.set_style(old_style) # reset previous style\n"; + + py::class_(m, "PrintStyle", printstyle_docs) + + .def_static("get_style", &PrintStyleHolder::getStyle, "Returns the style, static method") + .def_static("set_style", &PrintStyleHolder::setStyle, "Set the style, static method"); + + constexpr const char* checkpt_mode_docs = "CheckPt is enum that is used to control check pointing in the `ecflow_server`_\n\n" "- NEVER : Switches of check pointing\n" "- ON_TIME: `check point`_ file is saved periodically, specified by checkPtInterval. This is the default.\n" "- ALWAYS : `check point`_ file is saved after any state change, *not* recommended for large definitions\n" - "- UNDEFINED : None of the the above, used to provide default argument\n") + "- UNDEFINED : None of the the above, used to provide default argument\n"; + + py::enum_(m, "CheckPt", checkpt_mode_docs) + .value("NEVER", ecf::CheckPt::NEVER) .value("ON_TIME", ecf::CheckPt::ON_TIME) .value("ALWAYS", ecf::CheckPt::ALWAYS) .value("UNDEFINED", ecf::CheckPt::UNDEFINED); - py::class_("Ecf", "Singleton used to control ecf debugging\n\n", py::no_init) - .def("debug_equality", &Ecf::debug_equality, "Returns true if debugging of equality is enabled") - .staticmethod("debug_equality") - .def("set_debug_equality", &Ecf::set_debug_equality, "Set debugging for equality") - .staticmethod("set_debug_equality") - .def("debug_level", - &Ecf::debug_level, - "Returns integer showing debug level. debug_level > 0 will disable some warning messages") - .staticmethod("debug_level") - .def("set_debug_level", - &Ecf::set_debug_level, - "Set debug level. debug_level > 0 will disable some warning messages") - .staticmethod("set_debug_level"); - - py::enum_( - "State", + constexpr const char* ecf_docs = "Singleton used to control ecf debugging\n\n"; + + py::class_(m, "Ecf", ecf_docs) + + .def_static("debug_equality", &Ecf::debug_equality, "Returns true if debugging of equality is enabled") + .def_static("set_debug_equality", &Ecf::set_debug_equality, "Set debugging for equality") + .def_static("debug_level", + &Ecf::debug_level, + "Returns integer showing debug level. debug_level > 0 will disable some warning messages") + .def_static("set_debug_level", + &Ecf::set_debug_level, + "Set debug level. debug_level > 0 will disable some warning messages"); + + constexpr const char* nstate_state_docs = "Each `node`_ can have a status, which reflects the life cycle of a node.\n\n" "It varies as follows:\n\n" "- When the definition file is loaded into the `ecflow_server`_ the `task`_ status is `unknown`_\n" @@ -145,7 +132,10 @@ void export_Core() { "- On a successful submission the task is placed into the `active`_ state\n" "- Before a job ends, it may send other message to the server such as:\n" " Set an `event`_, Change a `meter`_, Change a `label`_, send a message to log file\n\n" - "Jobs end by becoming either `complete`_ or `aborted`_") + "Jobs end by becoming either `complete`_ or `aborted`_"; + + py::enum_(m, "State", nstate_state_docs) + .value("unknown", NState::UNKNOWN) .value("complete", NState::COMPLETE) .value("queued", NState::QUEUED) @@ -153,20 +143,22 @@ void export_Core() { .value("submitted", NState::SUBMITTED) .value("active", NState::ACTIVE); - py::enum_("DState", - "A DState is like a ecflow.State, except for the addition of SUSPENDED\n\n" - "Suspended stops job generation, and hence is an attribute of a Node.\n" - "DState can be used for setting the default state of node when it is\n" - "begun or re queued. DState is used for defining `defstatus`_.\n" - "See :py:class:`ecflow.Node.add_defstatus` and :py:class:`ecflow.Defstatus`\n" - "The default state of a `node`_ is `queued`_.\n" - "\nUsage::\n\n" - " task = ecflow.Task('t1')\n" - " task.add_defstatus(ecflow.DState.complete)" - " task = ecflow.Task('t2')\n" - " task += Defstatus('complete')\n" - " task = Task('t3',\n" - " Defstatus('complete')) # create in place\n") + constexpr const char* dstate_docs = "A DState is like a ecflow.State, except for the addition of SUSPENDED\n\n" + "Suspended stops job generation, and hence is an attribute of a Node.\n" + "DState can be used for setting the default state of node when it is\n" + "begun or re queued. DState is used for defining `defstatus`_.\n" + "See :py:class:`ecflow.Node.add_defstatus` and :py:class:`ecflow.Defstatus`\n" + "The default state of a `node`_ is `queued`_.\n" + "\nUsage::\n\n" + " task = ecflow.Task('t1')\n" + " task.add_defstatus(ecflow.DState.complete)" + " task = ecflow.Task('t2')\n" + " task += Defstatus('complete')\n" + " task = Task('t3',\n" + " Defstatus('complete')) # create in place\n"; + + py::enum_(m, "DState", dstate_docs) + .value("unknown", DState::UNKNOWN) .value("complete", DState::COMPLETE) .value("queued", DState::QUEUED) @@ -175,47 +167,49 @@ void export_Core() { .value("suspended", DState::SUSPENDED) .value("active", DState::ACTIVE); - py::class_("Defstatus", - "A `node`_ can be set with a default status other the `queued`_\n\n" - "The default state of a `node`_ is `queued`_.\n" - "This defines the state to take at 'begin' or 're-queue' time\n" - "See :py:class:`ecflow.Node.add_defstatus` and :py:class:`ecflow.DState`\n", - py::init()) + constexpr const char* defstatus_docs = "A `node`_ can be set with a default status other the `queued`_\n\n" + "The default state of a `node`_ is `queued`_.\n" + "This defines the state to take at 'begin' or 're-queue' time\n" + "See :py:class:`ecflow.Node.add_defstatus` and :py:class:`ecflow.DState`\n"; + + py::class_(m, "Defstatus", defstatus_docs) + + .def(py::init()) .def(py::init()) // constructor .def("state", &Defstatus::state) - .def("__str__", &Defstatus::to_string) // __str__ - ; + .def("__str__", &Defstatus::to_string); + + constexpr const char* sstate_state_docs = "A SState holds the `ecflow_server`_ state\n\n" + "See `server states`_"; + + py::enum_(m, "SState", sstate_state_docs) - py::enum_("SState", - "A SState holds the `ecflow_server`_ state\n\n" - "See `server states`_") .value("HALTED", SState::HALTED) .value("SHUTDOWN", SState::SHUTDOWN) .value("RUNNING", SState::RUNNING); - py::class_("TimeSlot", - "Represents a time slot.\n\n" - "It is typically used as an argument to a :py:class:`TimeSeries` or\n" - "other time dependent attributes of a node.\n" - "\n" - "\nConstructor::\n\n" - " TimeSlot(hour,min)\n" - " int hour: represent an hour:\n" - " int minute: represents a minute:\n" - "\nUsage::\n\n" - " ts = TimeSlot(10,11)\n", - py::init()) - .def("__str__", &ecf::TimeSlot::toString) // __str__ + constexpr const char* timeslot_docs = "Represents a time slot.\n\n" + "It is typically used as an argument to a :py:class:`TimeSeries` or\n" + "other time dependent attributes of a node.\n" + "\n" + "\nConstructor::\n\n" + " TimeSlot(hour,min)\n" + " int hour: represent an hour:\n" + " int minute: represents a minute:\n" + "\nUsage::\n\n" + " ts = TimeSlot(10,11)\n"; + + py::class_(m, "TimeSlot", timeslot_docs) + + .def(py::init()) + .def("__str__", &ecf::TimeSlot::toString) .def("__copy__", pyutil_copy_object) // __copy__ uses copy constructor - .def(py::self == py::self) // __eq__ - .def("hour", &ecf::TimeSlot::hour) // return int - .def("minute", &ecf::TimeSlot::minute) // return int - .def("empty", &ecf::TimeSlot::isNULL) // return bool - ; + .def(py::self == py::self) + .def("hour", &ecf::TimeSlot::hour) + .def("minute", &ecf::TimeSlot::minute) + .def("empty", &ecf::TimeSlot::isNULL); - // single slot, | start, finish, incr, bool relative to suite start - py::class_( - "TimeSeries", + constexpr const char* timeseries_docs = "A TimeSeries can hold a single time slot or a series.\n\n" "Time series can be created relative to the `suite`_ start or start of a repeating node.\n" "A Time series can be used as argument to the :py:class:`ecflow.Time`, :py:class:`ecflow.Today` and " @@ -243,10 +237,18 @@ void export_Core() { "\nExceptions:\n\n" "- Raises IndexError when an invalid time series is specified\n" "\nUsage::\n\n" - " time_series = TimeSeries(TimeSlot(10,11),False)\n", - py::init>()) - .def(py::init>()) - .def(py::init>()) + " time_series = TimeSeries(TimeSlot(10,11),False)\n"; + + // single slot, | start, finish, incr, bool relative to suite start + py::class_(m, "TimeSeries", timeseries_docs) + + .def(py::init(), py::arg("ts"), py::arg("relative") = false) + .def(py::init(), py::arg("hour"), py::arg("minute"), py::arg("relative") = false) + .def(py::init(), + py::arg("start"), + py::arg("finish"), + py::arg("increment"), + py::arg("relative") = false) .def(py::self == py::self) // __eq__ .def("__str__", &ecf::TimeSeries::toString) // __str__ .def("__copy__", pyutil_copy_object) // __copy__ uses copy constructor @@ -256,20 +258,18 @@ void export_Core() { // slot .def("start", &ecf::TimeSeries::start, - py::return_value_policy(), + py::return_value_policy::reference, "returns the start time") // returns a time slot .def("finish", &ecf::TimeSeries::finish, - py::return_value_policy(), + py::return_value_policy::reference, "returns the finish time if time series specified, else returns a NULL time slot") // returns a time slot .def("incr", &ecf::TimeSeries::incr, - py::return_value_policy(), + py::return_value_policy::reference, " returns the increment time if time series specified, else returns a NULL time slot") // returns a time // slot .def("relative", &ecf::TimeSeries::relative, "returns a boolean where true means that the time series is relative"); - - pair_to_tuple::register_to_python(); } diff --git a/libs/pyext/src/ecflow/python/ExportDefs.cpp b/libs/pyext/src/ecflow/python/ExportDefs.cpp index e485e2406..f6c668274 100644 --- a/libs/pyext/src/ecflow/python/ExportDefs.cpp +++ b/libs/pyext/src/ecflow/python/ExportDefs.cpp @@ -20,6 +20,7 @@ #include "ecflow/node/formatter/DefsWriter.hpp" #include "ecflow/python/DefsDoc.hpp" #include "ecflow/python/Edit.hpp" +#include "ecflow/python/ExportCollections.hpp" #include "ecflow/python/GlossaryDoc.hpp" #include "ecflow/python/PythonBinding.hpp" #include "ecflow/python/PythonUtil.hpp" @@ -46,19 +47,6 @@ std::string convert_to_string(const Defs& theDefs) { return buffer; } -static defs_ptr create_defs(const std::string& file_name) { - defs_ptr defs = Defs::create(); - - std::string errorMsg, warningMsg; - if (!defs->restore(file_name, errorMsg, warningMsg)) { - throw std::runtime_error(errorMsg); - } - if (!warningMsg.empty()) { - std::cerr << warningMsg; - } - return defs; -} - std::string check_defs(defs_ptr defs) { std::string error_msg; std::string warning_msg; @@ -188,54 +176,55 @@ bool defs_container(defs_ptr self, const std::string& name) { return (self->findSuite(name)) ? true : false; } -static py::object do_add(defs_ptr self, const py::object& arg) { - // std::cout << "defs::do_add \n"; - if (arg.ptr() == py::object().ptr()) { - return py::object(self); // *IGNORE* None +static py::object do_add(defs_ptr self, const py::handle& arg) { + // When arg is None, there is nothing to do... + if (arg == py::none()) { + return py::cast(self); } - else if (py::extract(arg).check()) { - self->addSuite(py::extract(arg)); + + if (auto found = py_extract(arg); found) { + self->addSuite(found.value()); } - else if (py::extract(arg).check()) { - add_variable_dict(self, py::extract(arg)); + else if (auto found = py_extract(arg); found) { + auto suite = std::make_shared(found.value()); + self->addSuite(suite); } - else if (py::extract(arg).check()) { - Edit edit = py::extract(arg); + else if (auto found = py_extract(arg); found) { + add_variable_dict(self, found.value()); + } + else if (auto found = py_extract(arg); found) { + Edit edit = found.value(); + const std::vector& vec = edit.variables(); for (const auto& i : vec) { self->server_state().add_or_update_user_variables(i.name(), i.theValue()); } } - else if (py::extract(arg).check()) { - py::list the_list = py::extract(arg); + else if (auto found = py_extract(arg); found) { + py::list the_list = found.value(); + int the_list_size = len(the_list); for (int i = 0; i < the_list_size; ++i) { (void)do_add(self, the_list[i]); // recursive } } - else if (py::extract(arg).check()) { - Variable var = py::extract(arg); + else if (auto found = py_extract(arg); found) { + Variable var = found.value(); self->server_state().add_or_update_user_variables(var.name(), var.theValue()); } else { throw std::runtime_error("ExportDefs::add : Unknown type"); } - return py::object(self); + return py::cast(self); } -static py::object add(py::tuple args, py::dict kwargs) { - int the_list_size = len(args); - defs_ptr self = py::extract(args[0]); // self - if (!self) { - throw std::runtime_error("ExportDefs::add() : first argument is not a Defs"); +static py::object defs_add(defs_ptr self, const py::args& args, const py::kwargs& kwargs) { + for (auto arg : args) { + do_add(self, arg); } + add_variable_dict(self, kwargs); - for (int i = 1; i < the_list_size; ++i) { - (void)do_add(self, args[i]); - } - (void)add_variable_dict(self, kwargs); - - return py::object(self); // return defs as python object, relies class_... for type registration + return py::cast(self); // return defs as python object, relies class_... for type registration } static py::object defs_iadd(defs_ptr self, const py::list& list) { @@ -244,58 +233,51 @@ static py::object defs_iadd(defs_ptr self, const py::list& list) { for (int i = 0; i < the_list_size; ++i) { (void)do_add(self, list[i]); } - return py::object(self); // return node_ptr as python object, relies class_... for type registration + return py::cast(self); // return defs_ptr as python object, relies class_... for type registration } static py::object defs_getattr(defs_ptr self, const std::string& attr) { - // cout << " defs_getattr self.name() : " << self->name() << " attr " << attr << "\n"; + std::cout << " defs_getattr self.name() : " << self->name() << " attr " << attr << "\n"; suite_ptr child = self->findSuite(attr); if (child) { - return py::object(child); + return py::cast(child); } Variable var = self->server_state().findVariable(attr); if (!var.empty()) { - return py::object(var); + return py::cast(var); } throw std::runtime_error(MESSAGE("ExportDefs::defs_getattr : function of name '" << attr << "' does not exist *OR* suite or defs variable")); } -py::object defs_raw_constructor(py::tuple args, py::dict kw) { - // cout << "defs_raw_constructor len(args):" << len(args) << endl; - // args[0] is Defs(i.e self) - py::list the_list; - std::string name; - for (int i = 1; i < len(args); ++i) { - if (py::extract(args[i]).check()) { - name = py::extract(args[i]); - } - else { - the_list.append(args[i]); - } +static defs_ptr defs_create(const std::string& filename) { + defs_ptr defs = Defs::create(); + + std::string errorMsg, warningMsg; + if (!defs->restore(filename, errorMsg, warningMsg)) { + throw std::runtime_error(errorMsg); } - if (!name.empty() && len(the_list) > 0) { - throw std::runtime_error("defs_raw_constructor: Can't mix string with other arguments. String argument " - "specifies a path(loads a definition from disk)"); + if (!warningMsg.empty()) { + std::cerr << warningMsg; } - return args[0].attr("__init__")(the_list, kw); // calls -> init(list attr, dict kw) + return defs; } -defs_ptr defs_init(py::list the_list, py::dict kw) { - // cout << " defs_init: the_list: " << len(the_list) << " dict: " << len(kw) << endl; +static defs_ptr defs_init(const py::args& args, const py::kwargs& kw) { defs_ptr defs = Defs::create(); - (void)add_variable_dict(defs, kw); - (void)defs_iadd(defs, the_list); + defs_add(defs, args, kw); return defs; } -void export_Defs() { - py::class_("Defs", DefsDoc::add_definition_doc(), py::init<>("Create a empty Defs")) - .def("__init__", py::raw_function(&defs_raw_constructor, 0)) // will call -> task_init - .def("__init__", py::make_constructor(&defs_init)) - .def("__init__", py::make_constructor(&create_defs), DefsDoc::add_definition_doc()) +void export_Defs(py::module& m) { + + py::class_(m, "Defs", py::dynamic_attr()) + + .def(py::init<>(), "Create an empty Defs") + .def(py::init(&defs_create), DefsDoc::add_definition_doc()) + .def(py::init(&defs_init)) .def(py::self == py::self) // __eq__ .def("__copy__", pyutil_copy_object) // __copy__ uses copy constructor .def("__str__", convert_to_string) // __str__ @@ -303,13 +285,16 @@ void export_Defs() { .def("__exit__", &defs_exit) // allow with statement, hence indentation support .def("__len__", &defs_len) // Sized protocol .def("__contains__", &defs_container) // Container protocol - .def("__iter__", py::range(&Defs::suite_begin, &Defs::suite_end)) // iterable protocol + .def( + "__iter__", + [](const Defs& s) { return py::make_iterator(s.suiteVec().begin(), s.suiteVec().end()); }, + py::keep_alive<0, 1>()) .def("__getattr__", &defs_getattr) /* Any attempt to resolve a property, method, or field name that doesn't actually exist on the object itself will be passed to __getattr__*/ .def("__iadd__", &defs_iadd) // defs += [ Suite('s1'), Edit(var='value'), Variable('a','b') [ Suite('t2') ] ] .def("__iadd__", &do_add) // defs += Suite("s1") .def("__add__", &do_add) - .def("add", raw_function(add, 1), DefsDoc::add()) + .def("add", &defs_add, DefsDoc::add()) .def("add_suite", &add_suite, DefsDoc::add_suite_doc()) .def("add_suite", &Defs::add_suite, GlossaryDoc::list()) .def("add_extern", &Defs::add_extern, DefsDoc::add_extern_doc()) @@ -323,8 +308,14 @@ void export_Defs() { .def("sort_attributes", &sort_attributes2) .def("sort_attributes", &sort_attributes3, - (py::arg("attribute_type"), py::arg("recursive") = true, py::arg("no_sort") = py::list())) - .def("sort_attributes", &Defs::sort_attributes, (py::arg("attribute_type"), py::arg("recursive") = true)) + py::arg("attribute_type"), + py::arg("recursive") = true, + py::arg("no_sort") = py::list()) + .def("sort_attributes", + &Defs::sort_attributes, + py::arg("attribute_type"), + py::arg("recursive") = true, + py::arg("no_sort") = py::list()) .def("delete_variable", &delete_variable, "An empty string will delete all user variables") .def("find_suite", &Defs::findSuite, "Given a name, find the corresponding `suite`_") .def("find_abs_node", &Defs::findAbsNode, "Given a path, find the the `node`_") @@ -354,20 +345,24 @@ void export_Defs() { .def("simulate", &simulate, DefsDoc::simulate()) .def("check_job_creation", &check_job_creation, - (py::arg("throw_on_error") = false, py::arg("verbose") = false), + py::arg("throw_on_error") = false, + py::arg("verbose") = false, DefsDoc::check_job_creation_doc()) .def("check_job_creation", &Defs::check_job_creation) .def("generate_scripts", &Defs::generate_scripts, DefsDoc::generate_scripts_doc()) .def("get_state", &Defs::state) .def("get_server_state", &get_server_state, DefsDoc::get_server_state()) - .add_property("suites", py::range(&Defs::suite_begin, &Defs::suite_end), "Returns a list of `suite`_\\ s") - .add_property("externs", py::range(&Defs::extern_begin, &Defs::extern_end), "Returns a list of `extern`_\\ s") - .add_property("user_variables", - py::range(&Defs::user_variables_begin, &Defs::user_variables_end), - "Returns a list of user defined `variable`_\\ s") - .add_property("server_variables", - py::range(&Defs::server_variables_begin, &Defs::server_variables_end), - "Returns a list of server `variable`_\\ s"); + .def_property_readonly( + "suites", + [](const Defs& s) { return py::make_iterator(s.suiteVec().begin(), s.suiteVec().end()); }, + py::keep_alive<0, 1>(), + "Returns a list of `suite`_\\ s") + .def_property_readonly("externs", &Defs::externs, "Returns a list of `extern`_\\ s") + .def_property_readonly( + "user_variables", &Defs::user_variables, "Returns a list of user defined `variable`_\\ s") + .def_property_readonly("server_variables", &Defs::server_variables, "Returns a list of server `variable`_\\ s") + + .doc() = DefsDoc::add_definition_doc(); #if ECF_ENABLE_PYTHON_PTR_REGISTER py::register_ptr_to_python(); // needed for mac and boost 1.6 diff --git a/libs/pyext/src/ecflow/python/ExportNode.cpp b/libs/pyext/src/ecflow/python/ExportNode.cpp index 2186219f7..fe041b57d 100644 --- a/libs/pyext/src/ecflow/python/ExportNode.cpp +++ b/libs/pyext/src/ecflow/python/ExportNode.cpp @@ -25,6 +25,8 @@ #include "ecflow/node/NodeAlgorithms.hpp" #include "ecflow/node/NodeContainer.hpp" #include "ecflow/python/DefsDoc.hpp" +#include "ecflow/python/Edit.hpp" +#include "ecflow/python/ExportCollections.hpp" #include "ecflow/python/NodeAttrDoc.hpp" #include "ecflow/python/NodeUtil.hpp" #include "ecflow/python/PythonBinding.hpp" @@ -48,6 +50,12 @@ node_ptr add_variable_var(node_ptr self, const Variable& var) { self->addVariable(var); return self; } +node_ptr add_variable_edit(node_ptr self, const Edit& edit) { + for (const auto& var : edit.variables()) { + self->addVariable(var); + } + return self; +} node_ptr add_event(node_ptr self, const Event& e) { self->addEvent(e); @@ -399,16 +407,25 @@ void generated_variables_using_variablelist(node_ptr self, std::vector ///////////////////////////////////////////////////////////////////////////////////////// +static py::object node_add(node_ptr self, const py::object& arg) { + NodeUtil::add(*self, arg); + return py::cast(self); +} + +static py::object node_iadd(node_ptr self, const py::object& arg) { + NodeUtil::add(*self, arg); + return py::cast(self); +} + static py::object do_rshift(node_ptr self, const py::object& arg) { - // std::cout << "do_rshift\n"; - (void)NodeUtil::do_add(self, arg); + NodeContainer* nc = self->isNodeContainer(); + if (!nc) { + throw std::runtime_error("ExportNode::do_rshift() : Can only add a child to Suite or Family"); + } - if (py::extract(arg).check()) { - NodeContainer* nc = self->isNodeContainer(); - if (!nc) { - throw std::runtime_error("ExportNode::do_rshift() : Can only add a child to Suite or Family"); - } - node_ptr child = py::extract(arg); + if (auto found = py_extract(arg); found) { + node_ptr child = found.value(); + self->addChild(child); std::vector children; nc->immediateChildren(children); @@ -429,19 +446,21 @@ static py::object do_rshift(node_ptr self, const py::object& arg) { } } } - return py::object(self); + else { + throw std::runtime_error("ExportNode::do_rshift() : Argument must be a node_ptr"); + } + return py::cast(self); } -static py::object do_lshift(node_ptr self, const py::object& arg) { - // std::cout << "do_lshift : " << self->name() << "\n"; cout << flush; - (void)NodeUtil::do_add(self, arg); - if (py::extract(arg).check()) { +static py::object do_lshift(node_ptr self, const py::object& arg) { + NodeContainer* nc = self->isNodeContainer(); + if (!nc) { + throw std::runtime_error("ExportNode::do_lshift() : Can only add a child to Suite or Family"); + } - NodeContainer* nc = self->isNodeContainer(); - if (!nc) { - throw std::runtime_error("ExportNode::do_lshift() : Can only add a child to Suite or Family"); - } - node_ptr child = py::extract(arg); + if (auto found = py_extract(arg); found) { + node_ptr child = found.value(); + self->addChild(child); std::vector children; nc->immediateChildren(children); @@ -466,62 +485,51 @@ static py::object do_lshift(node_ptr self, const py::object& arg) { } } } - return py::object(self); + else { + throw std::runtime_error("ExportNode::do_lshift() : Argument must be a node_ptr"); + } + return py::cast(self); } -static py::object add(py::tuple args, py::dict kwargs) { - int the_list_size = len(args); - node_ptr self = py::extract(args[0]); // self - if (!self) { - throw std::runtime_error("ExportNode::add() : first argument is not a node"); - } +static py::object add(node_ptr self, const py::args& args, const py::kwargs& kwargs) { - for (int i = 1; i < the_list_size; ++i) { - (void)NodeUtil::do_add(self, args[i]); - } + NodeUtil::add(*self, args); + NodeUtil::add(*self, kwargs); - // key word arguments are use for adding variable only - (void)NodeUtil::add_variable_dict(self, kwargs); - - return py::object(self); // return node_ptr as python object, relies class_... for type registration + return py::cast(self); // return node_ptr as python object, relies class_... for type registration } static py::object node_getattr(node_ptr self, const std::string& attr) { - // cout << " node_getattr self.name() : " << self->name() << " attr " << attr << "\n"; - size_t pos = 0; - node_ptr child = self->findImmediateChild(attr, pos); - if (child) { - return py::object(child); + + size_t pos = 0; + if (node_ptr child = self->findImmediateChild(attr, pos); child) { + return py::cast(child); } - const Variable& var = self->findVariable(attr); - if (!var.empty()) { - return py::object(var); + if (const Variable& var = self->findVariable(attr); !var.empty()) { + return py::cast(var); } - const Variable& gvar = self->findGenVariable(attr); - if (!gvar.empty()) { - return py::object(gvar); + if (const Variable& gvar = self->findGenVariable(attr); !gvar.empty()) { + return py::cast(gvar); } - const Event& event = self->findEventByNameOrNumber(attr); - if (!event.empty()) { - return py::object(event); + if (const Event& event = self->findEventByNameOrNumber(attr); !event.empty()) { + return py::cast(event); } - const Meter& meter = self->findMeter(attr); - if (!meter.empty()) { - return py::object(meter); + if (const Meter& meter = self->findMeter(attr); !meter.empty()) { + return py::cast(meter); } - limit_ptr limit = self->find_limit(attr); - if (limit.get()) { - return py::object(limit); + if (limit_ptr limit = self->find_limit(attr); limit.get()) { + return py::cast(limit); } - throw std::runtime_error(MESSAGE( - "ExportNode::node_getattr: function of name '" - << attr << "' does not exist *OR* child node,variable,meter,event or limit on node " << self->absNodePath())); + throw std::runtime_error(MESSAGE("ExportNode::node_getattr: function of name '" + << attr + << "' does not exist *OR* child node, variable, meter, event or limit on node " + << self->absNodePath())); return py::object(); } @@ -581,26 +589,21 @@ const ecf::Flag& get_flag_attr(node_ptr self) { return self->get_flag(); } -void export_Node() { - // Turn off proxies by passing true as the NoProxy template parameter. - // shared_ptrs don't need proxies because calls on one a copy of the - // shared_ptr will affect all of them (duh!). - py::class_>("NodeVec", "Hold a list of Nodes (i.e `suite`_, `family`_ or `task`_\\ s)") - .def(py::vector_indexing_suite, true>()); +void export_Node(py::module& m) { + + py::class_>(m, "Node", py::dynamic_attr(), DefsDoc::node_doc()) - py::class_("Node", DefsDoc::node_doc(), py::no_init) - .def("name", &Node::name, py::return_value_policy()) - .def("add", raw_function(add, 1), DefsDoc::add()) // a.add(b) & a.add([b]) - .def(py::self < py::self) // __lt__ - .def("__add__", &NodeUtil::do_add, DefsDoc::add()) // a + b + .def("name", &Node::name, py::return_value_policy::reference) + .def("add", &add, DefsDoc::add()) // a.add(b) & a.add([b]) + .def(py::self < py::self) // __lt__ + .def("__add__", &node_add, DefsDoc::add()) // a + b .def("__rshift__", &do_rshift) // nc >> a >> b >> c a + (b.add(Trigger('a==complete')) + (c.add(Trigger('b==complete'))) .def("__lshift__", &do_lshift) // nc << a << b << c (a.add(Trigger('b==complete')) + (b.add(Trigger('c==complete'))) + c - .def("__iadd__", &NodeUtil::do_add) // a += b - .def("__iadd__", &NodeUtil::node_iadd) // a += [ b ] - .def("__getattr__", &node_getattr) /* Any attempt to resolve a property, method, or field name that doesn't - actually exist on the object itself will be passed to __getattr__*/ + .def("__iadd__", &node_iadd) // a += b + .def("__getattr__", &node_getattr) /* Any attempt to resolve a property, method, or field name that doesn't + actually exist on the object itself will be passed to __getattr__*/ .def("remove", &Node::remove, "Remove the node from its parent. and returns it") .def("add_trigger", &add_trigger, DefsDoc::add_trigger_doc()) .def("add_trigger", &add_trigger_expr) @@ -617,7 +620,8 @@ void export_Node() { .def("add_variable", &add_variable, DefsDoc::add_variable_doc()) .def("add_variable", &add_variable_int) .def("add_variable", &add_variable_var) - .def("add_variable", &NodeUtil::add_variable_dict) + .def("add_variable", &add_variable_edit) + .def("add_variable", &NodeUtil::add1) .def("add_label", &add_label, DefsDoc::add_label_doc()) .def("add_label", &add_label_1) .def("add_aviso", &add_aviso, DefsDoc::add_aviso_doc()) @@ -626,10 +630,10 @@ void export_Node() { .def("add_limit", &add_limit_1) .def("add_inlimit", &add_in_limit, - (py::arg("limit_name"), - py::arg("path_to_node_containing_limit") = "", - py::arg("tokens") = 1, - py::arg("limit_this_node_only") = false), + py::arg("limit_name"), + py::arg("path_to_node_containing_limit") = "", + py::arg("tokens") = 1, + py::arg("limit_this_node_only") = false, DefsDoc::add_inlimit_doc()) .def("add_inlimit", &add_in_limit_1) .def("add_event", &add_event, DefsDoc::add_event_doc()) @@ -664,12 +668,16 @@ void export_Node() { .def("add_autocancel", &add_autocancel_3) .def("add_autoarchive", &add_autoarchive, - (py::arg("days"), py::arg("idle") = false), + py::arg("days"), + py::arg("idle") = false, DefsDoc::add_autoarchive_doc()) .def("add_autoarchive", &add_autoarchive_1, - (py::arg("hour"), py::arg("min"), py::arg("relative"), py::arg("idle") = false)) - .def("add_autoarchive", &add_autoarchive_2, (py::arg("TimeSlot"), py::arg("relative"), py::arg("idle") = false)) + py::arg("hour"), + py::arg("min"), + py::arg("relative"), + py::arg("idle") = false) + .def("add_autoarchive", &add_autoarchive_2, py::arg("TimeSlot"), py::arg("relative"), py::arg("idle") = false) .def("add_autoarchive", &add_autoarchive_3) .def("add_autorestore", &add_autorestore, DefsDoc::add_autorestore_doc()) .def("add_autorestore", &add_autorestore1) @@ -714,8 +722,14 @@ void export_Node() { .def("sort_attributes", &sort_attributes2) .def("sort_attributes", &sort_attributes3, - (py::arg("attribute_type"), py::arg("recursive") = true, py::arg("no_sort") = py::list())) - .def("sort_attributes", &Node::sort_attributes, (py::arg("attribute_type"), py::arg("recursive") = true)) + py::arg("attribute_type"), + py::arg("recursive") = true, + py::arg("no_sort") = py::list()) + .def("sort_attributes", + &Node::sort_attributes, + py::arg("attribute_type"), + py::arg("recursive") = true, + py::arg("no_sort") = py::list()) .def("get_abs_node_path", &Node::absNodePath, DefsDoc::abs_node_path_doc()) .def("has_time_dependencies", &Node::hasTimeDependencies) .def("update_generated_variables", &Node::update_generated_variables) @@ -728,100 +742,111 @@ void export_Node() { .def("is_suspended", &Node::isSuspended, "Returns true if the `node`_ is in a `suspended`_ state") .def("find_variable", &Node::findVariable, - py::return_value_policy(), + py::return_value_policy::reference, "Find user variable on the node only. Returns an object") .def("find_gen_variable", &Node::findGenVariable, - py::return_value_policy(), + py::return_value_policy::reference, "Find generated variable on the node only. Returns an object") .def("find_parent_variable", &Node::find_parent_variable, - py::return_value_policy(), + py::return_value_policy::reference, "Find user variable variable up the parent hierarchy. Returns an object") .def("find_parent_variable_sub_value", &Node::find_parent_variable_sub_value, "Find user variable *up* node tree, then variable substitute the value, otherwise return empty string") .def("find_meter", &Node::findMeter, - py::return_value_policy(), + py::return_value_policy::reference, "Find the `meter`_ on the node only. Returns an object") .def("find_event", &Node::findEventByNameOrNumber, - py::return_value_policy(), + py::return_value_policy::reference, "Find the `event`_ on the node only. Returns a object") .def("find_label", &Node::find_label, - py::return_value_policy(), + py::return_value_policy::reference, "Find the `label`_ on the node only. Returns a object") .def("find_queue", &Node::find_queue, - py::return_value_policy(), + py::return_value_policy::reference, "Find the queue on the node only. Returns a queue object") .def("find_generic", &Node::find_generic, - py::return_value_policy(), + py::return_value_policy::reference, "Find the `generic`_ on the node only. Returns a Generic object") .def("find_limit", &Node::find_limit, "Find the `limit`_ on the node only. returns a limit ptr") .def("find_node_up_the_tree", &Node::find_node_up_the_tree, "Search immediate node, then up the node hierarchy") .def("get_state", &Node::state, "Returns the state of the node. This excludes the suspended state") .def("get_state_change_time", &get_state_change_time, - (py::arg("format") = "iso_extended"), + py::arg("format") = "iso_extended", "Returns the time of the last state change as a string. Default format is iso_extended, (iso_extended, " "iso, simple)") .def("get_dstate", &Node::dstate, "Returns the state of node. This will include suspended state") .def("get_defstatus", &Node::defStatus) - .def("get_repeat", &Node::repeat, py::return_value_policy()) - .def("get_late", &get_late_attr, py::return_internal_reference<>()) - .def("get_autocancel", &get_autocancel_attr, py::return_internal_reference<>()) - .def("get_autoarchive", &get_autoarchive_attr, py::return_internal_reference<>()) - .def("get_autorestore", &get_autorestore_attr, py::return_internal_reference<>()) - .def("get_trigger", &Node::get_trigger, py::return_internal_reference<>()) - .def("get_complete", &Node::get_complete, py::return_internal_reference<>()) - .def("get_defs", get_defs, py::return_internal_reference<>()) - .def("get_parent", &Node::parent, py::return_internal_reference<>()) + .def("get_repeat", &Node::repeat, py::return_value_policy::reference) + .def("get_late", &get_late_attr, py::return_value_policy::reference_internal) + .def("get_autocancel", &get_autocancel_attr, py::return_value_policy::reference_internal) + .def("get_autoarchive", &get_autoarchive_attr, py::return_value_policy::reference_internal) + .def("get_autorestore", &get_autorestore_attr, py::return_value_policy::reference_internal) + .def("get_trigger", &Node::get_trigger, py::return_value_policy::reference_internal) + .def("get_complete", &Node::get_complete, py::return_value_policy::reference_internal) + .def("get_defs", get_defs, py::return_value_policy::reference_internal) + .def("get_parent", &Node::parent, py::return_value_policy::reference_internal) .def("get_all_nodes", &get_all_nodes, "Returns all the child nodes") .def("get_flag", &get_flag_attr, - py::return_value_policy(), + py::return_value_policy::reference, "Return additional state associated with a node.") .def("replace_on_server", &replace_on_server, - (py::arg("suspend_node_first") = true, py::arg("force") = true), + py::arg("suspend_node_first") = true, + py::arg("force") = true, "replace node on the server.") .def("replace_on_server", &replace_on_server1, - (py::arg("suspend_node_first") = true, py::arg("force") = true), + py::arg("host"), + py::arg("port"), + py::arg("suspend_node_first") = true, + py::arg("force") = true, "replace node on the server.") .def("replace_on_server", &replace_on_server2, - (py::arg("suspend_node_first") = true, py::arg("force") = true), + py::arg("host_port"), + py::arg("suspend_node_first") = true, + py::arg("force") = true, "replace node on the server.") .def("replace_on_server", &do_replace_on_server, - (py::arg("suspend_node_first") = true, py::arg("force") = true), + py::arg("client"), + py::arg("suspend_node_first") = true, + py::arg("force") = true, "replace node on the server.") - .add_property("meters", py::range(&Node::meter_begin, &Node::meter_end), "Returns a list of `meter`_\\ s") - .add_property("events", py::range(&Node::event_begin, &Node::event_end), "Returns a list of `event`_\\ s") - .add_property("variables", - py::range(&Node::variable_begin, &Node::variable_end), - "Returns a list of user defined `variable`_\\ s") - .add_property("labels", py::range(&Node::label_begin, &Node::label_end), "Returns a list of `label`_\\ s") - .add_property("avisos", py::range(&Node::aviso_begin, &Node::aviso_end), "Returns a list of `aviso`_\\ s") - .add_property("mirrors", py::range(&Node::mirror_begin, &Node::mirror_end), "Returns a list of `mirror`_\\ s") - .add_property("limits", py::range(&Node::limit_begin, &Node::limit_end), "Returns a list of `limit`_\\ s") - .add_property( - "inlimits", py::range(&Node::inlimit_begin, &Node::inlimit_end), "Returns a list of `inlimit`_\\ s") - .add_property("verifies", py::range(&Node::verify_begin, &Node::verify_end), "Returns a list of Verify's") - .add_property("times", py::range(&Node::time_begin, &Node::time_end), "Returns a list of `time`_\\ s") - .add_property("todays", py::range(&Node::today_begin, &Node::today_end), "Returns a list of `today`_\\ s") - .add_property("dates", py::range(&Node::date_begin, &Node::date_end), "Returns a list of `date`_\\ s") - .add_property("days", py::range(&Node::day_begin, &Node::day_end), "Returns a list of `day`_\\ s") - .add_property("crons", py::range(&Node::cron_begin, &Node::cron_end), "Returns a list of `cron`_\\ s") - .add_property("zombies", py::range(&Node::zombie_begin, &Node::zombie_end), "Returns a list of `zombie`_\\ s") - .add_property("queues", py::range(&Node::queue_begin, &Node::queue_end), "Returns a list of `queue`_\\ s") - .add_property( - "generics", py::range(&Node::generic_begin, &Node::generic_end), "Returns a list of `generic`_\\ s"); + .def_property_readonly("meters", &Node::meters, "Returns a list of `meter`_\\ s") + .def_property_readonly("events", &Node::events, "Returns a list of `event`_\\ s") + .def_property_readonly("variables", &Node::variables, "Returns a list of user defined `variable`_\\ s") + .def_property_readonly("labels", &Node::labels, "Returns a list of `label`_\\ s") + .def_property_readonly("avisos", + static_cast& (Node::*)() const>(&Node::avisos), + py::return_value_policy::reference_internal, + "Returns a list of `aviso`_\\ s") + .def_property_readonly("mirrors", + static_cast& (Node::*)() const>(&Node::mirrors), + py::return_value_policy::reference_internal, + "Returns a list of `mirror`_\\ s") + .def_property_readonly("limits", &Node::limits, "Returns a list of `limit`_\\ s") + .def_property_readonly("inlimits", &Node::inlimits, "Returns a list of `inlimit`_\\ s") + .def_property_readonly("verifies", &Node::verifys, "Returns a list of Verify's") + .def_property_readonly("times", &Node::timeVec, "Returns a list of `time`_\\ s") + .def_property_readonly("todays", &Node::todayVec, "Returns a list of `today`_\\ s") + .def_property_readonly("dates", &Node::dates, "Returns a list of `date`_\\ s") + .def_property_readonly("days", &Node::days, "Returns a list of `day`_\\ s") + .def_property_readonly("crons", &Node::crons, "Returns a list of `cron`_\\ s") + .def_property_readonly("zombies", &Node::zombies, "Returns a list of `zombie`_\\ s") + .def_property_readonly("queues", &Node::queues, "Returns a list of `queue`_\\ s") + .def_property_readonly("generics", &Node::generics, "Returns a list of `generic`_\\ s"); + #if ECF_ENABLE_PYTHON_PTR_REGISTER py::register_ptr_to_python(); // needed for mac and boost 1.6 #endif diff --git a/libs/pyext/src/ecflow/python/ExportNodeAttr.cpp b/libs/pyext/src/ecflow/python/ExportNodeAttr.cpp index 4243de339..bda93eedb 100644 --- a/libs/pyext/src/ecflow/python/ExportNodeAttr.cpp +++ b/libs/pyext/src/ecflow/python/ExportNodeAttr.cpp @@ -37,6 +37,7 @@ #include "ecflow/node/Limit.hpp" #include "ecflow/node/MirrorAttr.hpp" #include "ecflow/python/DefsDoc.hpp" +#include "ecflow/python/ExportCollections.hpp" #include "ecflow/python/NodeAttrDoc.hpp" #include "ecflow/python/PythonBinding.hpp" #include "ecflow/python/PythonUtil.hpp" @@ -44,7 +45,7 @@ // See: http://wiki.python.org/moin/boost.python/HowTo#boost.function_objects /////////////////////////////////////////////////////////////////////////////////////////////////// -py::object late_raw_constructor(py::tuple args, py::dict kw) { +py::object late_raw_constructor(const py::args& args, const py::kwargs& kw) { // cout << "late_raw_constructor len(args):" << len(args) << endl; // args[0] is Late(i.e self) if (len(args) > 1) { @@ -54,97 +55,96 @@ py::object late_raw_constructor(py::tuple args, py::dict kw) { return args[0].attr("__init__")(kw); // calls -> late_init(dict kw) } -static void extract_late_keyword_arguments(std::shared_ptr late, py::dict& dict) { - py::list keys = dict.keys(); - const int no_of_keys = len(keys); - for (int i = 0; i < no_of_keys; ++i) { - if (py::extract(keys[i]).check()) { - std::string first = py::extract(keys[i]); - if (py::extract(dict[keys[i]]).check()) { - std::string second = py::extract(dict[keys[i]]); - int hour = 0; - int min = 0; - bool relative = ecf::TimeSeries::getTime(second, hour, min); - if (first == "submitted") { - late->add_submitted(hour, min); - } - else if (first == "active") { - late->add_active(hour, min); - } - else if (first == "complete") { - late->add_complete(hour, min, relative); - } - else { - throw std::runtime_error( - "extract_late_keyword_arguments: keyword arguments, expected [submitted | active | complete]"); - } - } - else { - throw std::runtime_error("extract_late_keyword_arguments: expected keyword arguments to be a string, " - "ie Late(submitted='00:20',active='15:00',complete='+30:00')"); - } - } - } -} +static void extract_late_keyword_arguments(std::shared_ptr late, const py::kwargs& kwargs) { -static std::shared_ptr late_init(py::dict& dict) { - std::shared_ptr late = std::make_shared(); - extract_late_keyword_arguments(late, dict); - return late; -} + for (const auto& entry : kwargs) { + // 1. Extract the keywork argument name + std::string first; + if (auto found = py_extract(entry.first); found) { + first = found.value(); + } + else if (auto found = py_extract(entry.first); found) { + first = found.value(); + } + else { + throw std::runtime_error("extract_late_keyword_arguments: keyword argument name expected to be a string"); + } -static std::shared_ptr late_create() { - return std::make_shared(); -} + // 1.a Ensure the keyword argument is an expected value + static const std::vector valid_keywords = {"submitted", "active", "complete"}; + if (find(valid_keywords.begin(), valid_keywords.end(), first) == valid_keywords.end()) { + throw std::runtime_error("extract_late_keyword_arguments: keyword argument expected to be one of " + "[submitted | active | complete]"); + } -/////////////////////////////////////////////////////////////////////////////////////////////////// -py::object cron_raw_constructor(py::tuple args, py::dict kw) { - // cout << "cron_raw_constructor len(args):" << len(args) << endl; - // args[0] is Cron(i.e self) args[1] is string name - for (int i = 1; i < len(args); ++i) { - if (py::extract(args[i]).check()) { - std::string time_series = py::extract(args[i]); - if (time_series.empty()) { - throw std::runtime_error("cron_raw_constructor: Empty string, please pass a valid time, i.e '12:30'"); - } - return args[0].attr("__init__")(time_series, kw); // calls -> init(const std::string& ts, dict kw) + // 2. Extract keyword argument value + std::string second; + if (auto found = py_extract(entry.second); found) { + second = found.value(); } - if (py::extract(args[i]).check()) { - ecf::TimeSeries time_series = py::extract(args[i]); - return args[0].attr("__init__")(time_series, kw); // calls -> init(const ecf::TimeSeries& ts, dict kw) + else if (auto found = py_extract(entry.second); found) { + second = found.value(); } else { - throw std::runtime_error("cron_raw_constructor: expects string | TimeSeries and keyword arguments"); + throw std::runtime_error("extract_late_keyword_arguments: keyword argument value expected to be a string, " + "ie Late(submitted='00:20',active='15:00',complete='+30:00')"); + } + + // 3. Convert the value to hour/min/relative, and set the Late attributes + int hour = 0; + int min = 0; + bool relative = ecf::TimeSeries::getTime(second, hour, min); + if (first == "submitted") { + late->add_submitted(hour, min); + } + else if (first == "active") { + late->add_active(hour, min); } + else if (first == "complete") { + late->add_complete(hour, min, relative); + } + } +} + +static std::shared_ptr late_init_default() { + return std::make_shared(); +} + +std::shared_ptr late_init(const py::args& args, const py::kwargs& kwargs) { + int arg_count = len(args); + if (arg_count > 0) { + throw std::runtime_error("late_raw_constructor: Late only expects keyword arguments, ie. " + "Late(submitted='00:20',active='15:00',complete='+30:00')"); } - throw std::runtime_error("cron_raw_constructor: expects string | TimeSeries and keyword arguments !!"); - return py::object(); + auto late = std::make_shared(); + extract_late_keyword_arguments(late, kwargs); + return late; } +/////////////////////////////////////////////////////////////////////////////////////////////////// static void extract_cron_keyword_arguments(std::shared_ptr cron, py::dict& dict) { - py::list keys = dict.keys(); - const int no_of_keys = len(keys); - for (int i = 0; i < no_of_keys; ++i) { + for (auto entry : dict) { + if (auto found_key = py_extract(entry.first); found_key) { + std::string key = found_key.value(); - if (py::extract(keys[i]).check()) { - std::string first = py::extract(keys[i]); - if (py::extract(dict[keys[i]]).check()) { + if (auto found_value = py_extract(entry.second); found_value) { + py::list value = found_value.value(); + std::cout << "[1]key: " << key << " value: " << value << std::endl; - py::list second = py::extract(dict[keys[i]]); std::vector int_vec; - pyutil_list_to_int_vec(second, int_vec); + pyutil_list_to_int_vec(value, int_vec); // expected keywords are: days_of_week,last_week_days_ofThe_month, days_of_month, months - if (first == "days_of_week") { + if (key == "days_of_week") { cron->addWeekDays(int_vec); } - else if (first == "days_of_month") { + else if (key == "days_of_month") { cron->addDaysOfMonth(int_vec); } - else if (first == "months") { + else if (key == "months") { cron->addMonths(int_vec); } - else if (first == "last_week_days_of_the_month") { + else if (key == "last_week_days_of_the_month") { cron->add_last_week_days_of_month(int_vec); } else { @@ -153,8 +153,9 @@ static void extract_cron_keyword_arguments(std::shared_ptr cron, "last_week_days_of_the_month | days_of_month | months | last_day_of_the_month"); } } - else if (py::extract(dict[keys[i]]).check()) { - if (first == "last_day_of_the_month") { + else if (auto found_value = py_extract(entry.second); found_value) { + std::cout << "[2]key: " << key << " value: " << found_value.value() << std::endl; + if (key == "last_day_of_the_month") { cron->add_last_day_of_month(); } else { @@ -170,15 +171,15 @@ static void extract_cron_keyword_arguments(std::shared_ptr cron, } } -static std::shared_ptr cron_init(const std::string& ts, py::dict& dict) { +static std::shared_ptr cron_init(const std::string& ts, py::kwargs& kwargs) { std::shared_ptr cron = std::make_shared(ts); - extract_cron_keyword_arguments(cron, dict); + extract_cron_keyword_arguments(cron, kwargs); return cron; } -static std::shared_ptr cron_init1(const ecf::TimeSeries& ts, py::dict& dict) { +static std::shared_ptr cron_init1(const ecf::TimeSeries& ts, py::kwargs& kwargs) { std::shared_ptr cron = std::make_shared(ts); - extract_cron_keyword_arguments(cron, dict); + extract_cron_keyword_arguments(cron, kwargs); return cron; } @@ -224,59 +225,64 @@ void set_months(ecf::CronAttr* cron, const py::list& list) { // *AND* the only way make_constructor works is with a pointer. // The Node::add function seem to cope with this, some boost python magic,must do a conversion // from shared_ptr to pass by reference -static std::shared_ptr create_RepeatEnumerated(const std::string& name, const py::list& list) { +static RepeatEnumerated create_RepeatEnumerated(const std::string& name, const py::list& list) { std::vector vec; pyutil_list_to_str_vec(list, vec); - return std::make_shared(name, vec); + return RepeatEnumerated(name, vec); } -static std::shared_ptr create_RepeatDateList(const std::string& name, const py::list& list) { +static RepeatDateList create_RepeatDateList(const std::string& name, const py::list& list) { std::vector vec; pyutil_list_to_int_vec(list, vec); - return std::make_shared(name, vec); + return RepeatDateList(name, vec); } -static std::shared_ptr create_RepeatString(const std::string& name, const py::list& list) { +static RepeatString create_RepeatString(const std::string& name, const py::list& list) { std::vector vec; pyutil_list_to_str_vec(list, vec); - return std::make_shared(name, vec); + return RepeatString(name, vec); } -static std::shared_ptr create_AutoRestoreAttr(const py::list& list) { +static ecf::AutoRestoreAttr create_AutoRestoreAttr(const py::list& list) { std::vector vec; pyutil_list_to_str_vec(list, vec); - return std::make_shared(vec); + return ecf::AutoRestoreAttr(vec); } -static std::shared_ptr create_queue(const std::string& name, const py::list& list) { +static QueueAttr create_queue(const std::string& name, const py::list& list) { std::vector vec; pyutil_list_to_str_vec(list, vec); - return std::make_shared(name, vec); + return QueueAttr(name, vec); } -static std::shared_ptr create_generic(const std::string& name, const py::list& list) { +static GenericAttr create_generic(const std::string& name, const py::list& list) { std::vector vec; pyutil_list_to_str_vec(list, vec); - return std::make_shared(name, vec); + return GenericAttr(name, vec); } -static std::shared_ptr +static ZombieAttr create_ZombieAttr(ecf::Child::ZombieType zt, const py::list& list, ecf::ZombieCtrlAction uc, int life_time_in_server) { - std::vector vec; + int the_list_size = len(list); + + std::vector vec; vec.reserve(the_list_size); + for (int i = 0; i < the_list_size; ++i) { - vec.push_back(py::extract(list[i])); + vec.push_back(list[i].cast()); } - return std::make_shared(zt, vec, uc, life_time_in_server); + return ZombieAttr(zt, vec, uc, life_time_in_server); } -static std::shared_ptr -create_ZombieAttr1(ecf::Child::ZombieType zt, const py::list& list, ecf::ZombieCtrlAction uc) { - std::vector vec; +static ZombieAttr create_ZombieAttr1(ecf::Child::ZombieType zt, const py::list& list, ecf::ZombieCtrlAction uc) { + int the_list_size = len(list); + + std::vector vec; vec.reserve(the_list_size); + for (int i = 0; i < the_list_size; ++i) { - vec.push_back(py::extract(list[i])); + vec.push_back(list[i].cast()); } - return std::make_shared(zt, vec, uc); + return ZombieAttr(zt, vec, uc); } static py::list wrap_set_of_strings(Limit* limit) { @@ -292,95 +298,31 @@ static job_creation_ctrl_ptr makeJobCreationCtrl() { return std::make_shared(); } -static std::shared_ptr aviso_init(const std::string& name, - const std::string& listener, - const std::string& url = ecf::AvisoAttr::default_url, - const std::string& schema = ecf::AvisoAttr::default_schema, - const std::string& polling = ecf::AvisoAttr::default_polling, - const std::string& auth = ecf::AvisoAttr::default_auth) { - auto attr = std::make_shared(nullptr, name, listener, url, schema, polling, 0, auth, ""); - return attr; +static ecf::AvisoAttr aviso_init(const std::string& name, + const std::string& listener, + const std::string& url = ecf::AvisoAttr::default_url, + const std::string& schema = ecf::AvisoAttr::default_schema, + const std::string& polling = ecf::AvisoAttr::default_polling, + const std::string& auth = ecf::AvisoAttr::default_auth) { + return ecf::AvisoAttr(nullptr, name, listener, url, schema, polling, 0, auth, ""); } -static std::shared_ptr aviso_init_defaults_0(const std::string& name, const std::string& listener) { - return aviso_init(name, listener); +static ecf::MirrorAttr mirror_init(const std::string& name, + const std::string& path, + const std::string& host = ecf::MirrorAttr::default_remote_host, + const std::string& port = ecf::MirrorAttr::default_remote_port, + const std::string& polling = ecf::MirrorAttr::default_polling, + bool ssl = false, + const std::string& auth = ecf::MirrorAttr::default_remote_auth) { + return ecf::MirrorAttr(nullptr, name, path, host, port, polling, ssl, auth, ""); } -static std::shared_ptr -aviso_init_defaults_1(const std::string& name, const std::string& listener, const std::string& url) { - return aviso_init(name, listener, url); -} - -static std::shared_ptr aviso_init_defaults_2(const std::string& name, - const std::string& listener, - const std::string& url, - const std::string& schema) { - return aviso_init(name, listener, url, schema); -} - -static std::shared_ptr aviso_init_defaults_3(const std::string& name, - const std::string& listener, - const std::string& url, - const std::string& schema, - const std::string& polling) { - return aviso_init(name, listener, url, schema, polling); -} - -static std::shared_ptr mirror_init(const std::string& name, - const std::string& path, - const std::string& host = ecf::MirrorAttr::default_remote_host, - const std::string& port = ecf::MirrorAttr::default_remote_port, - const std::string& polling = ecf::MirrorAttr::default_polling, - bool ssl = false, - const std::string& auth = ecf::MirrorAttr::default_remote_auth) { - auto attr = std::make_shared(nullptr, name, path, host, port, polling, ssl, auth, ""); - return attr; -} - -static std::shared_ptr mirror_init_defaults_0(const std::string& name, const std::string& path) { - return mirror_init(name, path); -} - -static std::shared_ptr -mirror_init_defaults_1(const std::string& name, const std::string& path, const std::string& host) { - return mirror_init(name, path, host); -} - -static std::shared_ptr mirror_init_defaults_2(const std::string& name, - const std::string& path, - const std::string& host, - const std::string& port) { - return mirror_init(name, path, host, port); -} - -static std::shared_ptr mirror_init_defaults_3(const std::string& name, - const std::string& path, - const std::string& host, - const std::string& port, - const std::string& polling) { - return mirror_init(name, path, host, port, polling); -} +void export_NodeAttr(py::module& m) { -static std::shared_ptr mirror_init_defaults_4(const std::string& name, - const std::string& path, - const std::string& host, - const std::string& port, - const std::string& polling, - bool ssl) { - return mirror_init(name, path, host, port, polling, ssl); -} - -static std::string to_python_string_aviso(const ecf::AvisoAttr& aviso) { - return ecf::to_python_string(aviso); -} - -static std::string to_python_string_mirror(const ecf::MirrorAttr& mirror) { - return ecf::to_python_string(mirror); -} - -void export_NodeAttr() { // Trigger & Complete thin wrapper over Expression, allows us to call: Task("a").add(Trigger("a=1"),Complete("b=1")) - py::class_>("Trigger", DefsDoc::trigger(), py::init()) + py::class_>(m, "Trigger", DefsDoc::trigger()) + + .def(py::init()) .def(py::init()) .def(py::init()) .def(py::init()) @@ -388,7 +330,9 @@ void export_NodeAttr() { .def("__str__", &Trigger::expression) // __str__ .def("get_expression", &Trigger::expression, "returns the trigger expression as a string"); - py::class_>("Complete", DefsDoc::trigger(), py::init()) + py::class_>(m, "Complete", DefsDoc::trigger()) + + .def(py::init()) .def(py::init()) .def(py::init()) .def(py::init()) @@ -399,18 +343,21 @@ void export_NodeAttr() { // mimic PartExpression(const std::string& expression ) // mimic PartExpression(const std::string& expression, bool andExpr /* true means AND , false means OR */ ) // Use to adding large trigger and complete expressions - py::class_("PartExpression", DefsDoc::part_expression_doc(), py::init()) + py::class_(m, "PartExpression", DefsDoc::part_expression_doc()) + + .def(py::init()) .def(py::init()) .def(py::self == py::self) // __eq__ .def("get_expression", &PartExpression::expression, - py::return_value_policy(), + py::return_value_policy::reference, "returns the part expression as a string") .def("and_expr", &PartExpression::andExpr) .def("or_expr", &PartExpression::orExpr); - py::class_>( - "Expression", DefsDoc::expression_doc(), py::init()) + py::class_>(m, "Expression", DefsDoc::expression_doc()) + + .def(py::init()) .def(py::init()) .def(py::self == py::self) // __eq__ .def("__str__", &Expression::expression) // __str__ @@ -418,37 +365,10 @@ void export_NodeAttr() { .def("add", &Expression::add, "Add a part expression, the second and subsequent part expressions must have 'and/or' set") - .add_property( - "parts", py::range(&Expression::part_begin, &Expression::part_end), "Returns a list of PartExpression's"); - - py::enum_( - "FlagType", - "Flags store state associated with a node\n\n" - "- FORCE_ABORT - Node* do not run when try_no > ECF_TRIES, and task aborted by user\n" - "- USER_EDIT - task\n" - "- TASK_ABORTED - task*\n" - "- EDIT_FAILED - task*\n" - "- JOBCMD_FAILED - task*\n" - "- KILLCMD_FAILED - task*\n" - "- STATUSCMD_FAILED - task*\n" - "- NO_SCRIPT - task*\n" - "- KILLED - task* do not run when try_no > ECF_TRIES, and task killed by user\n" - "- STATUS - task* indicates that the status command has been run\n" - "- LATE - Node attribute, Task is late, or Defs checkpt takes to long\n" - "- MESSAGE - Node\n" - "- BYRULE - Node*, set if node is set to complete by complete trigger expression\n" - "- QUEUELIMIT - Node\n" - "- WAIT - task* \n" - "- LOCKED - Server\n" - "- ZOMBIE - task*\n" - "- NO_REQUE - task\n" - "- ARCHIVED - Suite/Family\n" - "- RESTORED - Family/Family\n" - "- THRESHOLD - task\n" - "- SIGTERM - Defs, records that server received a SIGTERM signal\n" - "- LOG_ERROR - Error in opening or writing to log file\n" - "- CHECKPT_ERROR - Error in opening or writing to checkpt file \n" - "- NOT_SET\n") + .def("parts", &Expression::expr, "Returns a list of PartExpression's"); + + py::enum_(m, "FlagType", NodeAttrDoc::flag_type_doc()) + .value("force_abort", ecf::Flag::FORCE_ABORT) .value("user_edit", ecf::Flag::USER_EDIT) .value("task_aborted", ecf::Flag::TASK_ABORTED) @@ -476,23 +396,23 @@ void export_NodeAttr() { .value("checkpt_error", ecf::Flag::CHECKPT_ERROR) .value("remote_error", ecf::Flag::REMOTE_ERROR); - py::class_("Flag", "Represents additional state associated with a Node.\n\n", py::init<>()) + py::class_(m, "Flag", "Represents additional state associated with a Node.\n\n") + + .def(py::init<>()) .def("__str__", &ecf::Flag::to_string) // __str__ .def(py::self == py::self) // __eq__ .def("is_set", &ecf::Flag::is_set, "Queries if a given flag is set") .def("set", &ecf::Flag::set, "Sets the given flag. Used in test only") .def("clear", &ecf::Flag::clear, "Clear the given flag. Used in test only") .def("reset", &ecf::Flag::reset, "Clears all flags. Used in test only") - .def("list", &ecf::Flag::list, "Returns the list of all flag types. returns FlagTypeVec. Used in test only") - .staticmethod("list") - .def("type_to_string", &ecf::Flag::enum_to_string, "Convert type to a string. Used in test only") - .staticmethod("type_to_string"); + .def_static("list", &ecf::Flag::list, "Returns the list of all flag types. returns FlagTypeVec. Tests only") + .def_static("type_to_string", &ecf::Flag::enum_to_string, "Convert type to a string. Tests only"); + + py::class_>(m, "FlagTypeVec", "Hold a list of flag types"); - py::class_>("FlagTypeVec", "Hold a list of flag types") - .def(py::vector_indexing_suite, true>()); + py::class_>(m, "JobCreationCtrl", DefsDoc::jobgenctrl_doc()) - py::class_("JobCreationCtrl", DefsDoc::jobgenctrl_doc()) - .def("__init__", py::make_constructor(makeJobCreationCtrl), DefsDoc::jobgenctrl_doc()) + .def(py::init(&makeJobCreationCtrl), DefsDoc::jobgenctrl_doc()) .def("set_node_path", &JobCreationCtrl::set_node_path, "The node we want to check job creation for. If no node specified check all tasks") @@ -502,18 +422,19 @@ void export_NodeAttr() { .def("set_verbose", &JobCreationCtrl::set_verbose, "Output each task as its being checked.") .def("get_dir_for_job_creation", &JobCreationCtrl::dir_for_job_creation, - py::return_value_policy(), + py::return_value_policy::reference, "Returns the directory set for job creation") - .def( - "generate_temp_dir", - &JobCreationCtrl::generate_temp_dir, - "Automatically generated temporary directory for job creation. Directory written to stdout for information") + .def("generate_temp_dir", + &JobCreationCtrl::generate_temp_dir, + "Automatically generated temporary directory for job creation. Directory written to stdout for " + "information") .def("get_error_msg", &JobCreationCtrl::get_error_msg, - py::return_value_policy(), + py::return_value_policy::reference, "Returns an error message generated during checking of job creation"); - py::enum_("ZombieType", NodeAttrDoc::zombie_type_doc()) + py::enum_(m, "ZombieType", NodeAttrDoc::zombie_type_doc()) + .value("ecf", ecf::Child::ECF) .value("ecf_pid", ecf::Child::ECF_PID) .value("ecf_pid_passwd", ecf::Child::ECF_PID_PASSWD) @@ -521,7 +442,8 @@ void export_NodeAttr() { .value("user", ecf::Child::USER) .value("path", ecf::Child::PATH); - py::enum_("ZombieUserActionType", NodeAttrDoc::zombie_user_action_type_doc()) + py::enum_(m, "ZombieUserActionType", NodeAttrDoc::zombie_user_action_type_doc()) + .value("fob", ecf::ZombieCtrlAction::FOB) .value("fail", ecf::ZombieCtrlAction::FAIL) .value("remove", ecf::ZombieCtrlAction::REMOVE) @@ -529,7 +451,8 @@ void export_NodeAttr() { .value("block", ecf::ZombieCtrlAction::BLOCK) .value("kill", ecf::ZombieCtrlAction::KILL); - py::enum_("ChildCmdType", NodeAttrDoc::child_cmd_type_doc()) + py::enum_(m, "ChildCmdType", NodeAttrDoc::child_cmd_type_doc()) + .value("init", ecf::Child::INIT) .value("event", ecf::Child::EVENT) .value("meter", ecf::Child::METER) @@ -539,8 +462,8 @@ void export_NodeAttr() { .value("abort", ecf::Child::ABORT) .value("complete", ecf::Child::COMPLETE); - py::enum_("AttrType", - "Sortable attribute type, currently [event | meter | label | limit | variable | all ]") + py::enum_(m, "AttrType", NodeAttrDoc::sortable_attribute_type_doc()) + .value("event", ecf::Attr::EVENT) .value("meter", ecf::Attr::METER) .value("label", ecf::Attr::LABEL) @@ -548,9 +471,10 @@ void export_NodeAttr() { .value("variable", ecf::Attr::VARIABLE) .value("all", ecf::Attr::ALL); - py::class_("ZombieAttr", NodeAttrDoc::zombie_doc()) - .def("__init__", make_constructor(&create_ZombieAttr)) - .def("__init__", make_constructor(&create_ZombieAttr1)) + py::class_(m, "ZombieAttr", NodeAttrDoc::zombie_doc()) + + .def(py::init(&create_ZombieAttr)) + .def(py::init(&create_ZombieAttr1)) .def("__str__", &ZombieAttr::toString) // __str__ .def("__copy__", pyutil_copy_object) // __copy__ uses copy constructor .def(py::self == py::self) // __eq__ @@ -560,14 +484,16 @@ void export_NodeAttr() { .def("zombie_lifetime", &ZombieAttr::zombie_lifetime, "Returns the lifetime in seconds of `zombie`_ in the server") - .add_property("child_cmds", - py::range(&ZombieAttr::child_begin, &ZombieAttr::child_end), - "The list of child commands. If empty action applies to all child cmds"); + .def("child_cmds", + &ZombieAttr::child_cmds, + "The list of child commands. If empty action applies to all child cmds"); + + constexpr const char* zombievec_doc = "Hold a list of zombies"; + + py::class_>(m, "ZombieVec", zombievec_doc); - py::class_>("ZombieVec", "Hold a list of zombies") - .def(py::vector_indexing_suite, true>()); + py::class_(m, "Zombie", NodeAttrDoc::plain_zombie_doc()) - py::class_("Zombie", "Represent a zombie process stored by the server") .def("__str__", &Zombie::to_string) // __str__ .def("__copy__", pyutil_copy_object) // __copy__ uses copy constructor .def(py::self == py::self) // __eq__ @@ -582,114 +508,90 @@ void export_NodeAttr() { .def("type", &Zombie::type) .def("type_str", &Zombie::type_str) .def("last_child_cmd", &Zombie::last_child_cmd) - .def("attr", &Zombie::attr, py::return_value_policy()) + .def("attr", &Zombie::attr, py::return_value_policy::reference) .def("calls", &Zombie::calls) - .def("jobs_password", &Zombie::jobs_password, py::return_value_policy()) - .def("path_to_task", &Zombie::path_to_task, py::return_value_policy()) - .def("process_or_remote_id", &Zombie::process_or_remote_id, py::return_value_policy()) - .def("user_cmd", &Zombie::user_cmd, py::return_value_policy()) + .def("jobs_password", &Zombie::jobs_password, py::return_value_policy::reference) + .def("path_to_task", &Zombie::path_to_task, py::return_value_policy::reference) + .def("process_or_remote_id", &Zombie::process_or_remote_id, py::return_value_policy::reference) + .def("user_cmd", &Zombie::user_cmd, py::return_value_policy::reference) .def("try_no", &Zombie::try_no) .def("duration", &Zombie::duration) .def("user_action", &Zombie::user_action) .def("user_action_str", &Zombie::user_action_str) - .def("host", &Zombie::host, py::return_value_policy()) + .def("host", &Zombie::host, py::return_value_policy::reference) .def("allowed_age", &Zombie::allowed_age); - py::class_("Variable", NodeAttrDoc::variable_doc(), py::init()) + py::class_(m, "Variable", NodeAttrDoc::variable_doc()) + + .def(py::init()) .def("__str__", &Variable::toString) // __str__ .def("__copy__", pyutil_copy_object) // __copy__ uses copy constructor .def(py::self < py::self) // __lt__ .def(py::self == py::self) // __eq__ - .def("name", - &Variable::name, - py::return_value_policy(), - "Return the variable name as string") - .def("value", - &Variable::theValue, - py::return_value_policy(), - "Return the variable value as a string") + .def("name", &Variable::name, py::return_value_policy::reference, "Return the variable name as string") + .def("value", &Variable::theValue, py::return_value_policy::reference, "Return the variable value as a string") .def("empty", &Variable::empty, "Return true if the variable is empty. Used when returning a Null variable, from a find"); - // .add_property("value", make_function(&Variable::theValue ,py::return_value_policy()), - // &Variable::set_value, "get/set the value as a property") - // The following works v = Variable('name','fred') - // print v.value - // But it will then break v.value() - // We need to return pass a list of Variable as arguments, to retrieve the generated variables - py::class_>("VariableList", "Hold a list of Variables") - .def(py::vector_indexing_suite>()); + py::class_