Add a timer plugin

CHANGELOG.md

@@ -12,6 +12,7 @@ and this project adheres to
 ### For users
 
 #### Added
+* A timer plugin is added to reflect the time consumed by pdi and its plugins.
 * Improved messages for specification tree errors, with file & line numbers and
   support for file names from Paraconf 1.1,
   [#657](https://github.com/pdidev/pdi/issues/657)

CMakeLists.txt

@@ -76,10 +76,9 @@ option(BUILD_SHARED_LIBS        "Build shared libraries rather than static ones"
 option(BUILD_TRACE_PLUGIN       "Build Trace plugin" ON)
 option(BUILD_USER_CODE_PLUGIN   "Build User-code plugin" ON)
 option(BUILD_JSON_PLUGIN        "Build JSON plugin" OFF)
+option(BUILD_TIMER_PLUGIN       "Build Timer plugin" ON)
 option(ENABLE_BENCHMARKING      "Activate benchmarks in the test suite" OFF)
 
-
-
 ### Default build type
 
 if(NOT DEFINED CMAKE_BUILD_TYPE)
@@ -295,9 +294,9 @@ endif()
 ## HDF5
 
 if("${BUILD_DECL_HDF5_PLUGIN}" OR "${BUILD_DECL_NETCDF_PLUGIN}")
-	
+
 	set(HDF5_CMAKE_CACHE_ARGS)
-	
+
 	if("${BUILD_HDF5_PARALLEL}")
 		set(HDF5_PREFER_PARALLEL ON)
 		list(APPEND HDF5_CMAKE_CACHE_ARGS "-DHDF5_ENABLE_PARALLEL:BOOL=ON")
@@ -536,3 +535,10 @@ sbuild_add_module(PDI_API_TESTS
 	INSTALL_COMMAND ""
 	SUBSTEPS test
 )
+
+sbuild_add_module(TIMER_PLUGIN
+	ENABLE_BUILD_FLAG BUILD_TIMER_PLUGIN
+	SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/plugins/timer"
+	DEPENDS PDI
+	# SUBSTEPS indent test
+)

example/timer.yml

@@ -0,0 +1,66 @@
+# duration in seconds
+duration: 0.75
+# global [height, width] (excluding boundary conditions or ghosts)
+datasize: [60, 12]
+# degree of parallelism
+parallelism: { height: 3, width: 1 }
+
+# only the following config is passed to PDI
+pdi:
+  metadata: # type of small values for which PDI keeps a copy
+    iter:   int                    # current iteration id
+    dsize:  { size: 2, type: array, subtype: int } # local data size including ghosts/boundary
+    psize:  { size: 2, type: array, subtype: int } # number of processes in each dimension
+    pcoord: { size: 2, type: array, subtype: int } # coordinate of the process
+  data: # type of values for which PDI does not keep a copy
+    main_field: { size: [ '$dsize[0]', '$dsize[1]' ], type: array, subtype: double }
+  timer: true
+  plugins:
+    trace: debug
+    timer:
+      - timer_h5: "decl_hdf5"
+      - timer_nc:
+          start: "decl_netcdf_start_timer"
+          stop: "decl_netcdf_stop_timer"
+      - timer_python: [pycall, user_code]
+      - timer_json: {start: "json_start_timer", stop: "json_stop_timer"}
+      - timer_pdi: "pdi"
+    mpi:
+    decl_hdf5:
+      file: data.h5
+      communicator: $MPI_COMM_WORLD # the MPI communicator used for HDF5 parallel synchronized write
+      datasets: # type of the datasets to create in file
+        data:
+          type: array
+          subtype: double
+          size: [10, '$psize[0]*($dsize[0]-2)', '$psize[1]*($dsize[1]-2)']
+      write:
+        main_field: # the name of the data to write
+          dataset: data
+          mpio: COLLECTIVE # or INDEPENDENT
+          when: '$iter<10'   # do only write the first 10 iterations (0...9)
+          memory_selection:  # exclude ghosts from the data in memory
+            size:  ['$dsize[0]-2', '$dsize[1]-2']
+            start: [1, 1]
+          dataset_selection: # only write into a single slice in time
+            size:  [1, '$dsize[0]-2', '$dsize[1]-2']
+            start: [$iter, '($dsize[0]-2)*$pcoord[0]', '($dsize[1]-2)*$pcoord[1]']
+    decl_netcdf:
+      file: data.nc
+      communicator: $MPI_COMM_WORLD # the MPI communicator used for HDF5 parallel synchronized write
+      variables: # type of the datasets to create in file
+        v_data:
+          type: array
+          subtype: double
+          size: [10, '$psize[0]*$dsize[0]', '$psize[1]*$dsize[1]']
+      write:
+        main_field: # the name of the data to write
+          variable: v_data
+          when: '$iter<10'   # do only write the first 10 iterations (0...9)
+          variable_selection: # only write into a single slice in time
+            subsize:  [1, '$dsize[0]', '$dsize[1]']
+            start: [$iter, '$dsize[0]*$pcoord[0]', '$dsize[1]*$pcoord[1]']
+    json:
+      file: data_${pcoord[0]}_${pcoord[1]}.json
+      when: '$iter<10'
+      write: [main_field]

pdi/docs/CMakeLists.txt

@@ -52,6 +52,7 @@ set(DOXYGEN_INPUT
 		"${PDI_SOURCE_DIR}/../plugins/decl_hdf5/README.md"
 		"${PDI_SOURCE_DIR}/../plugins/decl_netcdf/README.md"
 		"${PDI_SOURCE_DIR}/../plugins/mpi/README.md"
+		"${PDI_SOURCE_DIR}/../plugins/timer/README.md"
 		"${PDI_SOURCE_DIR}/../plugins/trace/README.md"
 		"${PDI_SOURCE_DIR}/../plugins/user_code/README.md"
 		"${PDI_SOURCE_DIR}/../plugins/pycall/README.md"

pdi/docs/Plugins.md

@@ -9,6 +9,7 @@
 |\subpage JSON_plugin "JSON plugin"                   |Export data in JSON format.                                        |
 |\subpage serialize_plugin "Serialize plugin"         |Serializes and deserializes shared data.                           |
 |\subpage set_value_plugin "Set Value plugin"         |Set values to data and metadata from yaml file.                    |
+|\subpage timer_plugin "Timer plugin"                 |Measure the time spent by PDI and its plugins.                     |
 |\subpage trace_plugin "Trace plugin"                 |Generate a trace of what happens in %PDI data store.               |
 |\subpage user_code_plugin "user-code plugin"         |Call your function on event or when data becomes available.        |
 

pdi/src/pdi.cxx

@@ -191,6 +191,7 @@ try {
 	g_transaction.clear();
 	g_transaction_data.clear();
 	Global_context::init(conf);
+	Global_context::context().event("pdi_start_timer");
 	return PDI_OK;
 } catch (const Error& e) {
 	return g_error_context.return_err(e);
@@ -205,6 +206,7 @@ try {
 	Paraconf_wrapper fw;
 	g_transaction.clear();
 	g_transaction_data.clear();
+	Global_context::context().event("pdi_stop_timer");
 	Global_context::finalize();
 	return PDI_OK;
 } catch (const Error& e) {

plugins/decl_hdf5/decl_hdf5.cxx

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (C) 2015-2019 Commissariat a l'energie atomique et aux energies alternatives (CEA)
+ * Copyright (C) 2015-2026 Commissariat a l'energie atomique et aux energies alternatives (CEA)
  * Copyright (C) 2021 Institute of Bioorganic Chemistry Polish Academy of Science (PSNC)
  * All rights reserved.
  *
@@ -114,10 +114,12 @@ class decl_hdf5_plugin: public Plugin
 
 	void data(const std::string& name, Ref ref)
 	{
+		context().event("decl_hdf5_start_timer");
 		Hdf5_error_handler _;
 		for (auto&& op: m_data[name]) {
 			op.execute(context());
 		}
+		context().event("decl_hdf5_stop_timer");
 	}
 
 	void event(const std::string& event)

plugins/decl_netcdf/dnc_file_context.cxx

@@ -212,6 +212,7 @@ Dnc_variable* Dnc_file_context::variable(const std::string& desc_name, const std
 
 void Dnc_file_context::execute(const std::string& desc_name, PDI::Ref ref)
 {
+	m_ctx.event("decl_netcdf_start_timer");
 	if (m_when.to_long(m_ctx)) {
 		std::list<Dnc_variable> variables_holder; // memory for Variables created from descriptor
 
@@ -276,10 +277,12 @@ void Dnc_file_context::execute(const std::string& desc_name, PDI::Ref ref)
 			nc_file.get_sizeof_variable(size_it->first, dataset_name, ref);
 		}
 	}
+	m_ctx.event("decl_netcdf_stop_timer");
 }
 
 void Dnc_file_context::execute()
 {
+	m_ctx.event("decl_netcdf_start_timer");
 	if (m_when.to_long(m_ctx)) {
 		std::list<Dnc_variable> variables_holder;
 		std::vector<Dnc_variable*> variables_to_get;
@@ -354,6 +357,7 @@ void Dnc_file_context::execute()
 			i++;
 		}
 	}
+	m_ctx.event("decl_netcdf_stop_timer");
 }
 
 } // namespace decl_netcdf

plugins/decl_netcdf/dnc_file_context.h

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (C) 2024 Commissariat a l'energie atomique et aux energies alternatives (CEA)
+ * Copyright (C) 2024-2026 Commissariat a l'energie atomique et aux energies alternatives (CEA)
  * Copyright (C) 2020 Institute of Bioorganic Chemistry Polish Academy of Science (PSNC)
  * All rights reserved.
  *
@@ -109,6 +109,8 @@ class Dnc_file_context
 	 * \param other Dnc_file_context to move
 	 */
 	Dnc_file_context(Dnc_file_context&& other) noexcept;
+
+	static std::string pretty_name() { return "Decl'NetCDF"; }
 };
 
 } // namespace decl_netcdf

plugins/json/json.cxx

@@ -69,6 +69,7 @@ class json_plugin: public PDI::Plugin
 
 	~json_plugin() { context().logger().info("Closing plugin"); }
 
+	static std::string pretty_name() { return "JSON"; }
 
 private:
 	/** Read the configuration file
@@ -328,6 +329,7 @@ class json_plugin: public PDI::Plugin
 	 */
 	void write_data(const std::string& data_name, Ref_r&& reference)
 	{
+		context().event("json_start_timer");
 		Logger& logger = context().logger();
 
 		for (const auto& [condition, fpath]: m_data_to_path_map[data_name]) {
@@ -374,6 +376,7 @@ class json_plugin: public PDI::Plugin
 			}
 			logger.debug("Done ! {} ", data_name);
 		}
+		context().event("json_stop_timer");
 	}
 };
 

plugins/pycall/pycall.cxx

@@ -153,7 +153,9 @@ class Trigger
 			alias.expose(ctx, pyscope);
 		}
 		try {
+			ctx.event("pycall_start_timer");
 			pybind11::exec(m_code, pyscope);
+			ctx.event("pycall_stop_timer");
 		} catch (const std::exception& e) {
 			ctx.logger().error("while calling python, caught exception: {}", e.what());
 		} catch (...) {

plugins/serialize/serialize.cxx

@@ -272,6 +272,7 @@ struct serialize_plugin: PDI::Plugin {
 	 */
 	void share_serialized(const std::string& desc_name, PDI::Ref ref)
 	{
+		context().event("serialize_start_timer");
 		std::string serialized_name = m_desc_to_serialize[desc_name];
 		context().logger().debug("Serializing `{}` as `{}`", desc_name, serialized_name);
 		PDI::Datatype_sptr serialized_type = serialize_type(ref.type());
@@ -333,6 +334,7 @@ struct serialize_plugin: PDI::Plugin {
 			);
 			m_serialized_remove_callback.emplace_back(serialized_name, remove_callback, PDI_IN);
 		}
+		context().event("serialize_stop_timer");
 	}
 
 	void release_serialized(const std::string& desc_name, PDI::Ref ref)

plugins/timer/CMakeLists.txt

@@ -0,0 +1,48 @@
+#=============================================================================
+# Copyright (C) 2026 Commissariat a l'energie atomique et aux energies alternatives (CEA)
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# * Redistributions of source code must retain the above copyright
+#   notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+#   notice, this list of conditions and the following disclaimer in the
+#   documentation and/or other materials provided with the distribution.
+# * Neither the name of CEA nor the names of its contributors may be used to
+#   endorse or promote products derived from this software without specific
+#   prior written permission.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#=============================================================================
+
+cmake_minimum_required(VERSION 3.22...4.2)
+project(pdi_timer_plugin LANGUAGES C CXX)
+
+# Includes
+include(CTest)
+include(GNUInstallDirs)
+
+# PDI
+find_package(PDI REQUIRED COMPONENTS plugins)
+
+# The plugin
+add_library(pdi_timer_plugin MODULE timer.cxx)
+target_link_libraries(pdi_timer_plugin PUBLIC PDI::PDI_plugins )
+
+# Installation
+set(INSTALL_PDIPLUGINDIR "${PDI_DEFAULT_PLUGINDIR}" CACHE PATH "PDI plugins (${PDI_DEFAULT_PLUGINDIR})")
+install(TARGETS pdi_timer_plugin
+		LIBRARY DESTINATION "${INSTALL_PDIPLUGINDIR}"
+)
+
+# Tests
+if("${BUILD_TESTING}")
+	add_subdirectory(tests/)
+endif()

plugins/timer/README.md

@@ -0,0 +1,28 @@
+# Timer plugin {#timer_plugin}
+
+The timer plugin allows measuring time spent by PDI and its plugins.
+
+## Configuration {#timer_configuration}
+
+Simple plugin build:
+```yaml
+plugins:
+  timer: 
+    - timer_A: {start: "decl_hdf5_start_timer", stop: "decl_hdf5_stop_timer"}
+    - timer_B: "decl_hdf5"
+    - timer_C: [toto, titi]
+    - timer_D: 
+        start: "begin_timing"
+        stop: "end_timing"
+```
+
+The timer plugin configuration contains a list of timer names (e.g. `- timer_A`, `- timer_B`, etc). Each timer will record the time spent between the `start` and `stop` events.
+
+`timer_A` uses a map-styled definition where both keys `start` and `stop` are mandatory.
+
+`timer_B` uses a scalar-styled definition where only the prefix of the timer events is provided. The prefix can be wither a user-defined name, or the name of the PDI plugins, such as `decl_hdf5`, `decl_netcdf`, `pycall`, etc. If the prefix is the PDI plugin, internal events named with `prefix_start_timer` and `prefix_stop_timer` will be emitted inside the plugin (except `set_value`). In the above example, `timer_B` is equivalent to `timer_A`.
+
+`timer_C` uses a list-styled definition where a list of prefix is provided. If the name is not one of the PDI plugins' name, then it is the user's responsibility to emit the `prefix_start_timer` and `prefix_stop_timer` events. In this example, `timer_C` will recored the time spent between `toto_start_timer`, `toto_stop_timer`, and between `titi_start_timer`, `titi_stop_timer`.
+
+It is also possible to have different names for start and stop events such as `timer_D`. 
+

plugins/timer/tests/CMakeLists.txt

@@ -0,0 +1,47 @@
+#=============================================================================
+# Copyright (C) 2026 Commissariat a l'energie atomique et aux energies alternatives (CEA)
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# * Redistributions of source code must retain the above copyright
+#   notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+#   notice, this list of conditions and the following disclaimer in the
+#   documentation and/or other materials provided with the distribution.
+# * Neither the name of CEA nor the names of its contributors may be used to
+#   endorse or promote products derived from this software without specific
+#   prior written permission.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLEq FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#=============================================================================
+
+cmake_minimum_required(VERSION 3.22...4.2)
+
+set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" PROPERTY TEST_INCLUDE_FILE "${CMAKE_CURRENT_BINARY_DIR}/TestPath.cmake")
+
+file(GENERATE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/TestPath.cmake"
+     CONTENT "
+set(PDI_PLUGIN_PATH \"\$ENV{PDI_PLUGIN_PATH}\")\n
+if(\"x\${PDI_PLUGIN_PATH}x\" STREQUAL xx)\n
+set(ENV{PDI_PLUGIN_PATH} \"\$<TARGET_FILE_DIR:pdi_timer_plugin>\")\n
+else()\n
+set(ENV{PDI_PLUGIN_PATH} \"\$<TARGET_FILE_DIR:pdi_timer_plugin>:\${PDI_PLUGIN_PATH}\")\n
+endif()
+"
+)
+
+find_package(MPI REQUIRED)
+
+# 3. Set the C++ standard (using the variable from your matrix)
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+add_executable(timer_test timer_test.cxx )
+target_link_libraries(timer_test PDI::PDI_C MPI::MPI_CXX)