adding Chameleon and callgraph docs and examples

.devcontainer/devcontainer.json

@@ -6,7 +6,7 @@
 		// Sets the run context to one level up instead of the .devcontainer folder.
 		"context": "..",
 		// Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename.
-		"dockerfile": "../Dockerfile",
+		"dockerfile": "../infrastructure/docker/Dockerfile",
 	},
 	"runArgs": [
 		"-ti",

.github/workflows/docker-build-test.yml

@@ -3,7 +3,7 @@ name: Docker Build and Test
 on:
   push:
   pull_request:
-    branches: [ develop, stage-in ]
+    branches: [develop, stage-in]
   workflow_dispatch:
 
 env:
@@ -13,37 +13,37 @@ env:
 jobs:
   docker-build-and-test:
     runs-on: ubuntu-latest
-    
+
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-      with:
-        submodules: recursive
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
-    - name: Build Docker image
-      run: |
-        docker buildx build \
-                --cache-from=type=gha \
-                --cache-to=type=gha,mode=max \
-                --load \
-                -t ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DOCKER_TAG }} \
-                -f infrastrutcure/docker/Dockerfile \
-                .
+      - name: Build Docker image
+        run: |
+          docker buildx build \
+                  --cache-from=type=gha \
+                  --cache-to=type=gha,mode=max \
+                  --load \
+                  -t ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DOCKER_TAG }} \
+                  -f infrastructure/docker/Dockerfile \
+                  .
 
-    - name: Run DataCrumbs tests
-      run: |
-        docker run --rm \
-              -v /lib/modules/:/lib/modules:ro \
-              -v /sys/kernel/debug/:/sys/kernel/debug:rw \
-              --net=host \
-              --pid=host \
-              --privileged \
-              --cap-add sys_admin \
-              --cap-add sys_ptrace \
-              --hostname docker \
-              -v /sys/fs/bpf:/sys/fs/bpf \
-              ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DOCKER_TAG }} \
-              bash -c /opt/datacrumbs/infrastrutcure/docker/test_run.sh
+      - name: Run DataCrumbs tests
+        run: |
+          docker run --rm \
+                -v /lib/modules/:/lib/modules:ro \
+                -v /sys/kernel/debug/:/sys/kernel/debug:rw \
+                --net=host \
+                --pid=host \
+                --privileged \
+                --cap-add sys_admin \
+                --cap-add sys_ptrace \
+                --hostname docker \
+                -v /sys/fs/bpf:/sys/fs/bpf \
+                ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DOCKER_TAG }} \
+                bash -c /opt/datacrumbs/infrastructure/docker/test_run.sh

.gitignore

@@ -78,4 +78,8 @@ poc/bpf-core/vmlinux.h
 deps
 
 #nfs
-*.nfs*
+*.nfs*
+
+# Sphinx documentation builds
+docs/_build/
+docs/_static/__pycache__/

.vscode/settings.json

@@ -1,7 +1,4 @@
 {
-    "cmake.configureArgs": [
-        "-DCMAKE_PREFIX_PATH=/home/cc/datacrumbs/dependency/.spack-env/view"
-    ],
     "editor.formatOnSave": true,
     "files.associations": {
         "string": "cpp",

CMakeLists.txt

@@ -62,6 +62,7 @@ option(DATACRUMBS_TRACE_ALL_PROCESSES_OPT "Trace all processes" OFF)
 set(DATACRUMBS_INCLUSION_PATH "NONE" CACHE STRING "Inclusion path for datacrumbs")
 set(DATACRUMBS_CONFIGURED_TRACE_DIR "/tmp" CACHE STRING "Trace directory for datacrumbs")
 set(DATACRUMBS_CONFIGURED_LOG_DIR "/tmp" CACHE STRING "Log directory for datacrumbs")
+set(DATACRUMBS_CONFIGURED_RUN_DIR "NONE" CACHE STRING "Run directory for datacrumbs")
 
 option(DATACRUMBS_BPF_PRINT_ENABLE "Enable BPF print" OFF)
 set(DATACRUMBS_MODE_STR "TRACE" CACHE STRING "Mode for datacrumbs (TRACE or PROFILE)")
@@ -84,6 +85,15 @@ set(DATACRUMBS_SKIP_SMALL_EVENTS_THRESHOLD_NS "1000" CACHE STRING "Threshold in
 # Profiling Options
 set(DATACRUMBS_TIME_INTERVAL_NS "1000000" CACHE STRING "Profiling time interval in nanosecond")
 
+# Scheduling Options
+set(DATACRUMBS_SCHEDULER_TYPE "FLUX" CACHE STRING "Scheduler type for datacrumbs (FLUX, SLURM, OPENMPI, or NONE)")
+set_property(CACHE DATACRUMBS_SCHEDULER_TYPE PROPERTY STRINGS FLUX SLURM OPENMPI NONE)
+set(DATACRUMBS_SCHEDULER_JOBID_ENV_VAR "NONE" CACHE STRING "Scheduler job ID environment variable name")
+set(DATACRUMBS_SCHEDULER_NODES_CMD_OPT "NONE" CACHE STRING "Scheduler option for setting nodes")
+set(DATACRUMBS_SCHEDULER_PPN_CMD_OPT "NONE" CACHE STRING "Scheduler option for setting processes per node")
+set(DATACRUMBS_SCHEDULER_RUN_CMD "NONE" CACHE STRING "Scheduler run command")
+set(DATACRUMBS_SCHEDULER_RUN_EXTRA_ARGS "" CACHE STRING "Scheduler run extra args")
+
 # Other options
 set(DATACRUMBS_LOG_LEVEL_STR "INFO" CACHE STRING "Log Level for compiling ${UPPER_PROJECT_NAME}")
 set_property(CACHE DATACRUMBS_LOG_LEVEL_STR PROPERTY STRINGS ERROR WARN INFO DEBUG TRACE)
@@ -92,7 +102,6 @@ set(DATACRUMBS_KERNEL_HEADERS_PATH "" CACHE STRING "Path to the kernel headers d
 option(DATACRUMBS_BUILD_ONLY "Build only the datacrumbs library without installation" OFF)
 set(DATACRUMBS_KERNEL_PATH "" CACHE STRING "Path to the kernel source directory")
 
-
 # ------------------------------------------------------------------------------
 # Derive CMake variables based on options
 # ------------------------------------------------------------------------------
@@ -140,10 +149,16 @@ message(STATUS "             - Found kernel headers at: ${DATACRUMBS_KERNEL_HEAD
 message(STATUS "             - DATACRUMBS_KERNEL_UNAME_R: ${DATACRUMBS_KERNEL_UNAME_R}")
 message(STATUS "             - DATACRUMBS_CONFIGURED_TRACE_DIR: ${DATACRUMBS_CONFIGURED_TRACE_DIR}")
 message(STATUS "             - DATACRUMBS_CONFIGURED_LOG_DIR: ${DATACRUMBS_CONFIGURED_LOG_DIR}")
+message(STATUS "             - DATACRUMBS_SCHEDULER_TYPE: ${DATACRUMBS_SCHEDULER_TYPE}")
+message(STATUS "             - DATACRUMBS_SCHEDULER_JOBID_ENV_VAR: ${DATACRUMBS_SCHEDULER_JOBID_ENV_VAR}")
+message(STATUS "             - DATACRUMBS_SCHEDULER_NODES_CMD_OPT: ${DATACRUMBS_SCHEDULER_NODES_CMD_OPT}")
+message(STATUS "             - DATACRUMBS_SCHEDULER_PPN_CMD_OPT: ${DATACRUMBS_SCHEDULER_PPN_CMD_OPT}")
+message(STATUS "             - DATACRUMBS_SCHEDULER_RUN_CMD: ${DATACRUMBS_SCHEDULER_RUN_CMD}")
 
 if(NOT EXISTS "${DATACRUMBS_CONFIGURED_TRACE_DIR}")
     message(FATAL_ERROR "[${UPPER_PROJECT_NAME}] DATACRUMBS_CONFIGURED_TRACE_DIR does not exist: ${DATACRUMBS_CONFIGURED_TRACE_DIR}")
 endif()
+
 if(NOT EXISTS "${DATACRUMBS_CONFIGURED_LOG_DIR}")
     message(FATAL_ERROR "[${UPPER_PROJECT_NAME}] DATACRUMBS_CONFIGURED_LOG_DIR does not exist: ${DATACRUMBS_CONFIGURED_LOG_DIR}")
 endif()
@@ -424,7 +439,7 @@ add_dependencies(datacrumbs_obj datacrumbs_bpf_skel)
 
 # Build the final executable by linking server.cpp and the object library
 add_executable(${PROJECT_NAME} ${DATACRUMBS_SERVER_SRC})
-target_link_libraries(${PROJECT_NAME} ${DEPENDENCY_LIB} datacrumbs_obj)
+target_link_libraries(${PROJECT_NAME} datacrumbs_obj ${DEPENDENCY_LIB})
 add_dependencies(${PROJECT_NAME} datacrumbs_bpf_skel datacrumbs_obj)
 
 # Install the main processing executable into DATACRUMBS_INSTALL_BINARYDIR
@@ -587,6 +602,8 @@ set(_DATACRUMBS_DATA_FILES
     "${CMAKE_DATA_OUTPUT_DIRECTORY}/categories-${DATACRUMBS_USER}-${DATACRUMBS_HOST}.json"
     "${CMAKE_DATA_OUTPUT_DIRECTORY}/probes-${DATACRUMBS_USER}-${DATACRUMBS_HOST}.json"
 )
+# Ensure the data output directory exists
+file(MAKE_DIRECTORY ${CMAKE_DATA_OUTPUT_DIRECTORY})
 
 if(_DATACRUMBS_DATA_FILES)
     install(FILES ${_DATACRUMBS_DATA_FILES}
@@ -795,15 +812,15 @@ if(NOT DATACRUMBS_INCLUSION_PATH STREQUAL "NONE")
     set(DATACRUMBS_VARS ${DATACRUMBS_VARS} --inclusion_path ${DATACRUMBS_INCLUSION_PATH})
 endif()
 
-add_test(NAME datacrumbs_start COMMAND ${DATACRUMBS_PATH}/datacrumbs start ${DATACRUMBS_HOST} ${DATACRUMBS_VARS})
+add_test(NAME datacrumbs_start COMMAND ${DATACRUMBCMAKE_RUNTIME_OUTPUT_DIRECTORYS_PATH}/datacrumbs start ${DATACRUMBS_HOST} ${DATACRUMBS_VARS})
 set_tests_properties(datacrumbs_start PROPERTIES DEPENDS datacrumbs_pretest)
 set_tests_properties(datacrumbs_start PROPERTIES DEPENDS ${PROJECT_NAME})
 
-add_test(NAME datacrumbs_run COMMAND ${DATACRUMBS_PATH}/datacrumbs run ${DATACRUMBS_HOST} ${DATACRUMBS_VARS})
+add_test(NAME datacrumbs_run COMMAND ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/datacrumbs run ${DATACRUMBS_HOST} ${DATACRUMBS_VARS})
 set_tests_properties(datacrumbs_run PROPERTIES DEPENDS datacrumbs_pretest)
 set_tests_properties(datacrumbs_run PROPERTIES DEPENDS ${PROJECT_NAME})
 
-add_test(NAME datacrumbs_stop COMMAND ${DATACRUMBS_PATH}/datacrumbs stop ${DATACRUMBS_HOST} ${DATACRUMBS_VARS})
+add_test(NAME datacrumbs_stop COMMAND ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/datacrumbs stop ${DATACRUMBS_HOST} ${DATACRUMBS_VARS})
 set_tests_properties(datacrumbs_stop PROPERTIES DEPENDS datacrumbs_pretest)
 set_tests_properties(datacrumbs_stop PROPERTIES DEPENDS ${PROJECT_NAME})
 

cmake/configure_files/datacrumbs_config.h.in

@@ -42,6 +42,7 @@
 #cmakedefine DATACRUMBS_CONFIG_PATH "@DATACRUMBS_CONFIG_PATH@"
 #cmakedefine DATACRUMBS_LOG_DIR "@DATACRUMBS_LOG_DIR@"
 #cmakedefine DATACRUMBS_DATA_DIR "@DATACRUMBS_DATA_DIR@"
+#cmakedefine DATACRUMBS_INSTALL_RUNSTATEDIR "@DATACRUMBS_INSTALL_RUNSTATEDIR@/@PROJECT_NAME@"
 
 #cmakedefine DATACRUMBS_ENABLE_INCLUSION_PATH @DATACRUMBS_ENABLE_INCLUSION_PATH@
 #ifndef DATACRUMBS_ENABLE_INCLUSION_PATH

cmake/modules/datacrumbs-utils.cmake

@@ -31,6 +31,8 @@ macro(include_dependencies)
         message(FATAL_ERROR "[${UPPER_PROJECT_NAME}] libbpf version 1.0.0 or newer is required, but found ${LIBBPF_VERSION}")
     endif()
 
+    set(PKG_CONFIG_PATH $ENV{PKG_CONFIG_PATH})
+
     # include all links
     if(LIBBPF_FOUND)
         include_directories(${LIBBPF_INCLUDEDIR})
@@ -94,7 +96,7 @@ macro(include_dependencies)
     if(json-c_FOUND)
         get_filename_component(json-c_INCLUDE_DIR "${json-c_DIR}/../../../include" ABSOLUTE)
         get_filename_component(json-c_LIBRARY_DIR "${json-c_DIR}/../../" ABSOLUTE)
-        include_directories(json-c::json-c)
+        include_directories(${json-c_INCLUDE_DIR})
         list(APPEND DEPENDENCY_LIBRARY_DIRS ${json-c_LIBRARY_DIR})
         set(DEPENDENCY_LIB ${DEPENDENCY_LIB} -ljson-c)
     else()
@@ -105,7 +107,7 @@ macro(include_dependencies)
         include_directories(${ZLIB_INCLUDE_DIRS})
         get_filename_component(ZLIB_LIBRARY_DIRS "${ZLIB_LIBRARIES}/../" ABSOLUTE)
         list(APPEND DEPENDENCY_LIBRARY_DIRS ${ZLIB_LIBRARY_DIRS})
-        set(DEPENDENCY_LIB ${DEPENDENCY_LIB} ZLIB::ZLIB)
+        set(DEPENDENCY_LIB ${DEPENDENCY_LIB} -lz)
     else()
         message(FATAL_ERROR "-- [${UPPER_PROJECT_NAME}] zlib is needed for ${PROJECT_NAME} build")
     endif()
@@ -119,11 +121,13 @@ macro(include_dependencies)
             if(MPI_CXX_LIBRARIES)
                 # If MPI_CXX_LIBRARIES is a list, get parent dir of each library
                 set(MPI_CXX_LIBRARY_DIR "")
+
                 foreach(_lib ${MPI_CXX_LIBRARIES})
                     get_filename_component(_lib_dir "${_lib}" DIRECTORY)
                     get_filename_component(_lib_dir "${_lib_dir}" ABSOLUTE)
                     list(APPEND MPI_CXX_LIBRARY_DIR "${_lib_dir}")
                 endforeach()
+
                 list(REMOVE_DUPLICATES MPI_CXX_LIBRARY_DIR)
             else()
                 get_filename_component(MPI_CXX_LIBRARY_DIR "${MPI_CXX_LIBRARIES}" DIRECTORY)
@@ -152,6 +156,7 @@ macro(include_dependencies)
     string(REPLACE ";" ":" DEPENDENCY_LIBRARY_DIRS_COLON "${DEPENDENCY_LIBRARY_DIRS}")
     set(CMAKE_INSTALL_RPATH "${DEPENDENCY_LIBRARY_DIRS}")
     set(CMAKE_BUILD_RPATH "${DEPENDENCY_LIBRARY_DIRS}")
+
     # print_all_variables()
 endmacro(include_dependencies)
 
@@ -259,6 +264,59 @@ macro(derive_configurations)
     endif()
 
     set(DATACRUMBS_CMAKE_KERNEL_VERSION "(${KERNEL_VERSION_MAJOR}, ${KERNEL_VERSION_MINOR}, ${KERNEL_VERSION_PATCH})")
+
+    # Set Scheduler options
+    # Check if scheduler configuration is complete
+    if((DATACRUMBS_SCHEDULER_JOBID_ENV_VAR STREQUAL "NONE" OR 
+        DATACRUMBS_SCHEDULER_NODES_CMD_OPT STREQUAL "NONE" OR 
+        DATACRUMBS_SCHEDULER_PPN_CMD_OPT STREQUAL "NONE" OR 
+        DATACRUMBS_SCHEDULER_RUN_CMD STREQUAL "NONE") AND 
+        DATACRUMBS_SCHEDULER_TYPE STREQUAL "NONE")
+        message(FATAL_ERROR "[${UPPER_PROJECT_NAME}] Incomplete scheduler configuration. Either use a predefined scheduler option by setting DATACRUMBS_SCHEDULER_TYPE, or set all of the following variables: DATACRUMBS_SCHEDULER_ALLOC_CMD, DATACRUMBS_SCHEDULER_JOBID_ENV_VAR, DATACRUMBS_SCHEDULER_NODES_CMD_OPT, DATACRUMBS_SCHEDULER_PPN_CMD_OPT, DATACRUMBS_SCHEDULER_RUN_CMD")
+    endif()
+
+    if (DATACRUMBS_SCHEDULER_JOBID_ENV_VAR STREQUAL "NONE")
+        if(DATACRUMBS_SCHEDULER_TYPE STREQUAL "SLURM")
+            set(DATACRUMBS_SCHEDULER_JOBID_ENV_VAR "SLURM_JOB_ID")
+        elseif(DATACRUMBS_SCHEDULER_TYPE STREQUAL "OPENMPI")
+            set(DATACRUMBS_SCHEDULER_JOBID_ENV_VAR "OMPI_COMM_WORLD_RANK")
+        elseif(DATACRUMBS_SCHEDULER_TYPE STREQUAL "FLUX")
+            set(DATACRUMBS_SCHEDULER_JOBID_ENV_VAR "FLUX_JOB_ID")
+        endif()
+    endif()
+
+    if (DATACRUMBS_SCHEDULER_NODES_CMD_OPT STREQUAL "NONE")
+        if(DATACRUMBS_SCHEDULER_TYPE STREQUAL "SLURM")
+            set(DATACRUMBS_SCHEDULER_NODES_CMD_OPT "-N")
+        elseif(DATACRUMBS_SCHEDULER_TYPE STREQUAL "OPENMPI")
+            set(DATACRUMBS_SCHEDULER_NODES_CMD_OPT "-N")
+        elseif(DATACRUMBS_SCHEDULER_TYPE STREQUAL "FLUX")
+            set(DATACRUMBS_SCHEDULER_NODES_CMD_OPT "-N")
+        endif()
+    endif()
+
+    if (DATACRUMBS_SCHEDULER_PPN_CMD_OPT STREQUAL "NONE")
+        if(DATACRUMBS_SCHEDULER_TYPE STREQUAL "SLURM")
+            set(DATACRUMBS_SCHEDULER_PPN_CMD_OPT "--ntasks-per-node")
+        elseif(DATACRUMBS_SCHEDULER_TYPE STREQUAL "OPENMPI")
+            set(DATACRUMBS_SCHEDULER_PPN_CMD_OPT "--map-by ppr:")
+        elseif(DATACRUMBS_SCHEDULER_TYPE STREQUAL "FLUX")
+            set(DATACRUMBS_SCHEDULER_PPN_CMD_OPT "--tasks-per-node")
+        endif()
+    endif()
+
+    if (DATACRUMBS_SCHEDULER_RUN_CMD STREQUAL "NONE")
+        if(DATACRUMBS_SCHEDULER_TYPE STREQUAL "SLURM")
+            set(DATACRUMBS_SCHEDULER_RUN_CMD "srun")
+        elseif(DATACRUMBS_SCHEDULER_TYPE STREQUAL "OPENMPI")
+            set(DATACRUMBS_SCHEDULER_RUN_CMD "mpirun")
+        elseif(DATACRUMBS_SCHEDULER_TYPE STREQUAL "FLUX")
+            set(DATACRUMBS_SCHEDULER_RUN_CMD "flux run")
+        endif()
+    endif()
+
+
+
 endmacro(derive_configurations)
 
 macro(find_system_details)
@@ -419,10 +477,10 @@ macro(load_build_variables)
             set(CMAKE_INSTALL_INCLUDEDIR include)
             set(CMAKE_INSTALL_DOCDIR doc)
             set(CMAKE_INSTALL_SYSCONFDIR etc)
+            set(CMAKE_INSTALL_LIBEXECDIR libexec)
+            set(CMAKE_INSTALL_RUNSTATEDIR run)
         endif()
 
-        set(CMAKE_INSTALL_LIBEXEC_DIR libexec)
-
         set(DATACRUMBS_INSTALL_BINARYDIR
             ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR})
         set(DATACRUMBS_INSTALL_SBINARYDIR
@@ -436,7 +494,7 @@ macro(load_build_variables)
         set(DATACRUMBS_INSTALL_SYSCONFDIR
             ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_SYSCONFDIR}/${PROJECT_NAME})
         set(DATACRUMBS_INSTALL_LIBEXEC
-            ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBEXEC_DIR}/${PROJECT_NAME})
+            ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBEXECDIR}/${PROJECT_NAME})
     else()
         set(DATACRUMBS_INSTALL_BINARYDIR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}")
         set(DATACRUMBS_INSTALL_SBINARYDIR "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}")
@@ -445,9 +503,8 @@ macro(load_build_variables)
         set(DATACRUMBS_INSTALL_DOCDIR "${CMAKE_BINARY_DIR}/doc")
         set(DATACRUMBS_INSTALL_SYSCONFDIR "${CMAKE_BINARY_DIR}/etc/${PROJECT_NAME}")
         set(DATACRUMBS_INSTALL_LIBEXEC ${CMAKE_LIBEXEC_OUTPUT_DIRECTORY})
+        set(DATACRUMBS_RUNSTATEDIR "run")
     endif()
-
-    set(CMAKE_INSTALL_LIBEXEC_DIR libexec)
     set(CMAKE_INSTALL_CONFIGS_DIR configs)
     set(CMAKE_INSTALL_DATA_DIR data)
     set(CMAKE_INSTALL_MODULES_DIR lmod/modulefiles)
@@ -459,6 +516,11 @@ macro(load_build_variables)
     set(DATACRUMBS_INSTALL_ETC_CMAKE ${DATACRUMBS_INSTALL_SYSCONFDIR}/cmake)
     set(DATACRUMBS_INSTALL_ETC_SYSTEMD ${DATACRUMBS_INSTALL_SYSCONFDIR}/systemd)
     set(DATACRUMBS_INSTALL_ETC_FLUX ${DATACRUMBS_INSTALL_SYSCONFDIR}/flux)
+    if (DATACRUMBS_CONFIGURED_RUN_DIR AND NOT DATACRUMBS_CONFIGURED_RUN_DIR STREQUAL "" AND NOT DATACRUMBS_CONFIGURED_RUN_DIR STREQUAL "NONE")
+        set(DATACRUMBS_INSTALL_RUNSTATEDIR ${DATACRUMBS_CONFIGURED_RUN_DIR})
+    else()
+        set(DATACRUMBS_INSTALL_RUNSTATEDIR /${CMAKE_INSTALL_RUNSTATEDIR})
+    endif()
 
     # Set this at the top level of your project, before any install commands
     set(CMAKE_SKIP_INSTALL_ALL_DEPENDENCY TRUE)