From efb67f093f870d007690da72749b1fdcde176c4c Mon Sep 17 00:00:00 2001 From: Evan Wang Date: Wed, 10 Jun 2026 14:38:47 -0400 Subject: [PATCH 1/2] feat: collect pod resource usage and cgroup metrics in skupper-gather --- scripts/skupper-gather.sh | 97 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/scripts/skupper-gather.sh b/scripts/skupper-gather.sh index 5cfc2572f..0bdaf46c5 100644 --- a/scripts/skupper-gather.sh +++ b/scripts/skupper-gather.sh @@ -16,6 +16,102 @@ get_log_collection_args() { fi } +function getPodResources() { + local ns="${1}" + local podName="${2}" + local container="${3}" + local podDirName="${podName##"pod/"}" + + local resourcePath=${BASE_COLLECTION_PATH}/namespaces/${ns}/pods/${podDirName}/${container}/${container}/resources + mkdir -p "${resourcePath}" + + echo "Collecting resource usage for pod ${podName} in ${ns}" + + # Memory Info + oc exec -n "${ns}" "${podName}" -c "${container}" -- bash -c ' + curr=$(cat /sys/fs/cgroup/memory.current) + max=$(cat /sys/fs/cgroup/memory.max) + curr_kb=$((curr / 1024)) + curr_mb=$((curr / 1024 / 1024)) + + if [ "$max" = "max" ]; then + limit_str="max (uncapped)" + else + max_kb=$((max / 1024)) + max_mb=$((max / 1024 / 1024)) + limit_str="$max bytes ($max_kb KiB / $max_mb MiB)" + fi + + echo "Usage: $curr bytes ($curr_kb KiB / $curr_mb MiB) - Limit: $limit_str" > /tmp/memory.info + ' 2>/dev/null + oc exec -n "${ns}" "${podName}" -c "${container}" -- cat /tmp/memory.info > "${resourcePath}/memory.info" 2>&1 + +# Get previous container termination info +oc get pod "${podName##"pod/"}" -n "${ns}" -o jsonpath="{.status.containerStatuses[?(@.name=='${container}')].lastState}" > "${resourcePath}/last.terminated" 2>/dev/null +if [ ! -s "${resourcePath}/last.terminated" ]; then + echo "No previous termination recorded" > "${resourcePath}/last.terminated" +fi + + # Memory pressure +oc exec -n "${ns}" "${podName}" -c "${container}" -- bash -c ' + if [ -f /sys/fs/cgroup/memory.pressure ]; then + cat /sys/fs/cgroup/memory.pressure + else + echo "PSI not available on this node (kernel psi=1 not enabled)" + fi +' > "${resourcePath}/memory.pressure" 2>&1 + +# CPU usage +oc exec -n "${ns}" "${podName}" -c "${container}" -- bash -c " + declare -A cpu_descriptions + cpu_descriptions=( + [usage_usec]='Total CPU time consumed (user + system)' + [user_usec]='Time spent in user-space (app logic)' + [system_usec]='Time spent in kernel-space (syscalls, I/O)' + [core_sched.force_idle_usec]='Time CPU forced idle for security (cross-HT side-channel protection)' + [nr_periods]='Number of CPU quota enforcement periods (0 = no quota set)' + [nr_throttled]='Times the container was throttled for exceeding CPU quota' + [throttled_usec]='Total time spent throttled/waiting for CPU quota' + [nr_bursts]='Times container used burst CPU capacity beyond quota' + [burst_usec]='Total time spent using burst CPU capacity' + ) + + cat /sys/fs/cgroup/cpu.stat | while read line; do + val=\$(echo \$line | awk '{print \$2}') + key=\$(echo \$line | awk '{print \$1}') + desc=\${cpu_descriptions[\$key]} + if [[ \$key == *_usec ]] && [ \$val -ge 1000000 ]; then + sec=\$(awk \"BEGIN {printf \\\"%.2f\\\", \$val/1000000}\") + echo \"\$line (\$sec s) | \$desc\" + else + echo \"\$line | \$desc\" + fi + done +" > "${resourcePath}/cpu.stat" 2>&1 + + # CPU limit + oc exec -n "${ns}" "${podName}" -c "${container}" -- bash -c ' + cpu_max=$(cat /sys/fs/cgroup/cpu.max) + quota=$(echo $cpu_max | awk "{print \$1}") + period=$(echo $cpu_max | awk "{print \$2}") + if [ "$quota" = "max" ]; then + echo "CPU Limit: uncapped (no quota set)" + else + cores=$(awk "BEGIN {printf \"%.2f\", $quota/$period}") + echo "CPU Limit: $quota/$period us = $cores cores" + fi + ' > "${resourcePath}/cpu.max" 2>&1 + + # CPU pressure +oc exec -n "${ns}" "${podName}" -c "${container}" -- bash -c ' + if [ -f /sys/fs/cgroup/cpu.pressure ]; then + cat /sys/fs/cgroup/cpu.pressure + else + echo "PSI not available on this node (kernel psi=1 not enabled)" + fi +' > "${resourcePath}/cpu.pressure" 2>&1 +} + function getSkupperRouterSkstatInNamespace() { local routerNamespace="${1}" local flags=("g" "c" "l" "n" "e" "a" "m" "p") @@ -28,6 +124,7 @@ function getSkupperRouterSkstatInNamespace() { for flag in "${flags[@]}"; do oc exec -n "${sitens}" "${routerName}" -c router -- skstat -"${flag}" > "${logPath}/skstat.${flag}" 2>&1 done + getPodResources "${sitens}" "${routerName}" "router" done } From 976865f9a83eccb5844eface33b89eeb243df9c9 Mon Sep 17 00:00:00 2001 From: Evan Date: Sun, 5 Jul 2026 22:23:26 -0700 Subject: [PATCH 2/2] Changed code to retrieve from all containers --- scripts/skupper-gather.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/scripts/skupper-gather.sh b/scripts/skupper-gather.sh index 0bdaf46c5..db9a434d6 100644 --- a/scripts/skupper-gather.sh +++ b/scripts/skupper-gather.sh @@ -124,8 +124,17 @@ function getSkupperRouterSkstatInNamespace() { for flag in "${flags[@]}"; do oc exec -n "${sitens}" "${routerName}" -c router -- skstat -"${flag}" > "${logPath}/skstat.${flag}" 2>&1 done - getPodResources "${sitens}" "${routerName}" "router" - done + done +} + +function getSkupperResourcesInNamespace() { + local ns="${1}" + + for podName in $(oc get pods -n "${ns}" -l app.kubernetes.io/part-of=skupper -oname); do + for container in $(oc get "${podName}" -n "${ns}" -o jsonpath='{.spec.containers[*].name}'); do + getPodResources "${ns}" "${podName}" "${container}" + done + done } function getCRDs() { @@ -210,12 +219,14 @@ function main() { inspectNamespace "${sitens}" getSkupperRouterSkstatInNamespace "${sitens}" + getSkupperResourcesInNamespace "${sitens}" done # iterate over all namespaces which have AttachedConnectors for acns in $(oc get attachedconnector -A -o=jsonpath="{.items[*].metadata.namespace}"); do echo "Inspecting AttachedConnector in ${acns} namespace" inspectNamespace "${acns}" + getSkupperResourcesInNamespace "${acns}" done echo