UNOMI-957: Add cross-platform memory sampling for integration test runs

.github/workflows/unomi-ci-build-tests.yml

@@ -70,6 +70,16 @@ jobs:
           else
             ./build.sh --ci --integration-tests
           fi
+      - name: Archive IT memory metrics
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: it-memory-metrics-jdk17-${{ matrix.search-engine }}-${{ github.run_number }}
+          path: |
+            itests/target/memory-samples.tsv
+            itests/target/memory-summary.txt
+            itests/target/memory-sampler.log
+          if-no-files-found: ignore
       - name: Archive code coverage logs
         uses: actions/upload-artifact@v4
         if: false # UNOMI-746 Reactivate if necessary

build.sh

@@ -272,6 +272,8 @@ IT_DEBUG_PORT=5006
 IT_DEBUG_SUSPEND=false
 SKIP_MIGRATION_TESTS=false
 KEEP_CONTAINER=false
+IT_MEMORY_SAMPLER=true
+IT_MEMORY_INTERVAL=30
 JAVADOC=false
 LOG_FILE=""
 LOG_FILE_ONLY=false
@@ -315,6 +317,8 @@ EOF
         echo -e "  ${CYAN}--it-debug-suspend${NC}         Suspend integration test until debugger connects"
         echo -e "  ${CYAN}--skip-migration-tests${NC}     Skip migration-related tests"
         echo -e "  ${CYAN}--keep-container${NC}           Keep search engine container running after tests (for post-failure inspection)"
+        echo -e "  ${CYAN}--no-memory-sampler${NC}        Disable JVM/system memory sampling during integration tests"
+        echo -e "  ${CYAN}--memory-interval SEC${NC}    Memory sample interval in seconds (default: 30)"
         echo -e "  ${CYAN}--javadoc${NC}                  Build and validate Javadoc after install (fails on doclint errors)"
         echo -e "  ${CYAN}--ci${NC}                       CI mode: no Karaf, no Maven build cache, non-interactive, includes Javadoc"
         echo -e "  ${CYAN}--log-file PATH${NC}            Tee all output to PATH (console + file)"
@@ -354,6 +358,8 @@ EOF
         echo "  --it-debug-suspend        Suspend integration test until debugger connects"
         echo "  --skip-migration-tests    Skip migration-related tests"
         echo "  --keep-container          Keep search engine container running after tests (for post-failure inspection)"
+        echo "  --no-memory-sampler       Disable JVM/system memory sampling during integration tests"
+        echo "  --memory-interval SEC     Memory sample interval in seconds (default: 30)"
         echo "  --javadoc                 Build and validate Javadoc after install (fails on doclint errors)"
         echo "  --ci                      CI mode: no Karaf, no Maven build cache, non-interactive, includes Javadoc"
         echo "  --log-file PATH           Tee all output to PATH (console + file)"
@@ -505,6 +511,13 @@ while [ "$1" != "" ]; do
         --keep-container)
             KEEP_CONTAINER=true
             ;;
+        --no-memory-sampler)
+            IT_MEMORY_SAMPLER=false
+            ;;
+        --memory-interval)
+            shift
+            IT_MEMORY_INTERVAL="$1"
+            ;;
         --javadoc)
             JAVADOC=true
             ;;
@@ -1037,6 +1050,8 @@ write_it_run_trace_start() {
         echo "it.debug.suspend=$IT_DEBUG_SUSPEND"
         echo "skip.migration.tests=$SKIP_MIGRATION_TESTS"
         echo "it.keep.container=$KEEP_CONTAINER"
+        echo "it.memory.sampler=$IT_MEMORY_SAMPLER"
+        echo "it.memory.interval=$IT_MEMORY_INTERVAL"
         echo "maven.debug=$MAVEN_DEBUG"
         echo "maven.offline=$MAVEN_OFFLINE"
         echo "maven.quiet=$MAVEN_QUIET"
@@ -1048,6 +1063,31 @@ write_it_run_trace_start() {
     } > "$trace_file"
 }
 
+
+start_it_memory_sampler() {
+    local sampler="$DIRNAME/itests/sample-it-memory.sh"
+    if [ "$IT_MEMORY_SAMPLER" != true ] || [ ! -f "$sampler" ]; then
+        return 0
+    fi
+    bash "$sampler" start --target-dir "$DIRNAME/itests/target" --interval "$IT_MEMORY_INTERVAL" || \
+        print_status "warning" "Could not start IT memory sampler"
+}
+
+write_it_run_operator_note() {
+    local sampler="$DIRNAME/itests/sample-it-memory.sh"
+    if [ -f "$sampler" ]; then
+        bash "$sampler" operator-note --target-dir "$DIRNAME/itests/target" 2>/dev/null || true
+    fi
+}
+
+stop_it_memory_sampler() {
+    local sampler="$DIRNAME/itests/sample-it-memory.sh"
+    if [ ! -f "$sampler" ]; then
+        return 0
+    fi
+    bash "$sampler" stop --target-dir "$DIRNAME/itests/target" 2>/dev/null || true
+}
+
 finalize_it_run_trace() {
     local exit_code="$1"
     local trace_file="$DIRNAME/itests/target/it-run-trace.properties"
@@ -1074,6 +1114,7 @@ $MVN_CMD clean $MVN_OPTS || {
 
 if [ "$RUN_INTEGRATION_TESTS" = true ]; then
     write_it_run_trace_start
+    start_it_memory_sampler
 fi
 
 print_progress $((++current_step)) $total_steps "Compiling and installing artifacts..."
@@ -1085,7 +1126,9 @@ fi
 INSTALL_EXIT=0
 $MVN_CMD install $MVN_OPTS || INSTALL_EXIT=$?
 if [ "$RUN_INTEGRATION_TESTS" = true ]; then
+    stop_it_memory_sampler
     finalize_it_run_trace "$INSTALL_EXIT"
+    write_it_run_operator_note
 fi
 if [ "$INSTALL_EXIT" -ne 0 ]; then
     print_status "error" "Maven install failed"

itests/README.md

@@ -45,8 +45,60 @@ run under `itests/target/exam/` with a UUID directory name.
 | `kt.sh` | Live Karaf inspection during a run (log, tail, grep, debug) |
 | `archive-it-run.sh` | Capture IT run artifacts before the next build wipes them |
 | `compare-it-runs.sh` | Diff multiple captures to classify flaky vs systematic failures |
+| `sample-it-memory.sh` | Memory sampling, summarize, operator note, and cross-platform verify |
 | `jacoco-report.sh` | Generate a JaCoCo coverage report after a run |
 
+
+---
+
+## Memory sizing analysis
+
+`./build.sh --integration-tests` starts a background memory sampler automatically.
+Results land in `itests/target/`:
+
+| File | Purpose |
+|------|---------|
+| `memory-samples.tsv` | Time-series samples (Karaf heap, search heap, Docker RSS, swap) |
+| `memory-summary.txt` | Peak usage, headroom %, peak Karaf GCT (seconds), swap warnings |
+| `memory-sampler.log` | Sampler diagnostics |
+
+Disable sampling with `--no-memory-sampler`, or change the interval with
+`--memory-interval 60` (default: 30s).
+
+Each sample uses one header-aware `jstat -gc` attach (cached `MaxHeapSize` per Karaf PID,
+with `jinfo`/`jcmd`/trace fallbacks), a filtered ES `/_nodes/stats/jvm` request, and
+`docker stats` on the IT container only (`itests-elasticsearch` or `itests-opensearch`).
+System metrics use `vm_stat`/`sysctl` on macOS and `free`/`/proc/loadavg` on Linux.
+
+Swap warnings fire only when available RAM drops below 2 GB during the run or swap
+grows by more than 256 MB (avoids false positives from baseline macOS swap).
+
+Verify locally:
+
+```bash
+cd itests
+./sample-it-memory.sh verify
+```
+
+Manual usage:
+
+```bash
+./itests/sample-it-memory.sh start --interval 30
+./itests/sample-it-memory.sh stop    # writes memory-summary.txt
+./itests/sample-it-memory.sh summarize
+./itests/sample-it-memory.sh operator-note --print-only
+```
+
+After a run, `build.sh` writes `itests/target/it-run-operator-note.txt` with outcome,
+heap config, test counts, and memory peaks. `archive-it-run.sh` uses that file as the
+default operator note when you omit `-m` (override with `-m` or `--message-file` as before).
+
+The archive also includes memory files and adds peak metrics to `run-summary.properties`.
+On GitHub Actions, download the `it-memory-metrics-*` artifact from the workflow run.
+
+Compare heap configurations across runs using `archives/runs-index.tsv` (configured
+heaps + observed peaks from `memory.peak.*` fields in each capture).
+
 ---
 
 ## Writing Integration Tests
@@ -325,8 +377,8 @@ wipes `itests/target/`:
 
 ```bash
 cd itests
-./archive-it-run.sh
-./archive-it-run.sh -m "Heavy swap, 2 failures in GraphQLListIT"   # with an operator note
+./archive-it-run.sh                                    # uses auto-generated operator note
+./archive-it-run.sh -m "Heavy swap, 2 failures in GraphQLListIT"   # override auto note
 ./archive-it-run.sh --full-karaf                                    # include complete Karaf log
 ```
 

itests/archive-it-run.sh

@@ -48,6 +48,8 @@
 #   - llm-it-run-analysis-guide.md, expected-karaf-log-patterns.txt
 #   - exam/.../karaf-unexpected-candidates.log (errors not matching expected patterns)
 #   - test-results.tsv, run-summary.properties, failed-tests.txt (LLM-friendly per-run test manifest)
+#   - it-run-operator-note.txt (auto-generated operator note from build.sh)
+#   - memory-samples.tsv, memory-summary.txt (JVM/system memory observed during IT run)
 #   - run-context.txt, run-config/it-run-trace.properties (build/Maven/options trace)
 #   - archives/runs-index.tsv (updated each capture — cross-run comparison index)
 #   - comparison-last-3.txt, archives/latest-comparison.txt (auto when 2+ captures exist)
@@ -64,6 +66,8 @@ REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
 source "$SCRIPT_DIR/lib/it-run-bootstrap.sh"
 # shellcheck source=lib/it-run-karaf.sh disable=SC1091
 source "$SCRIPT_DIR/lib/it-run-karaf.sh"
+# shellcheck source=lib/it-run-memory.sh disable=SC1091
+source "$SCRIPT_DIR/lib/it-run-memory.sh"
 # shellcheck source=lib/it-run-context.sh disable=SC1091
 source "$SCRIPT_DIR/lib/it-run-context.sh"
 
@@ -85,6 +89,7 @@ RUN_CONFIG_FILES=(
     elasticsearch-port.properties
     opensearch-port.properties
     it-run-trace.properties
+    it-run-operator-note.txt
 )
 
 ENGINE_LOG_TREES=(
@@ -113,14 +118,25 @@ usage() {
     echo "  (default)     Unexploded directory: archives/it-run-YYYYMMDD-HHMMSS/"
     echo "  --tar         Write a .tar.gz instead (default name under archives/)"
     echo "  -o PATH       Output directory, or .tar.gz / .tgz archive path"
-    echo "  -m, --message Operator note about run context (quoted string)"
+    echo "  -m, --message Operator note (default: it-run-operator-note.txt from build.sh)"
     echo "  --message-file  Read operator note from a file"
     echo "  --full-karaf  Include complete karaf.log and rollover segments (default: tail + filtered errors)"
     echo "  --no-compare  Skip auto compare of last 3 captures (default: on when 2+ runs exist)"
     echo "  --force       Archive even if this target/ run was already captured"
     exit 1
 }
 
+
+load_default_operator_note() {
+    if [ -n "$RUN_MESSAGE" ]; then
+        return 0
+    fi
+    if RUN_MESSAGE="$(it_load_default_operator_note "$TARGET_DIR" "$SCRIPT_DIR" 2>/dev/null)"; then
+        ui_detail "Using auto-generated operator note from $IT_OPERATOR_NOTE_FILE"
+        return 0
+    fi
+}
+
 parse_args() {
     while [ $# -gt 0 ]; do
         case "$1" in
@@ -373,6 +389,10 @@ write_run_summary() {
             echo "tests.skipped=$(it_failsafe_summary_count "$summary_xml" skipped)"
         fi
         echo "failed.tests.count=$failed_count"
+        if [ -f "$(target_path "$IT_MEMORY_SUMMARY")" ]; then
+            grep -E '^memory\.(peak|min|warning|samples|karaf|search)\.' \
+                "$(target_path "$IT_MEMORY_SUMMARY")" 2>/dev/null || true
+        fi
         if [ -n "$RUN_FINGERPRINT" ]; then
             echo "${IT_RUN_FINGERPRINT_FIELD}=$RUN_FINGERPRINT"
         fi
@@ -416,11 +436,24 @@ archive_engine_log_trees() {
     done
 }
 
+archive_memory_artifacts() {
+    local name samples summary
+    for name in "$IT_MEMORY_SAMPLES" "$IT_MEMORY_SUMMARY" "$IT_MEMORY_SAMPLER_LOG"; do
+        copy_file "$(target_path "$name")" "$name"
+    done
+    samples="$(target_path "$IT_MEMORY_SAMPLES")"
+    summary="$(target_path "$IT_MEMORY_SUMMARY")"
+    if [ -f "$samples" ] && [ ! -f "$summary" ]; then
+        it_memory_summarize_samples "$samples" "$summary" && copy_file "$summary" "$IT_MEMORY_SUMMARY"
+    fi
+}
+
 archive_test_artifacts() {
     archive_report_trees
     capture_test_manifest
     archive_run_config_files
     archive_engine_log_trees
+    archive_memory_artifacts
     write_run_context
 }
 
@@ -493,6 +526,7 @@ finalize_output() {
 main() {
     it_run_entry_init "$SCRIPT_DIR"
     parse_args "$@"
+    load_default_operator_note
 
     require_target_dir
     reject_duplicate_archive

itests/lib/it-run-context.sh

@@ -20,6 +20,7 @@
 # Expects archive hooks: target_path, staging_path, log_staged_file, mark_included,
 # and globals: SCRIPT_DIR, REPO_ROOT, TARGET_DIR, MANIFEST, OUTPUT, RUN_MESSAGE,
 # RUN_ID, FULL_KARAF, CREATE_TAR, RUN_CONTEXT_REL, RUN_TRACE_FILE.
+# Requires it-run-memory.sh to be sourced by the caller (archive-it-run.sh).
 
 context_append_inferred_maven_properties() {
     local engine
@@ -72,6 +73,7 @@ context_append_system_snapshot() {
     if [ "$(uname -s 2>/dev/null)" = Darwin ] && command -v memory_pressure >/dev/null 2>&1; then
         echo "snapshot.memory_pressure=$(memory_pressure 2>/dev/null | head -1)"
     fi
+    it_memory_append_system_snapshot
 }
 
 context_emit_manifest_git_lines() {
@@ -151,6 +153,10 @@ write_run_context() {
         echo
         echo "## System snapshot (at archive time — not at IT start)"
         context_append_system_snapshot
+        echo
+        it_memory_append_context_section \
+            "$(target_path "$IT_MEMORY_SUMMARY")" \
+            "$(target_path "$IT_MEMORY_SAMPLES")"
     } > "$out"
     log_staged_file "$RUN_CONTEXT_REL" "Wrote $RUN_CONTEXT_REL"
     mark_included