diff --git a/.github/workflows/unomi-ci-build-tests.yml b/.github/workflows/unomi-ci-build-tests.yml index af90f2747..2b7e190dd 100644 --- a/.github/workflows/unomi-ci-build-tests.yml +++ b/.github/workflows/unomi-ci-build-tests.yml @@ -70,6 +70,16 @@ jobs: else ./build.sh --ci --integration-tests fi + - name: Archive IT memory metrics + uses: actions/upload-artifact@v4 + if: always() + with: + name: it-memory-metrics-jdk17-${{ matrix.search-engine }}-${{ github.run_number }} + path: | + itests/target/memory-samples.tsv + itests/target/memory-summary.txt + itests/target/memory-sampler.log + if-no-files-found: ignore - name: Archive code coverage logs uses: actions/upload-artifact@v4 if: false # UNOMI-746 Reactivate if necessary diff --git a/build.sh b/build.sh index a88ce8a0f..c6eaa7837 100755 --- a/build.sh +++ b/build.sh @@ -272,6 +272,8 @@ IT_DEBUG_PORT=5006 IT_DEBUG_SUSPEND=false SKIP_MIGRATION_TESTS=false KEEP_CONTAINER=false +IT_MEMORY_SAMPLER=true +IT_MEMORY_INTERVAL=30 JAVADOC=false LOG_FILE="" LOG_FILE_ONLY=false @@ -315,6 +317,8 @@ EOF echo -e " ${CYAN}--it-debug-suspend${NC} Suspend integration test until debugger connects" echo -e " ${CYAN}--skip-migration-tests${NC} Skip migration-related tests" echo -e " ${CYAN}--keep-container${NC} Keep search engine container running after tests (for post-failure inspection)" + echo -e " ${CYAN}--no-memory-sampler${NC} Disable JVM/system memory sampling during integration tests" + echo -e " ${CYAN}--memory-interval SEC${NC} Memory sample interval in seconds (default: 30)" echo -e " ${CYAN}--javadoc${NC} Build and validate Javadoc after install (fails on doclint errors)" echo -e " ${CYAN}--ci${NC} CI mode: no Karaf, no Maven build cache, non-interactive, includes Javadoc" echo -e " ${CYAN}--log-file PATH${NC} Tee all output to PATH (console + file)" @@ -354,6 +358,8 @@ EOF echo " --it-debug-suspend Suspend integration test until debugger connects" echo " --skip-migration-tests Skip migration-related tests" echo " --keep-container Keep search engine container running after tests (for post-failure inspection)" + echo " --no-memory-sampler Disable JVM/system memory sampling during integration tests" + echo " --memory-interval SEC Memory sample interval in seconds (default: 30)" echo " --javadoc Build and validate Javadoc after install (fails on doclint errors)" echo " --ci CI mode: no Karaf, no Maven build cache, non-interactive, includes Javadoc" echo " --log-file PATH Tee all output to PATH (console + file)" @@ -505,6 +511,13 @@ while [ "$1" != "" ]; do --keep-container) KEEP_CONTAINER=true ;; + --no-memory-sampler) + IT_MEMORY_SAMPLER=false + ;; + --memory-interval) + shift + IT_MEMORY_INTERVAL="$1" + ;; --javadoc) JAVADOC=true ;; @@ -1037,6 +1050,8 @@ write_it_run_trace_start() { echo "it.debug.suspend=$IT_DEBUG_SUSPEND" echo "skip.migration.tests=$SKIP_MIGRATION_TESTS" echo "it.keep.container=$KEEP_CONTAINER" + echo "it.memory.sampler=$IT_MEMORY_SAMPLER" + echo "it.memory.interval=$IT_MEMORY_INTERVAL" echo "maven.debug=$MAVEN_DEBUG" echo "maven.offline=$MAVEN_OFFLINE" echo "maven.quiet=$MAVEN_QUIET" @@ -1048,6 +1063,31 @@ write_it_run_trace_start() { } > "$trace_file" } + +start_it_memory_sampler() { + local sampler="$DIRNAME/itests/sample-it-memory.sh" + if [ "$IT_MEMORY_SAMPLER" != true ] || [ ! -f "$sampler" ]; then + return 0 + fi + bash "$sampler" start --target-dir "$DIRNAME/itests/target" --interval "$IT_MEMORY_INTERVAL" || \ + print_status "warning" "Could not start IT memory sampler" +} + +write_it_run_operator_note() { + local sampler="$DIRNAME/itests/sample-it-memory.sh" + if [ -f "$sampler" ]; then + bash "$sampler" operator-note --target-dir "$DIRNAME/itests/target" 2>/dev/null || true + fi +} + +stop_it_memory_sampler() { + local sampler="$DIRNAME/itests/sample-it-memory.sh" + if [ ! -f "$sampler" ]; then + return 0 + fi + bash "$sampler" stop --target-dir "$DIRNAME/itests/target" 2>/dev/null || true +} + finalize_it_run_trace() { local exit_code="$1" local trace_file="$DIRNAME/itests/target/it-run-trace.properties" @@ -1074,6 +1114,7 @@ $MVN_CMD clean $MVN_OPTS || { if [ "$RUN_INTEGRATION_TESTS" = true ]; then write_it_run_trace_start + start_it_memory_sampler fi print_progress $((++current_step)) $total_steps "Compiling and installing artifacts..." @@ -1085,7 +1126,9 @@ fi INSTALL_EXIT=0 $MVN_CMD install $MVN_OPTS || INSTALL_EXIT=$? if [ "$RUN_INTEGRATION_TESTS" = true ]; then + stop_it_memory_sampler finalize_it_run_trace "$INSTALL_EXIT" + write_it_run_operator_note fi if [ "$INSTALL_EXIT" -ne 0 ]; then print_status "error" "Maven install failed" diff --git a/itests/README.md b/itests/README.md index b2500f8fb..e3929d611 100644 --- a/itests/README.md +++ b/itests/README.md @@ -45,8 +45,60 @@ run under `itests/target/exam/` with a UUID directory name. | `kt.sh` | Live Karaf inspection during a run (log, tail, grep, debug) | | `archive-it-run.sh` | Capture IT run artifacts before the next build wipes them | | `compare-it-runs.sh` | Diff multiple captures to classify flaky vs systematic failures | +| `sample-it-memory.sh` | Memory sampling, summarize, operator note, and cross-platform verify | | `jacoco-report.sh` | Generate a JaCoCo coverage report after a run | + +--- + +## Memory sizing analysis + +`./build.sh --integration-tests` starts a background memory sampler automatically. +Results land in `itests/target/`: + +| File | Purpose | +|------|---------| +| `memory-samples.tsv` | Time-series samples (Karaf heap, search heap, Docker RSS, swap) | +| `memory-summary.txt` | Peak usage, headroom %, peak Karaf GCT (seconds), swap warnings | +| `memory-sampler.log` | Sampler diagnostics | + +Disable sampling with `--no-memory-sampler`, or change the interval with +`--memory-interval 60` (default: 30s). + +Each sample uses one header-aware `jstat -gc` attach (cached `MaxHeapSize` per Karaf PID, +with `jinfo`/`jcmd`/trace fallbacks), a filtered ES `/_nodes/stats/jvm` request, and +`docker stats` on the IT container only (`itests-elasticsearch` or `itests-opensearch`). +System metrics use `vm_stat`/`sysctl` on macOS and `free`/`/proc/loadavg` on Linux. + +Swap warnings fire only when available RAM drops below 2 GB during the run or swap +grows by more than 256 MB (avoids false positives from baseline macOS swap). + +Verify locally: + +```bash +cd itests +./sample-it-memory.sh verify +``` + +Manual usage: + +```bash +./itests/sample-it-memory.sh start --interval 30 +./itests/sample-it-memory.sh stop # writes memory-summary.txt +./itests/sample-it-memory.sh summarize +./itests/sample-it-memory.sh operator-note --print-only +``` + +After a run, `build.sh` writes `itests/target/it-run-operator-note.txt` with outcome, +heap config, test counts, and memory peaks. `archive-it-run.sh` uses that file as the +default operator note when you omit `-m` (override with `-m` or `--message-file` as before). + +The archive also includes memory files and adds peak metrics to `run-summary.properties`. +On GitHub Actions, download the `it-memory-metrics-*` artifact from the workflow run. + +Compare heap configurations across runs using `archives/runs-index.tsv` (configured +heaps + observed peaks from `memory.peak.*` fields in each capture). + --- ## Writing Integration Tests @@ -325,8 +377,8 @@ wipes `itests/target/`: ```bash cd itests -./archive-it-run.sh -./archive-it-run.sh -m "Heavy swap, 2 failures in GraphQLListIT" # with an operator note +./archive-it-run.sh # uses auto-generated operator note +./archive-it-run.sh -m "Heavy swap, 2 failures in GraphQLListIT" # override auto note ./archive-it-run.sh --full-karaf # include complete Karaf log ``` diff --git a/itests/archive-it-run.sh b/itests/archive-it-run.sh index cb9fafa60..afb161ef9 100755 --- a/itests/archive-it-run.sh +++ b/itests/archive-it-run.sh @@ -48,6 +48,8 @@ # - llm-it-run-analysis-guide.md, expected-karaf-log-patterns.txt # - exam/.../karaf-unexpected-candidates.log (errors not matching expected patterns) # - test-results.tsv, run-summary.properties, failed-tests.txt (LLM-friendly per-run test manifest) +# - it-run-operator-note.txt (auto-generated operator note from build.sh) +# - memory-samples.tsv, memory-summary.txt (JVM/system memory observed during IT run) # - run-context.txt, run-config/it-run-trace.properties (build/Maven/options trace) # - archives/runs-index.tsv (updated each capture — cross-run comparison index) # - comparison-last-3.txt, archives/latest-comparison.txt (auto when 2+ captures exist) @@ -64,6 +66,8 @@ REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" source "$SCRIPT_DIR/lib/it-run-bootstrap.sh" # shellcheck source=lib/it-run-karaf.sh disable=SC1091 source "$SCRIPT_DIR/lib/it-run-karaf.sh" +# shellcheck source=lib/it-run-memory.sh disable=SC1091 +source "$SCRIPT_DIR/lib/it-run-memory.sh" # shellcheck source=lib/it-run-context.sh disable=SC1091 source "$SCRIPT_DIR/lib/it-run-context.sh" @@ -85,6 +89,7 @@ RUN_CONFIG_FILES=( elasticsearch-port.properties opensearch-port.properties it-run-trace.properties + it-run-operator-note.txt ) ENGINE_LOG_TREES=( @@ -113,7 +118,7 @@ usage() { echo " (default) Unexploded directory: archives/it-run-YYYYMMDD-HHMMSS/" echo " --tar Write a .tar.gz instead (default name under archives/)" echo " -o PATH Output directory, or .tar.gz / .tgz archive path" - echo " -m, --message Operator note about run context (quoted string)" + echo " -m, --message Operator note (default: it-run-operator-note.txt from build.sh)" echo " --message-file Read operator note from a file" echo " --full-karaf Include complete karaf.log and rollover segments (default: tail + filtered errors)" echo " --no-compare Skip auto compare of last 3 captures (default: on when 2+ runs exist)" @@ -121,6 +126,17 @@ usage() { exit 1 } + +load_default_operator_note() { + if [ -n "$RUN_MESSAGE" ]; then + return 0 + fi + if RUN_MESSAGE="$(it_load_default_operator_note "$TARGET_DIR" "$SCRIPT_DIR" 2>/dev/null)"; then + ui_detail "Using auto-generated operator note from $IT_OPERATOR_NOTE_FILE" + return 0 + fi +} + parse_args() { while [ $# -gt 0 ]; do case "$1" in @@ -373,6 +389,10 @@ write_run_summary() { echo "tests.skipped=$(it_failsafe_summary_count "$summary_xml" skipped)" fi echo "failed.tests.count=$failed_count" + if [ -f "$(target_path "$IT_MEMORY_SUMMARY")" ]; then + grep -E '^memory\.(peak|min|warning|samples|karaf|search)\.' \ + "$(target_path "$IT_MEMORY_SUMMARY")" 2>/dev/null || true + fi if [ -n "$RUN_FINGERPRINT" ]; then echo "${IT_RUN_FINGERPRINT_FIELD}=$RUN_FINGERPRINT" fi @@ -416,11 +436,24 @@ archive_engine_log_trees() { done } +archive_memory_artifacts() { + local name samples summary + for name in "$IT_MEMORY_SAMPLES" "$IT_MEMORY_SUMMARY" "$IT_MEMORY_SAMPLER_LOG"; do + copy_file "$(target_path "$name")" "$name" + done + samples="$(target_path "$IT_MEMORY_SAMPLES")" + summary="$(target_path "$IT_MEMORY_SUMMARY")" + if [ -f "$samples" ] && [ ! -f "$summary" ]; then + it_memory_summarize_samples "$samples" "$summary" && copy_file "$summary" "$IT_MEMORY_SUMMARY" + fi +} + archive_test_artifacts() { archive_report_trees capture_test_manifest archive_run_config_files archive_engine_log_trees + archive_memory_artifacts write_run_context } @@ -493,6 +526,7 @@ finalize_output() { main() { it_run_entry_init "$SCRIPT_DIR" parse_args "$@" + load_default_operator_note require_target_dir reject_duplicate_archive diff --git a/itests/lib/it-run-context.sh b/itests/lib/it-run-context.sh index f09f73782..c7bbe52d5 100644 --- a/itests/lib/it-run-context.sh +++ b/itests/lib/it-run-context.sh @@ -20,6 +20,7 @@ # Expects archive hooks: target_path, staging_path, log_staged_file, mark_included, # and globals: SCRIPT_DIR, REPO_ROOT, TARGET_DIR, MANIFEST, OUTPUT, RUN_MESSAGE, # RUN_ID, FULL_KARAF, CREATE_TAR, RUN_CONTEXT_REL, RUN_TRACE_FILE. +# Requires it-run-memory.sh to be sourced by the caller (archive-it-run.sh). context_append_inferred_maven_properties() { local engine @@ -72,6 +73,7 @@ context_append_system_snapshot() { if [ "$(uname -s 2>/dev/null)" = Darwin ] && command -v memory_pressure >/dev/null 2>&1; then echo "snapshot.memory_pressure=$(memory_pressure 2>/dev/null | head -1)" fi + it_memory_append_system_snapshot } context_emit_manifest_git_lines() { @@ -151,6 +153,10 @@ write_run_context() { echo echo "## System snapshot (at archive time — not at IT start)" context_append_system_snapshot + echo + it_memory_append_context_section \ + "$(target_path "$IT_MEMORY_SUMMARY")" \ + "$(target_path "$IT_MEMORY_SAMPLES")" } > "$out" log_staged_file "$RUN_CONTEXT_REL" "Wrote $RUN_CONTEXT_REL" mark_included diff --git a/itests/lib/it-run-memory.sh b/itests/lib/it-run-memory.sh new file mode 100644 index 000000000..1c8f43466 --- /dev/null +++ b/itests/lib/it-run-memory.sh @@ -0,0 +1,651 @@ +################################################################################ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# JVM and system memory sampling helpers for integration test runs. +# Requires lib/it-run.sh to be sourced first (shared engine + POM helpers). + +IT_MEMORY_SAMPLES="memory-samples.tsv" +IT_MEMORY_SUMMARY="memory-summary.txt" +IT_MEMORY_SAMPLER_PID="memory-sampler.pid" +IT_MEMORY_SAMPLER_STOP="memory-sampler.stop" +IT_MEMORY_SAMPLER_LOG="memory-sampler.log" +IT_MEMORY_SAMPLER_CACHE="memory-sampler.cache" +IT_MEMORY_SWAP_PRESSURE_MB=2048 + +IT_MEMORY_TSV_HEADER=$'timestamp_utc\tkaraf_pid\tkaraf_heap_used_mb\tkaraf_heap_max_mb\tkaraf_gct_s\tes_heap_used_mb\tes_heap_max_mb\tdocker_rss_mb\tsystem_mem_available_mb\tsystem_swap_used_mb\tsystem_load_1m' + +_IT_MEMORY_OS="" + +it_memory_os_name() { + if [ -z "$_IT_MEMORY_OS" ]; then + _IT_MEMORY_OS="$(uname -s 2>/dev/null || echo unknown)" + fi + echo "$_IT_MEMORY_OS" +} + +it_memory_is_linux() { + [ "$(it_memory_os_name)" = Linux ] +} + +it_memory_is_darwin() { + [ "$(it_memory_os_name)" = Darwin ] +} + +it_memory_parse_heap_setting_mb() { + local value="${1:-}" + local number suffix + + value="$(echo "$value" | tr -d '[:space:]')" + if [ -z "$value" ]; then + echo "0" + return + fi + + suffix="${value: -1}" + number="${value%?}" + case "$suffix" in + g | G) awk -v n="$number" 'BEGIN { printf "%d", int(n * 1024 + 0.5) }' ;; + m | M) awk -v n="$number" 'BEGIN { printf "%d", int(n + 0.5) }' ;; + [0-9]) awk -v n="$value" 'BEGIN { printf "%d", int(n / 1048576 + 0.5) }' ;; + *) echo "0" ;; + esac +} + +it_memory_parse_jvm_flag_bytes() { + local output="$1" + local flag="$2" + + echo "$output" | sed -n "s/.*${flag}[[:space:]]*=[[:space:]]*\([0-9][0-9]*\).*/\1/p" | head -1 +} + +it_memory_read_trace_heap_mb() { + local target_dir="$1" + local trace_field="$2" + local pom_profile="$3" + local pom_field="$4" + local script_dir trace_file configured + + script_dir="$(cd "$(dirname "$target_dir")" && pwd)" + trace_file="$target_dir/it-run-trace.properties" + if [ ! -f "$script_dir/pom.xml" ]; then + echo "0" + return + fi + configured="$(it_resolve_configured_heap "$trace_file" "$trace_field" "$script_dir/pom.xml" "$pom_profile" "$pom_field")" + it_memory_parse_heap_setting_mb "$configured" +} + +it_memory_detect_karaf_max_mb() { + local target_dir="$1" + local pid="$2" + local cap_mb_from_jstat="${3:-0}" + local max_bytes max_mb jinfo_out jcmd_out profile + + max_mb="0" + + if command -v jinfo >/dev/null 2>&1; then + jinfo_out="$(jinfo -flag MaxHeapSize "$pid" 2>/dev/null || true)" + max_bytes="$(it_memory_parse_jvm_flag_bytes "$jinfo_out" "MaxHeapSize")" + if [ -n "$max_bytes" ]; then + max_mb="$(it_memory_mb_from_bytes "$max_bytes")" + fi + fi + + if [ "$max_mb" = "0" ] && command -v jcmd >/dev/null 2>&1; then + jcmd_out="$(jcmd "$pid" VM.flags 2>/dev/null || true)" + max_bytes="$(it_memory_parse_jvm_flag_bytes "$jcmd_out" "MaxHeapSize")" + if [ -n "$max_bytes" ]; then + max_mb="$(it_memory_mb_from_bytes "$max_bytes")" + fi + fi + + if [ "$max_mb" = "0" ] && [ "$cap_mb_from_jstat" -gt 0 ] 2>/dev/null; then + max_mb="$cap_mb_from_jstat" + fi + + if [ "$max_mb" = "0" ]; then + profile="$(it_search_engine_pom_profile "$(it_memory_resolve_search_engine "$target_dir")")" + max_mb="$(it_memory_read_trace_heap_mb "$target_dir" "karaf.heap" "$profile" "karaf.heap")" + fi + + echo "$max_mb" +} + +it_memory_jstat_gc_metrics() { + local pid="$1" + + if ! command -v jstat >/dev/null 2>&1; then + echo -e "0\t0\t0" + return + fi + + jstat -gc "$pid" 2>/dev/null | awk ' + NR == 1 { + for (i = 1; i <= NF; i++) { + h[$i] = i + } + next + } + NR == 2 { + if ("S0U" in h) { + used_kb = $(h["S0U"]) + $(h["S1U"]) + $(h["EU"]) + $(h["OU"]) + } else { + used_kb = $3 + $4 + $6 + $8 + } + if ("S0C" in h) { + cap_kb = $(h["S0C"]) + $(h["S1C"]) + $(h["EC"]) + $(h["OC"]) + } else { + cap_kb = $1 + $2 + $5 + $7 + } + if ("GCT" in h) { + gct = $(h["GCT"]) + 0 + } else { + gct = $NF + 0 + } + printf "%d\t%d\t%.3f", int(used_kb * 1024 / 1048576 + 0.5), int(cap_kb * 1024 / 1048576 + 0.5), gct + }' +} + +it_memory_system_mem_available_mb() { + local pagesize + + if it_memory_is_linux && command -v free >/dev/null 2>&1; then + free -m 2>/dev/null | awk '/^Mem:/ { + if ($7 != "") { + print $7 + } else { + print $4 + } + exit + }' + return + fi + + if it_memory_is_darwin && command -v vm_stat >/dev/null 2>&1; then + pagesize="$(sysctl -n hw.pagesize 2>/dev/null || echo 4096)" + vm_stat 2>/dev/null | awk -v ps="$pagesize" ' + /Pages free/ { gsub(/\./, "", $3); free = $3 + 0 } + /Pages inactive/ { gsub(/\./, "", $3); inactive = $3 + 0 } + END { + printf "%d", int((free + inactive) * ps / 1048576 + 0.5) + }' + return + fi + + echo "0" +} + +it_memory_system_swap_used_mb() { + if it_memory_is_linux && command -v free >/dev/null 2>&1; then + free -m 2>/dev/null | awk '/^Swap:/ { print $3; exit }' + return + fi + + if it_memory_is_darwin; then + sysctl -n vm.swapusage 2>/dev/null | sed -n 's/.* used = \([0-9.]*\)M.*/\1/p' | awk '{ printf "%d", int($1 + 0.5) }' + return + fi + + echo "0" +} + +it_memory_system_load_1m() { + if it_memory_is_linux && [ -r /proc/loadavg ]; then + awk '{ print $1 + 0 }' /proc/loadavg + return + fi + + if it_memory_is_darwin && command -v sysctl >/dev/null 2>&1; then + sysctl -n vm.loadavg 2>/dev/null | tr -d '{}\n' | awk '{ print $1 + 0 }' + return + fi + + if command -v uptime >/dev/null 2>&1; then + uptime 2>/dev/null | sed -n 's/.*load average[s]*: //p' | cut -d, -f1 | tr -d ' ' + return + fi + + echo "0" +} + +it_memory_sampler_cache_file() { + echo "$1/$IT_MEMORY_SAMPLER_CACHE" +} + +it_memory_read_cache_field() { + local cache_file="$1" + local field="$2" + if [ ! -f "$cache_file" ]; then + return 1 + fi + grep -m1 "^${field}=" "$cache_file" 2>/dev/null | cut -d= -f2- +} + +it_memory_write_cache_field() { + local cache_file="$1" + local field="$2" + local value="$3" + local tmp + + mkdir -p "$(dirname "$cache_file")" + touch "$cache_file" + tmp="$(mktemp "${TMPDIR:-/tmp}/unomi-mem-cache.XXXXXX")" + grep -v "^${field}=" "$cache_file" > "$tmp" 2>/dev/null || true + printf '%s=%s\n' "$field" "$value" >> "$tmp" + mv "$tmp" "$cache_file" +} + +it_memory_clear_sampler_cache() { + rm -f "$(it_memory_sampler_cache_file "$1")" +} + +it_memory_resolve_search_engine() { + local target_dir="$1" + local engine + + if [ -n "${IT_SEARCH_ENGINE:-}" ]; then + echo "$IT_SEARCH_ENGINE" + return + fi + engine="$(it_infer_search_engine "$target_dir")" + if [ -n "$engine" ] && [ "$engine" != "unknown" ]; then + echo "$engine" + return + fi + echo "elasticsearch" +} + +it_memory_resolve_docker_container() { + local target_dir="$1" + local cache_file engine container default_container + + cache_file="$(it_memory_sampler_cache_file "$target_dir")" + container="$(it_memory_read_cache_field "$cache_file" docker_container || true)" + if [ -n "$container" ]; then + echo "$container" + return + fi + + engine="$(it_memory_resolve_search_engine "$target_dir")" + default_container="$(it_search_engine_docker_container "$engine")" + + if command -v docker >/dev/null 2>&1; then + container="$(docker ps --filter "name=${default_container}" --format '{{.Names}}' 2>/dev/null | head -1)" + fi + + if [ -z "$container" ]; then + echo "$default_container" + return + fi + + it_memory_write_cache_field "$cache_file" docker_container "$container" + echo "$container" +} + +it_memory_mb_from_bytes() { + local bytes="$1" + if [ -z "$bytes" ] || [ "$bytes" = "0" ]; then + echo "0" + return + fi + echo $(( (bytes + 524288) / 1048576 )) +} + +it_memory_json_number() { + local json="$1" + local field="$2" + echo "$json" | sed -n "s/.*\"${field}\":\([0-9][0-9]*\).*/\1/p" | head -1 +} + +it_memory_resolve_search_port() { + local target_dir="$1" + local engine port_file port_line + + engine="$(it_memory_resolve_search_engine "$target_dir")" + port_file="$target_dir/$(it_search_engine_port_properties_file "$engine")" + + if [ -f "$port_file" ]; then + port_line="$(grep -m1 '\.port=' "$port_file" 2>/dev/null | cut -d= -f2- | tr -d '[:space:]')" + if [ -n "$port_line" ]; then + echo "$port_line" + return + fi + fi + + if [ -n "${IT_SEARCH_PORT:-}" ]; then + echo "$IT_SEARCH_PORT" + return + fi + + it_search_engine_default_port "$engine" +} + +it_memory_parse_docker_mem_to_mb() { + local raw="${1:-}" + + case "$raw" in + *GiB) echo "${raw%GiB}" | awk '{printf "%d", $1 * 1024 + 0.5}' ;; + *MiB) echo "${raw%MiB}" | awk '{printf "%d", $1 + 0.5}' ;; + *KiB) echo "${raw%KiB}" | awk '{printf "%d", int($1 / 1024 + 0.5)}' ;; + *G) echo "${raw%G}" | awk '{printf "%d", $1 * 1024 + 0.5}' ;; + *M) echo "${raw%M}" | awk '{printf "%d", $1 + 0.5}' ;; + *K) echo "${raw%K}" | awk '{printf "%d", int($1 / 1024 + 0.5)}' ;; + *) echo "0" ;; + esac +} + +it_memory_find_karaf_pid() { + pgrep -f 'org.apache.karaf.main.Main' 2>/dev/null | head -1 +} + +it_memory_karaf_max_mb_cached() { + local target_dir="$1" + local pid="$2" + local cap_mb_from_jstat="${3:-0}" + local cache_file cached_pid max_mb + + cache_file="$(it_memory_sampler_cache_file "$target_dir")" + cached_pid="$(it_memory_read_cache_field "$cache_file" karaf_pid || true)" + if [ "$cached_pid" = "$pid" ]; then + max_mb="$(it_memory_read_cache_field "$cache_file" karaf_max_mb || true)" + if [ -n "$max_mb" ] && [ "$max_mb" -gt 0 ] 2>/dev/null; then + echo "$max_mb" + return + fi + fi + + max_mb="$(it_memory_detect_karaf_max_mb "$target_dir" "$pid" "$cap_mb_from_jstat")" + if [ -z "$max_mb" ]; then + max_mb="0" + fi + + it_memory_write_cache_field "$cache_file" karaf_pid "$pid" + if [ "$max_mb" -gt 0 ] 2>/dev/null; then + it_memory_write_cache_field "$cache_file" karaf_max_mb "$max_mb" + else + it_memory_write_cache_field "$cache_file" karaf_max_mb "" + fi + echo "$max_mb" +} + +it_memory_karaf_stats() { + local target_dir="$1" + local pid="$2" + local used_mb max_mb cap_mb gct_s jstat_line + + if [ -z "$pid" ]; then + echo -e "0\t0\t0" + return + fi + + if ! kill -0 "$pid" 2>/dev/null; then + echo -e "0\t0\t0" + return + fi + + used_mb="0" + cap_mb="0" + gct_s="0" + jstat_line="$(it_memory_jstat_gc_metrics "$pid")" + if [ -n "$jstat_line" ]; then + read -r used_mb cap_mb gct_s </dev/null 2>&1; then + used_mb="$(ps -o rss= -p "$pid" 2>/dev/null | awk '{print int($1/1024+0.5)}')" + fi + fi + + if [ -z "${max_mb:-}" ] || [ "$max_mb" = "0" ]; then + max_mb="0" + fi + if [ -z "${used_mb:-}" ]; then + used_mb="0" + fi + if [ -z "${gct_s:-}" ]; then + gct_s="0" + fi + + echo -e "${used_mb}\t${max_mb}\t${gct_s}" +} + +it_memory_search_engine_stats() { + local port="$1" + local json used_bytes max_bytes + + json="$(curl -sf --max-time 5 \ + "http://localhost:${port}/_nodes/stats/jvm?filter_path=nodes.*.jvm.mem.heap_used_in_bytes,nodes.*.jvm.mem.heap_max_in_bytes" \ + 2>/dev/null || true)" + if [ -z "$json" ]; then + echo -e "0\t0" + return + fi + + used_bytes="$(it_memory_json_number "$json" "heap_used_in_bytes")" + max_bytes="$(it_memory_json_number "$json" "heap_max_in_bytes")" + echo -e "$(it_memory_mb_from_bytes "${used_bytes:-0}")\t$(it_memory_mb_from_bytes "${max_bytes:-0}")" +} + +it_memory_docker_rss_mb() { + local target_dir="$1" + local container rss + + if ! command -v docker >/dev/null 2>&1; then + echo "0" + return + fi + + container="$(it_memory_resolve_docker_container "$target_dir")" + rss="$(docker stats --no-stream --format '{{.MemUsage}}' "$container" 2>/dev/null | head -1 | cut -d/ -f1 | tr -d ' ')" + + it_memory_parse_docker_mem_to_mb "$rss" +} + +it_memory_system_stats() { + local mem_available swap_used load_1m + + mem_available="$(it_memory_system_mem_available_mb)" + swap_used="$(it_memory_system_swap_used_mb)" + load_1m="$(it_memory_system_load_1m)" + + echo -e "${mem_available:-0}\t${swap_used:-0}\t${load_1m:-0}" +} + +it_memory_sample_once() { + local target_dir="$1" + local port="${2:-}" + local karaf_pid karaf_line es_line sys_line + + port="${port:-$(it_memory_resolve_search_port "$target_dir")}" + karaf_pid="$(it_memory_find_karaf_pid)" + karaf_line="$(it_memory_karaf_stats "$target_dir" "$karaf_pid")" + es_line="$(it_memory_search_engine_stats "$port")" + sys_line="$(it_memory_system_stats)" + + printf '%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + "${karaf_pid:-0}" \ + "$karaf_line" \ + "$es_line" \ + "$(it_memory_docker_rss_mb "$target_dir")" \ + "$sys_line" +} + +it_memory_write_samples_header() { + local samples_file="$1" + if [ ! -f "$samples_file" ] || [ ! -s "$samples_file" ]; then + printf '%s\n' "$IT_MEMORY_TSV_HEADER" > "$samples_file" + fi +} + +it_memory_summarize_samples() { + local samples_file="$1" + local summary_file="$2" + + if [ ! -f "$samples_file" ] || [ ! -s "$samples_file" ]; then + return 1 + fi + + awk -F'\t' -v summary="$summary_file" -v swap_pressure_mb="$IT_MEMORY_SWAP_PRESSURE_MB" ' + NR == 1 { next } + NF < 11 { next } + { + samples++ + if ($2+0 > 0) { + if ($3+0 > peak_karaf_used) peak_karaf_used = $3+0 + if ($4+0 > peak_karaf_max) peak_karaf_max = $4+0 + if ($5+0 > peak_karaf_gct_s) peak_karaf_gct_s = $5+0 + } + if ($6+0 > peak_es_used) peak_es_used = $6+0 + if ($7+0 > peak_es_max) peak_es_max = $7+0 + if ($8+0 > peak_docker_rss) peak_docker_rss = $8+0 + if ($9+0 > 0 && (min_mem_avail == 0 || $9+0 < min_mem_avail)) min_mem_avail = $9+0 + if ($10+0 > peak_swap) peak_swap = $10+0 + if ($11+0 > peak_load) peak_load = $11+0 + if (samples == 1) first_swap = $10+0 + last_swap = $10+0 + } + END { + if (samples == 0) exit 1 + karaf_headroom = (peak_karaf_max > 0 ? (peak_karaf_max - peak_karaf_used) * 100 / peak_karaf_max : 0) + es_headroom = (peak_es_max > 0 ? (peak_es_max - peak_es_used) * 100 / peak_es_max : 0) + swap_pressure = (peak_swap+0 > 0 && min_mem_avail+0 > 0 && min_mem_avail+0 < swap_pressure_mb+0) || (last_swap+0 > first_swap+0 + 256) + printf("# Integration test memory summary (generated from %s)\n", FILENAME) > summary + printf("memory.samples.count=%d\n", samples) >> summary + printf("memory.peak.karaf.heap.used.mb=%d\n", peak_karaf_used+0) >> summary + printf("memory.peak.karaf.heap.max.mb=%d\n", peak_karaf_max+0) >> summary + printf("memory.peak.karaf.gct.s=%.2f\n", peak_karaf_gct_s+0) >> summary + printf("memory.peak.search.heap.used.mb=%d\n", peak_es_used+0) >> summary + printf("memory.peak.search.heap.max.mb=%d\n", peak_es_max+0) >> summary + printf("memory.peak.docker.rss.mb=%d\n", peak_docker_rss+0) >> summary + printf("memory.min.system.mem.available.mb=%d\n", min_mem_avail+0) >> summary + printf("memory.peak.system.swap.used.mb=%d\n", peak_swap+0) >> summary + printf("memory.peak.system.load.1m=%.2f\n", peak_load+0) >> summary + printf("memory.karaf.headroom.pct=%d\n", karaf_headroom+0) >> summary + printf("memory.search.headroom.pct=%d\n", es_headroom+0) >> summary + if (swap_pressure) { + printf("memory.warning.swap.detected=true\n") >> summary + } else { + printf("memory.warning.swap.detected=false\n") >> summary + } + if (peak_karaf_max > 0 && peak_karaf_used * 100 / peak_karaf_max >= 85) { + printf("memory.warning.karaf.heap.high=true\n") >> summary + } else { + printf("memory.warning.karaf.heap.high=false\n") >> summary + } + if (peak_es_max > 0 && peak_es_used * 100 / peak_es_max >= 85) { + printf("memory.warning.search.heap.high=true\n") >> summary + } else { + printf("memory.warning.search.heap.high=false\n") >> summary + } + } + ' "$samples_file" +} + +it_memory_append_context_section() { + local summary_file="$1" + local samples_file="$2" + + if [ -f "$summary_file" ]; then + echo "## Memory summary (observed during IT run)" + cat "$summary_file" + echo + elif [ -f "$samples_file" ]; then + echo "## Memory samples" + echo "# memory-summary.txt missing; raw samples available at $(basename "$samples_file")" + echo "memory.samples.lines=$(($(wc -l < "$samples_file" 2>/dev/null | tr -d ' ') - 1))" + echo + fi +} + +it_memory_append_system_snapshot() { + if it_memory_is_linux && command -v free >/dev/null 2>&1; then + echo "snapshot.mem.linux=$(free -h 2>/dev/null | awk '/^Mem:/ {print $0}')" + echo "snapshot.swap.linux=$(free -h 2>/dev/null | awk '/^Swap:/ {print $0}')" + fi + echo "snapshot.mem.available.mb=$(it_memory_system_mem_available_mb)" + echo "snapshot.swap.used.mb=$(it_memory_system_swap_used_mb)" + echo "snapshot.load.1m=$(it_memory_system_load_1m)" +} + +it_memory_verify() { + local target_dir="$1" + local script_dir="${2:-$(cd "$(dirname "$target_dir")" && pwd)}" + local pom="$script_dir/pom.xml" + local failures=0 karaf_pid sample_line + + _it_memory_verify_eq() { + if [ "$2" = "$3" ]; then + echo "OK $1" + else + echo "FAIL $1 (expected '$2', got '$3')" + failures=$((failures + 1)) + fi + } + + _it_memory_verify_gt_zero() { + if awk -v v="${2:-0}" 'BEGIN { exit !(v + 0 > 0) }'; then + echo "OK $1 ($2)" + else + echo "FAIL $1 (expected > 0, got '${2:-0}')" + failures=$((failures + 1)) + fi + } + + echo "Memory sampling verification on $(it_memory_os_name)" + _it_memory_verify_eq "parse 2g" "2048" "$(it_memory_parse_heap_setting_mb 2g)" + _it_memory_verify_eq "parse 1536m" "1536" "$(it_memory_parse_heap_setting_mb 1536m)" + _it_memory_verify_eq "parse docker GiB" "2048" "$(it_memory_parse_docker_mem_to_mb 2GiB)" + _it_memory_verify_eq "parse docker MiB" "512" "$(it_memory_parse_docker_mem_to_mb 512MiB)" + _it_memory_verify_eq "parse docker KiB" "1" "$(it_memory_parse_docker_mem_to_mb 1024KiB)" + _it_memory_verify_eq "parse jinfo flag" "2147483648" \ + "$(it_memory_parse_jvm_flag_bytes '-XX:MaxHeapSize=2147483648' 'MaxHeapSize')" + _it_memory_verify_eq "search engine default port" "9400" "$(it_search_engine_default_port elasticsearch)" + _it_memory_verify_eq "search engine container" "itests-elasticsearch" "$(it_search_engine_docker_container elasticsearch)" + _it_memory_verify_eq "pom elasticsearch.heap" "4g" \ + "$(it_read_pom_profile_property "$pom" elasticsearch elasticsearch.heap 2>/dev/null || true)" + _it_memory_verify_eq "pom karaf.heap" "2g" \ + "$(it_read_pom_profile_property "$pom" elasticsearch karaf.heap 2>/dev/null || true)" + _it_memory_verify_gt_zero "system load 1m" "$(it_memory_system_load_1m)" + _it_memory_verify_gt_zero "system mem available mb" "$(it_memory_system_mem_available_mb)" + echo "OK system swap used mb ($(it_memory_system_swap_used_mb))" + + karaf_pid="$(it_memory_find_karaf_pid || true)" + if [ -n "$karaf_pid" ]; then + echo "Karaf PID $karaf_pid — live checks" + sample_line="$(it_memory_sample_once "$target_dir")" + _it_memory_verify_gt_zero "sample karaf_heap_max_mb" "$(echo "$sample_line" | awk -F'\t' '{print $4}')" + _it_memory_verify_gt_zero "sample system_load_1m parseable" \ + "$(echo "$sample_line" | awk -F'\t' '{print ($11+0 > 0)}')" + echo "Sample: $sample_line" + else + echo "SKIP live Karaf checks (no Karaf process)" + fi + + if [ "$failures" -eq 0 ]; then + echo "All checks passed." + return 0 + fi + echo "$failures check(s) failed." + return 1 +} diff --git a/itests/lib/it-run.sh b/itests/lib/it-run.sh index ed50a8136..dbe9b83c4 100644 --- a/itests/lib/it-run.sh +++ b/itests/lib/it-run.sh @@ -37,7 +37,9 @@ IT_ENGINE_PORT_FILES=( elasticsearch-port.properties opensearch-port.properties ) -IT_RUN_CONTEXT_NO_OPERATOR_NOTE='(none — pass --message "..." to describe run conditions, e.g. heavy swap, CI runner, single-test rerun)' +IT_RUN_CONTEXT_NO_OPERATOR_NOTE='(none — auto-generated note missing; run via ./build.sh --integration-tests or pass -m "...")' +IT_RUN_TRACE="it-run-trace.properties" +IT_OPERATOR_NOTE_FILE="it-run-operator-note.txt" it_run_lib_init() { IT_SCRIPT_DIR="$1" @@ -77,16 +79,35 @@ it_read_pom_profile_property() { awk -v profile="$profile" -v prop="$property" ' $0 ~ "" profile "" { in_profile=1 } - in_profile && $0 ~ "<" prop ">" { - gsub(/.*<[^>]+>/, "") - gsub(/<.*/, "") - print + in_profile && index($0, "<" prop ">") { + start = index($0, "<" prop ">") + length(prop) + 2 + rest = substr($0, start) + end = index(rest, "<") + if (end > 0) { + print substr(rest, 1, end - 1) + } exit } in_profile && $0 ~ "" { exit } ' "$pom" } +it_resolve_configured_heap() { + local trace_file="$1" + local trace_field="$2" + local pom="$3" + local profile="$4" + local pom_field="$5" + local configured + + configured="$(it_read_properties_field "$trace_file" "$trace_field")" + if [ -n "$configured" ]; then + echo "$configured" + return + fi + it_read_pom_profile_property "$pom" "$profile" "$pom_field" +} + it_infer_search_engine() { local target_dir="$1" @@ -103,6 +124,41 @@ it_infer_search_engine() { fi } +it_search_engine_pom_profile() { + case "$1" in + opensearch) echo "opensearch" ;; + *) echo "elasticsearch" ;; + esac +} + +it_search_engine_heap_pom_field() { + case "$1" in + opensearch) echo "opensearch.heap" ;; + *) echo "elasticsearch.heap" ;; + esac +} + +it_search_engine_docker_container() { + case "$1" in + opensearch) echo "itests-opensearch" ;; + *) echo "itests-elasticsearch" ;; + esac +} + +it_search_engine_port_properties_file() { + case "$1" in + opensearch) echo "opensearch-port.properties" ;; + *) echo "elasticsearch-port.properties" ;; + esac +} + +it_search_engine_default_port() { + case "$1" in + opensearch) echo "9401" ;; + *) echo "9400" ;; + esac +} + it_run_label() { basename "$1" } @@ -469,3 +525,137 @@ it_update_runs_index() { } > "$index" ui_detail "Updated $index" } + +it_format_configured_heap() { + local trace_file="$1" + local trace_field="$2" + local pom="$3" + local profile="$4" + local pom_field="$5" + local trace_value resolved + + trace_value="$(it_read_properties_field "$trace_file" "$trace_field")" + if [ -n "$trace_value" ]; then + echo "$trace_value" + return + fi + resolved="$(it_resolve_configured_heap "$trace_file" "$trace_field" "$pom" "$profile" "$pom_field")" + if [ -n "$resolved" ]; then + echo "${resolved} (pom default)" + else + echo "unknown" + fi +} + +it_generate_operator_note() { + local target_dir="$1" + local script_dir="$2" + local trace_file="$target_dir/$IT_RUN_TRACE" + local memory_summary="$target_dir/${IT_MEMORY_SUMMARY:-memory-summary.txt}" + local failsafe_summary="$target_dir/failsafe-reports/failsafe-summary.xml" + local engine profile search_heap karaf_heap exit_code single_test + local failures errors skipped completed started completed_ts host ci + local swap_warn search_high karaf_high + + if [ ! -f "$trace_file" ]; then + return 1 + fi + + engine="$(it_read_properties_field "$trace_file" search.engine)" + if [ -z "$engine" ]; then + engine="$(it_infer_search_engine "$target_dir")" + fi + profile="$(it_search_engine_pom_profile "$engine")" + search_heap="$(it_format_configured_heap "$trace_file" search.heap "$script_dir/pom.xml" "$profile" "$(it_search_engine_heap_pom_field "$engine")")" + karaf_heap="$(it_format_configured_heap "$trace_file" karaf.heap "$script_dir/pom.xml" "$profile" karaf.heap)" + exit_code="$(it_read_properties_field "$trace_file" maven.exit.code)" + single_test="$(it_read_properties_field "$trace_file" single.test)" + started="$(it_read_properties_field "$trace_file" trace.started)" + completed_ts="$(it_read_properties_field "$trace_file" trace.completed)" + host="$(it_read_properties_field "$trace_file" host)" + if [ -z "$host" ]; then + host="$(it_hostname)" + fi + if [ -n "${GITHUB_ACTIONS:-}" ]; then + ci="yes (${GITHUB_WORKFLOW:-workflow}/${GITHUB_JOB:-job})" + else + ci="no" + fi + + failures="$(it_failsafe_summary_count "$failsafe_summary" failures)" + errors="$(it_failsafe_summary_count "$failsafe_summary" errors)" + skipped="$(it_failsafe_summary_count "$failsafe_summary" skipped)" + completed="$(it_failsafe_summary_count "$failsafe_summary" completed)" + + swap_warn="$(it_read_properties_field "$memory_summary" memory.warning.swap.detected)" + search_high="$(it_read_properties_field "$memory_summary" memory.warning.search.heap.high)" + karaf_high="$(it_read_properties_field "$memory_summary" memory.warning.karaf.heap.high)" + + echo "Auto-generated operator note ($(it_utc_now))" + if [ "$exit_code" = "0" ]; then + echo "Outcome: PASS (maven exit 0)" + elif [ -n "$exit_code" ]; then + echo "Outcome: FAIL (maven exit ${exit_code})" + else + echo "Outcome: unknown (run still in progress or trace incomplete)" + fi + echo "Search engine: ${engine}" + echo "Heaps: search=${search_heap}, karaf=${karaf_heap}" + echo "Host: ${host}; CI: ${ci}" + if [ -n "$started" ] && [ -n "$completed_ts" ]; then + echo "Run window: ${started} → ${completed_ts}" + elif [ -n "$started" ]; then + echo "Run started: ${started}" + fi + if [ -n "$single_test" ]; then + echo "Scope: single test ${single_test}" + else + echo "Scope: full AllITs suite" + fi + if [ -n "$completed" ]; then + echo "Tests: completed=${completed:-0} failures=${failures:-0} errors=${errors:-0} skipped=${skipped:-0}" + fi + if [ -f "$memory_summary" ]; then + echo "Memory peaks: karaf=$(it_read_properties_field "$memory_summary" memory.peak.karaf.heap.used.mb)/$(it_read_properties_field "$memory_summary" memory.peak.karaf.heap.max.mb) MB, karaf GCT=$(it_read_properties_field "$memory_summary" memory.peak.karaf.gct.s)s, search=$(it_read_properties_field "$memory_summary" memory.peak.search.heap.used.mb)/$(it_read_properties_field "$memory_summary" memory.peak.search.heap.max.mb) MB, swap=$(it_read_properties_field "$memory_summary" memory.peak.system.swap.used.mb) MB" + if [ "$swap_warn" = "true" ]; then + echo "Memory alert: swap detected during run — consider lowering JVM heaps or using a larger runner" + fi + if [ "$search_high" = "true" ]; then + echo "Memory alert: search engine heap usage high (>=85% of max) — may be GC-bound" + fi + if [ "$karaf_high" = "true" ]; then + echo "Memory alert: Karaf heap usage high (>=85% of max) — may be GC-bound" + fi + else + echo "Memory peaks: (no memory-summary.txt — sampler disabled or run archived before finalize)" + fi +} + +it_write_operator_note_file() { + local target_dir="$1" + local script_dir="$2" + local out="$target_dir/$IT_OPERATOR_NOTE_FILE" + + mkdir -p "$target_dir" + if ! it_generate_operator_note "$target_dir" "$script_dir" > "$out"; then + rm -f "$out" + return 1 + fi +} + +it_load_default_operator_note() { + local target_dir="$1" + local script_dir="$2" + local note_file="$target_dir/$IT_OPERATOR_NOTE_FILE" + + if [ -f "$note_file" ]; then + cat "$note_file" + return 0 + fi + if [ -f "$target_dir/$IT_RUN_TRACE" ]; then + it_write_operator_note_file "$target_dir" "$script_dir" || return 1 + cat "$note_file" + return 0 + fi + return 1 +} diff --git a/itests/sample-it-memory.sh b/itests/sample-it-memory.sh new file mode 100755 index 000000000..19e0d5d5c --- /dev/null +++ b/itests/sample-it-memory.sh @@ -0,0 +1,229 @@ +#!/bin/bash +################################################################################ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# +# Sample JVM and system memory while integration tests run. +# +# Usage: +# ./itests/sample-it-memory.sh start [--target-dir itests/target] [--interval 30] +# ./itests/sample-it-memory.sh stop [--target-dir itests/target] +# ./itests/sample-it-memory.sh sample-once [--target-dir itests/target] +# ./itests/sample-it-memory.sh summarize [--target-dir itests/target] +# ./itests/sample-it-memory.sh operator-note [--target-dir itests/target] [--print-only] +# ./itests/sample-it-memory.sh verify [--target-dir itests/target] +# Started automatically by ./build.sh --integration-tests (disable with --no-memory-sampler). +# + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=lib/it-run.sh disable=SC1091 +source "$SCRIPT_DIR/lib/it-run.sh" +# shellcheck source=lib/it-run-memory.sh disable=SC1091 +source "$SCRIPT_DIR/lib/it-run-memory.sh" + +TARGET_DIR="$SCRIPT_DIR/target" +INTERVAL=30 +SEARCH_PORT="" +PRINT_ONLY=false +COMMAND="" + +usage() { + cat < [options] + +Commands: + start Run the sampler in the background until "stop" is called + stop Stop the background sampler and write memory-summary.txt + sample-once Print one TSV sample line to stdout + summarize Build memory-summary.txt from memory-samples.tsv + operator-note Write it-run-operator-note.txt from trace + test + memory data + verify Cross-platform sanity checks (macOS + Linux) + +Options: + --target-dir DIR IT target directory (default: itests/target) + --interval SEC Sample interval in seconds for start (default: 30) + --port PORT Search engine HTTP port override + --print-only With operator-note: print to stdout instead of writing file + -h, --help Show this help +EOF + exit 1 +} + +parse_args() { + COMMAND="${1:-}" + shift || true + + while [ $# -gt 0 ]; do + case "$1" in + --target-dir) + shift + TARGET_DIR="${1:-}" + ;; + --interval) + shift + INTERVAL="${1:-30}" + ;; + --port) + shift + SEARCH_PORT="${1:-}" + ;; + --print-only) + PRINT_ONLY=true + ;; + -h | --help) + usage + ;; + *) + echo "Unknown option: $1" >&2 + usage + ;; + esac + shift + done + + if [ -z "$COMMAND" ]; then + usage + fi + + mkdir -p "$TARGET_DIR" + if [ -n "$SEARCH_PORT" ]; then + IT_SEARCH_PORT="$SEARCH_PORT" + export IT_SEARCH_PORT + fi +} + +sampler_paths() { + SAMPLES_FILE="$TARGET_DIR/$IT_MEMORY_SAMPLES" + SUMMARY_FILE="$TARGET_DIR/$IT_MEMORY_SUMMARY" + PID_FILE="$TARGET_DIR/$IT_MEMORY_SAMPLER_PID" + STOP_FILE="$TARGET_DIR/$IT_MEMORY_SAMPLER_STOP" + LOG_FILE="$TARGET_DIR/$IT_MEMORY_SAMPLER_LOG" +} + +sampler_loop() { + local interval="$1" + sampler_paths + it_memory_write_samples_header "$SAMPLES_FILE" + + while [ ! -f "$STOP_FILE" ]; do + it_memory_sample_once "$TARGET_DIR" >> "$SAMPLES_FILE" || true + sleep "$interval" + done +} + +cmd_start() { + sampler_paths + + if [ -f "$PID_FILE" ]; then + existing_pid="$(cat "$PID_FILE" 2>/dev/null || true)" + if [ -n "$existing_pid" ] && kill -0 "$existing_pid" 2>/dev/null; then + echo "Memory sampler already running (pid $existing_pid)" >&2 + exit 0 + fi + rm -f "$PID_FILE" + fi + + rm -f "$STOP_FILE" + it_memory_clear_sampler_cache "$TARGET_DIR" + it_memory_write_samples_header "$SAMPLES_FILE" + echo "# started $(date -u +%Y-%m-%dT%H:%M:%SZ) interval=${INTERVAL}s target=$TARGET_DIR" >> "$LOG_FILE" + + ( + sampler_loop "$INTERVAL" + ) >> "$LOG_FILE" 2>&1 & + + echo $! > "$PID_FILE" + echo "Memory sampler started (pid $(cat "$PID_FILE"), interval ${INTERVAL}s)" + echo "Samples: $SAMPLES_FILE" +} + +cmd_stop() { + sampler_paths + + if [ ! -f "$PID_FILE" ]; then + echo "Memory sampler is not running" >&2 + cmd_summarize || true + return 0 + fi + + pid="$(cat "$PID_FILE" 2>/dev/null || true)" + touch "$STOP_FILE" + + if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then + kill -TERM "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + fi + + rm -f "$PID_FILE" "$STOP_FILE" + echo "# stopped $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$LOG_FILE" + cmd_summarize || true + echo "Memory sampler stopped" +} + +cmd_sample_once() { + sampler_paths + it_memory_write_samples_header "$SAMPLES_FILE" + it_memory_sample_once "$TARGET_DIR" +} + +cmd_summarize() { + sampler_paths + if it_memory_summarize_samples "$SAMPLES_FILE" "$SUMMARY_FILE"; then + echo "Wrote $SUMMARY_FILE" + return 0 + fi + echo "No memory samples to summarize in $SAMPLES_FILE" >&2 + return 1 +} + +cmd_operator_note() { + if [ "$PRINT_ONLY" = true ]; then + it_generate_operator_note "$TARGET_DIR" "$SCRIPT_DIR" + return 0 + fi + if it_write_operator_note_file "$TARGET_DIR" "$SCRIPT_DIR"; then + echo "Wrote $TARGET_DIR/$IT_OPERATOR_NOTE_FILE" + return 0 + fi + echo "Could not generate operator note (missing $IT_RUN_TRACE in $TARGET_DIR)" >&2 + return 1 +} + +cmd_verify() { + it_memory_verify "$TARGET_DIR" "$SCRIPT_DIR" +} + +main() { + parse_args "$@" + case "$COMMAND" in + start) cmd_start ;; + stop) cmd_stop ;; + sample-once) cmd_sample_once ;; + summarize) cmd_summarize ;; + operator-note) cmd_operator_note ;; + verify) cmd_verify ;; + *) + echo "Unknown command: $COMMAND" >&2 + usage + ;; + esac +} + +main "$@" diff --git a/services/src/test/java/org/apache/unomi/services/impl/cluster/ClusterServiceImplTest.java b/services/src/test/java/org/apache/unomi/services/impl/cluster/ClusterServiceImplTest.java index 435360e48..bd7bc231f 100644 --- a/services/src/test/java/org/apache/unomi/services/impl/cluster/ClusterServiceImplTest.java +++ b/services/src/test/java/org/apache/unomi/services/impl/cluster/ClusterServiceImplTest.java @@ -383,13 +383,14 @@ public void testPurgeByDateDelegatesToPersistenceService() { persistenceService.save(recentNode); }); - // Purge items older than cutoff (between old and recent) - Date cutoff = new Date(System.currentTimeMillis() - 3L * 24 * 3600 * 1000); // 3 days ago - clusterService.purge(cutoff); + // Purge and verify in system tenant — items were saved there; load/purge are tenant-scoped. + executionContextManager.executeAsSystem(() -> { + Date cutoff = new Date(System.currentTimeMillis() - 3L * 24 * 3600 * 1000); // 3 days ago + clusterService.purge(cutoff); - // Verify: old node removed, recent node remains - assertNull(persistenceService.load("old-node", ClusterNode.class)); - assertNotNull(persistenceService.load("recent-node", ClusterNode.class)); + assertNull(persistenceService.load("old-node", ClusterNode.class)); + assertNotNull(persistenceService.load("recent-node", ClusterNode.class)); + }); } @Test @@ -411,11 +412,11 @@ public void testPurgeByScopeDelegatesToPersistenceService() { persistenceService.save(otherNode); }); - // Execute purge by scope - clusterService.purge("testScope"); + executionContextManager.executeAsSystem(() -> { + clusterService.purge("testScope"); - // Verify: scoped node removed, other node remains - assertNull(persistenceService.load("scoped-node", ClusterNode.class)); - assertNotNull(persistenceService.load("other-node", ClusterNode.class)); + assertNull(persistenceService.load("scoped-node", ClusterNode.class)); + assertNotNull(persistenceService.load("other-node", ClusterNode.class)); + }); } } diff --git a/services/src/test/java/org/apache/unomi/services/impl/scheduler/SchedulerServiceImplTest.java b/services/src/test/java/org/apache/unomi/services/impl/scheduler/SchedulerServiceImplTest.java index 7a31d62ab..593867e2a 100644 --- a/services/src/test/java/org/apache/unomi/services/impl/scheduler/SchedulerServiceImplTest.java +++ b/services/src/test/java/org/apache/unomi/services/impl/scheduler/SchedulerServiceImplTest.java @@ -2033,8 +2033,10 @@ public void execute(ScheduledTask task, TaskStatusCallback callback) { taskId.set(systemTask.getItemId()); - // Wait for task to execute once + // Wait for task to execute once, then cancel immediately so fixed-rate scheduling + // cannot fire a second execution before we assert the count (CI timing flake). assertTrue(executionLatch.await(TEST_TIMEOUT, TEST_TIME_UNIT), "Task should execute"); + schedulerService.cancelTask(systemTask.getItemId()); assertEquals(1, executionCount.get(), "Task should have executed once"); // Force persistence to ensure the task is saved @@ -2485,7 +2487,17 @@ public void execute(ScheduledTask task, TaskStatusCallback callback) { @Test @Tag("RecoveryTests") public void testPreDestroyMarksStaleRunningPersistentTaskAsCrashed() throws Exception { - ScheduledTask runningTask = createTestTask("predestroy-crash-test", ScheduledTask.TaskStatus.RUNNING); + // Persist once with a valid lock. createTestTask() saves RUNNING tasks without a + // lock first, which lets the background recovery loop (initial delay 0) mark them + // CRASHED before preDestroy() under CI scheduling. + ScheduledTask runningTask = new ScheduledTask(); + runningTask.setItemId(UUID.randomUUID().toString()); + runningTask.setTaskType("predestroy-crash-test"); + runningTask.setStatus(ScheduledTask.TaskStatus.RUNNING); + runningTask.setLockOwner(schedulerService.getNodeId()); + runningTask.setLockDate(new Date()); + runningTask.setExecutingNodeId(schedulerService.getNodeId()); + persistenceService.save(runningTask); persistenceService.refresh(); schedulerService.preDestroy();