Differential flames

[l0rinc]

Separate flames per run are useful, but most of the time their difference is more revealing.

This could be done via Differential flames, but we'd need the raw stacks for which we would need to use the official https://github.com/brendangregg/FlameGraph instead of the current Rust ones which merges the results directly.

The differential flames for an IBD would look something like this:

This compares master with the AutoFile and XOR batching PR.
If we zoom into it the diffs instantly show us that AutoFile::write is called a lot fewer times (blue means fewer times, the shade of it is proportional to the difference):

and

We could even do multiple before/after runs for better clarity of where the differences lie - and even per-thread diffs to make sure the color intensities don't affect each other from other threads.
After we have the framework for processing the before/after data, most of the other charts should also be unified to show before/after on the same plot with different colors instead.

The script I used to experiment with it looks like this (incomplete proof-of-concept, measurs loadtxoutset instead of actual IBD):

#!/bin/bash
# Bitcoin Differential Flame Graph Generator (Linux/perf version)
# Generates differential flame graphs for UTXO operations between commits
set -e

# Kill any existing bitcoind and perf processes
echo "Killing any existing bitcoind and perf processes..."
killall -SIGKILL bitcoind 2>/dev/null || true
pkill -9 perf 2>/dev/null || true
sleep 2

# Fixed paths
FLAMEGRAPH_DIR="/mnt/my_storage/FlameGraph"
BITCOIN_DIR="/mnt/my_storage/bitcoin"
BITCOIN_DATA_DIR="${BITCOIN_DIR}/demo"
OUTPUT_DIR="${BITCOIN_DIR}/flamegraphs"
UTXO_FILE="/mnt/my_storage/utxo-840000.dat"

# Configurable parameters
NUM_RUNS=${1:-1}
OPERATION=${2:-"loadtxoutset"}
BEFORE_COMMIT=${3:-"df8bf657450d5383870d40790ea9f4fdb03c360d"}
AFTER_COMMIT=${4:-"868413340f8d6058d74186b65ac3498d6b7f254a"}
NUM_SAMPLES=${NUM_SAMPLES:-100}
NPROC=$(nproc)

# Try to increase the perf sample rate limit
if [ -w "/proc/sys/kernel/perf_event_max_sample_rate" ]; then
  echo "Increasing perf_event_max_sample_rate..."
  echo 100000 > /proc/sys/kernel/perf_event_max_sample_rate
fi

# Create output directories
mkdir -p "${OUTPUT_DIR}/${OPERATION}"
cd "$BITCOIN_DIR"

# Update git repository
echo "Updating Git repository..."
git fetch --all

# Validate commits
for commit in "$BEFORE_COMMIT" "$AFTER_COMMIT"; do
  if ! git rev-parse --verify "$commit" >/dev/null 2>&1; then
    echo "Invalid commit: $commit"
    exit 1
  fi
done

initialize_bitcoind() {
  echo "Setting up environment for proper UTXO loading..."
  
  # Clean environment
  echo "Cleaning environment..."
  mkdir -p "${BITCOIN_DATA_DIR}"
  rm -rf "${BITCOIN_DATA_DIR}/chainstate" "${BITCOIN_DATA_DIR}/chainstate_snapshot" "${BITCOIN_DATA_DIR}/debug.log"
  
  # Step 1: Initialize data directory by running bitcoind with stopatheight=1
  echo "Step 1: Initializing data directory with stopatheight=1..."
  "${BITCOIN_DIR}/build/src/bitcoind" -datadir="${BITCOIN_DATA_DIR}" -stopatheight=1
  
  # Wait for bitcoind to stop
  echo -n "Waiting for initialization to complete"
  while pgrep -f "${BITCOIN_DIR}/build/src/bitcoind" >/dev/null; do
    sleep 1
    echo -n "."
  done
  echo ""
  sleep 2
  
  # Step 2: Start bitcoind as daemon
  echo "Step 2: Starting bitcoind in daemon mode..."
  "${BITCOIN_DIR}/build/src/bitcoind" -datadir="${BITCOIN_DATA_DIR}" -daemon -blocksonly=1 -connect=0 -dbcache=30000
  sleep 5
  
  # Verify bitcoind started
  if ! pgrep -f "${BITCOIN_DIR}/build/src/bitcoind" >/dev/null; then
    echo "Error: Failed to start bitcoind"
    exit 1
  fi
  
  echo "Environment initialization completed."
}

run_benchmark() {
  local version="$1"
  local merged_output="${OUTPUT_DIR}/${OPERATION}/stacks_${version}.txt"

  echo "===== Running ${version} benchmark for ${OPERATION} (${NUM_RUNS} iterations) ====="

  echo "Building bitcoind..."
  cmake -B build -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_WALLET=OFF
  cmake --build build -j"$NPROC" --target bitcoind bitcoin-cli

  > "$merged_output"  # Clear file

  for run in $(seq 1 "$NUM_RUNS"); do
    echo "Starting iteration ${run}/${NUM_RUNS}..."
    local temp_output="${OUTPUT_DIR}/${OPERATION}/stacks_${version}_run${run}.txt"
    local perf_data="${OUTPUT_DIR}/${OPERATION}/perf_${version}_run${run}.data"

    # Initialize bitcoind properly for each run
    initialize_bitcoind

    # Get bitcoind PID
    BITCOIN_PID=$(pgrep -f "${BITCOIN_DIR}/build/src/bitcoind")
    echo "Bitcoin Core running with PID: ${BITCOIN_PID}"

    # Make sure no perf is running
    pkill -9 perf 2>/dev/null || true
    sleep 1

    echo "Starting perf profiling..."
    # Using a lower frequency to avoid throttling, with large buffers
    # Use --no-buildid-cache to avoid addr2line issues
    perf record -F 99 -g --call-graph dwarf --no-buildid-cache -m 512 -p "$BITCOIN_PID" -o "$perf_data" &
    PROFILER_PID=$!

    sleep 2
    if ! kill -0 "$PROFILER_PID" 2>/dev/null; then
      echo "Profiler failed to start"
      exit 1
    fi

    echo "Executing ${OPERATION}..."
    if [ "$OPERATION" = "loadtxoutset" ]; then
      # Step 3: Load the UTXO snapshot
      "${BITCOIN_DIR}/build/src/bitcoin-cli" -datadir="${BITCOIN_DATA_DIR}" loadtxoutset "$UTXO_FILE"
      
      # Verify UTXO loading was successful
      if grep -q "FlushSnapshotToDisk: completed" "${BITCOIN_DATA_DIR}/debug.log"; then
        echo "UTXO snapshot loaded successfully!"
      else
        echo "Warning: Could not find confirmation of successful UTXO loading in logs."
      fi
    elif [ "$OPERATION" = "gettxoutsetinfo" ]; then
      for i in $(seq 1 "$NUM_SAMPLES"); do
        echo -n "."
        [ $((i % 50)) -eq 0 ] && echo " $i/$NUM_SAMPLES"
        "${BITCOIN_DIR}/build/src/bitcoin-cli" -datadir="${BITCOIN_DATA_DIR}" gettxoutsetinfo >/dev/null
        sleep 0.1  # Small delay between calls
      done
      echo ""
    fi

    echo "Stopping bitcoind..."
    "${BITCOIN_DIR}/build/src/bitcoin-cli" -datadir="${BITCOIN_DATA_DIR}" stop
    
    echo -n "Waiting for bitcoind to stop"
    while pgrep -f "${BITCOIN_DIR}/build/src/bitcoind" >/dev/null; do
      sleep 1
      echo -n "."
    done
    echo ""

    # Give perf time to finish writing its buffers BEFORE we try to kill it
    echo "Allowing perf to finalize buffers (10 seconds)..."
    sleep 10
    
    echo "Stopping perf properly..."
    # First send SIGUSR2 which tells perf to flush its buffers
    kill -USR2 $PROFILER_PID 2>/dev/null || true
    sleep 5
    
    # Now we can terminate perf normally
    kill -TERM $PROFILER_PID 2>/dev/null || true
    
    # Wait for perf to terminate on its own
    echo -n "Waiting for perf to exit"
    for i in {1..30}; do
      if ! kill -0 $PROFILER_PID 2>/dev/null; then
        break
      fi
      sleep 1
      echo -n "."
    done
    echo ""
    
    # Only force kill if it's still running after waiting
    if kill -0 $PROFILER_PID 2>/dev/null; then
      echo "Perf still running, force killing..."
      kill -9 $PROFILER_PID 2>/dev/null || true
      # Wait to be sure
      sleep 2
    else
      echo "Perf exited normally"
    fi
    
    # Make sure other perf processes are stopped too
    pkill perf 2>/dev/null || true
    sleep 2
    
    # Check if perf data file exists and has data
    if [ ! -f "$perf_data" ] || [ ! -s "$perf_data" ]; then
      echo "Error: Perf data file is missing or empty"
      ls -la "$perf_data" 2>/dev/null || echo "File does not exist"
      exit 1
    fi
    
    # Convert perf data to readable stacks - redirect stderr to suppress addr2line errors
    echo "Converting perf data to text format (ignoring addr2line errors)..."
    perf script --no-demangle -i "$perf_data" > "$temp_output" 2>/dev/null
    
    if [ ! -s "$temp_output" ]; then
      echo "Error: Generated stack trace is empty, trying again with different flags..."
      # Try again with simpler flags
      perf script -i "$perf_data" > "$temp_output" 2>/dev/null
      
      if [ ! -s "$temp_output" ]; then
        echo "Error: Perf output for run ${run} is still empty"
        exit 1
      fi
    fi
    
    # Append to merged output
    cat "$temp_output" >> "$merged_output"
    echo "Run ${run} completed, data appended ($(wc -l < "$temp_output") lines)."
    
    # Keep both files for debugging
    echo "Saving perf data files for reference..."
    
    if [ $run -lt $NUM_RUNS ]; then
      echo "Waiting for system to stabilize before next run..."
      sleep 5
    fi
  done

  echo "${version} benchmark completed. Combined data saved to ${merged_output}"
}

generate_flame_graphs() {
  local output_prefix="${OUTPUT_DIR}/${OPERATION}/diff_flame"

  echo "===== Generating flame graphs for ${OPERATION} ====="

  echo "Collapsing stack data..."
  echo "Collapsing before stacks..."
  "${FLAMEGRAPH_DIR}/stackcollapse-perf.pl" "${OUTPUT_DIR}/${OPERATION}/stacks_before.txt" > "${OUTPUT_DIR}/${OPERATION}/stacks_before.folded"
  
  echo "Collapsing after stacks..."
  "${FLAMEGRAPH_DIR}/stackcollapse-perf.pl" "${OUTPUT_DIR}/${OPERATION}/stacks_after.txt" > "${OUTPUT_DIR}/${OPERATION}/stacks_after.folded"

  for file in "${OUTPUT_DIR}/${OPERATION}/stacks_before.folded" "${OUTPUT_DIR}/${OPERATION}/stacks_after.folded"; do
    if [ ! -s "$file" ]; then
      echo "Error: $file is empty or missing"
      exit 1
    fi
    echo "Found $(wc -l < "$file") stack frames in $file"
  done

  echo "Generating differential flame graph..."
  "${FLAMEGRAPH_DIR}/difffolded.pl" -n -s "${OUTPUT_DIR}/${OPERATION}/stacks_before.folded" "${OUTPUT_DIR}/${OPERATION}/stacks_after.folded" > "${OUTPUT_DIR}/${OPERATION}/diff_stacks.txt"
  "${FLAMEGRAPH_DIR}/flamegraph.pl" --title "Bitcoin Core ${OPERATION} Differential (Before vs After)" --width 1800 --colors hot --negate "${OUTPUT_DIR}/${OPERATION}/diff_stacks.txt" > "${output_prefix}.svg"

  echo "Generating individual flame graphs..."
  "${FLAMEGRAPH_DIR}/flamegraph.pl" --title "Bitcoin Core ${OPERATION} - Before" --width 1400 "${OUTPUT_DIR}/${OPERATION}/stacks_before.folded" > "${output_prefix}_before.svg"
  "${FLAMEGRAPH_DIR}/flamegraph.pl" --title "Bitcoin Core ${OPERATION} - After" --width 1400 "${OUTPUT_DIR}/${OPERATION}/stacks_after.folded" > "${output_prefix}_after.svg"

  echo "Flame graphs generated:"
  ls -lh "${output_prefix}"*.svg
}

# Main execution
echo "=== Bitcoin Differential Flame Graph Generator (Linux/perf) ==="
echo "Operation: ${OPERATION}"
echo "Runs:      ${NUM_RUNS}"
echo "Samples:   ${NUM_SAMPLES} (for gettxoutsetinfo)"
echo "Before commit: ${BEFORE_COMMIT}"
echo "After commit:  ${AFTER_COMMIT}"

echo "Checking out before commit (${BEFORE_COMMIT})..."
git checkout "$BEFORE_COMMIT"
run_benchmark "before"

echo "Checking out after commit (${AFTER_COMMIT})..."
git checkout "$AFTER_COMMIT"
run_benchmark "after"

generate_flame_graphs

echo "===== Analysis complete ====="
echo "Red: More CPU in AFTER | Blue: More CPU in BEFORE"
echo "Open ${OUTPUT_DIR}/${OPERATION}/diff_flame.svg to view results"