diff --git a/.maestro/scripts/run-tests.sh b/.maestro/scripts/run-tests.sh index afaf96887..22402c76f 100755 --- a/.maestro/scripts/run-tests.sh +++ b/.maestro/scripts/run-tests.sh @@ -1,21 +1,24 @@ #!/bin/bash -# run-tests.sh - set up a device, build the example app, and run Maestro flows. +# run-tests.sh - set up devices, build the example app, and run Maestro flows. # # Usage: -# ./run-tests.sh --platform [--update-screenshots] [--rebuild] [flow ...] +# ./run-tests.sh --platform [--update-screenshots] [--rebuild] [--shards N] [flow ...] # # Options: # --platform Required. Target platform: ios or android. # --update-screenshots Refresh baselines. # --rebuild Force a rebuild and install, even if the app is # already installed on the device. +# --shards N Number of devices to boot and split flows across +# (default 1). Automatically clamped to the number of +# flows being run. # flow ... One or more Maestro flow files or directories to run. # Defaults to all component suites if omitted. # # Examples: # ./run-tests.sh --platform ios # ./run-tests.sh --platform android --update-screenshots .maestro/enrichedInput/flows/core_controls_smoke.yaml -# ./run-tests.sh --platform ios --rebuild +# ./run-tests.sh --platform ios --rebuild --shards 2 set -euo pipefail @@ -38,10 +41,12 @@ REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" MAESTRO_ROOT="$REPO_ROOT/.maestro" SCREENSHOT_ROOT="$MAESTRO_ROOT" BUNDLE_ID="swmansion.enriched.example" +EXAMPLE_DIR="$REPO_ROOT/apps/example" PLATFORM="" UPDATE_SCREENSHOTS="" REBUILD="" +SHARDS=1 FLOWS="" while [ $# -gt 0 ]; do @@ -49,10 +54,16 @@ while [ $# -gt 0 ]; do --platform) PLATFORM="$2"; shift 2 ;; --update-screenshots) UPDATE_SCREENSHOTS="true"; shift ;; --rebuild) REBUILD="true"; shift ;; + --shards) SHARDS="$2"; shift 2 ;; *) FLOWS="${FLOWS:+$FLOWS }$1"; shift ;; esac done +if ! [[ "$SHARDS" =~ ^[0-9]+$ ]]; then + echo "Error: --shards requires a numeric integer. Received: '$SHARDS'" >&2 + exit 1 +fi + [ -z "$FLOWS" ] && FLOWS=".maestro/enrichedInput/flows .maestro/enrichedText/flows" case "$PLATFORM" in @@ -61,28 +72,61 @@ case "$PLATFORM" in *) echo "Error: --platform must be ios or android" >&2; exit 1 ;; esac -DEVICE_ID=$("$SETUP" | tee /dev/tty | grep "^DEVICE_ID=" | cut -d= -f2) +# No point booting more devices than there are flows to run +# (e.g. when running a single flow file). +FLOW_COUNT=0 +for f in $FLOWS; do + if [ -d "$f" ]; then + n=$(find "$f" -name '*.yaml' | wc -l | tr -d ' ') + elif [ -f "$f" ]; then + n=1 + else + n=0 + fi + FLOW_COUNT=$((FLOW_COUNT + n)) +done +if [ "$FLOW_COUNT" -gt 0 ] && [ "$SHARDS" -gt "$FLOW_COUNT" ]; then + SHARDS=$FLOW_COUNT +fi +[ "$SHARDS" -lt 1 ] && SHARDS=1 + +# Stream setup progress to the terminal; fall back to stderr when +# no TTY is attached (e.g. CI). +TTY_OUT=/dev/tty +( : > /dev/tty ) 2>/dev/null || TTY_OUT=/dev/stderr + +DEVICE_IDS_CSV=$("$SETUP" "$SHARDS" | tee "$TTY_OUT" | grep "^DEVICE_IDS=" | cut -d= -f2) +if [ -z "$DEVICE_IDS_CSV" ]; then + echo "Error: setup script did not return any DEVICE_IDS" >&2 + exit 1 +fi +IFS=',' read -r -a DEVICES <<< "$DEVICE_IDS_CSV" +DEVICE_ID="${DEVICES[0]}" app_installed() { + local dev="$1" if [ "$PLATFORM" = ios ]; then - xcrun simctl listapps "$DEVICE_ID" 2>/dev/null | grep -q "$BUNDLE_ID" + xcrun simctl listapps "$dev" 2>/dev/null | grep -q "$BUNDLE_ID" else - adb -s "$DEVICE_ID" shell pm list packages "$BUNDLE_ID" 2>/dev/null | grep -q "$BUNDLE_ID" + adb -s "$dev" shell pm list packages "$BUNDLE_ID" 2>/dev/null | grep -q "$BUNDLE_ID" fi } -# Adjusts the system's text-size setting. +# Adjusts the system's text-size setting on every shard device. set_font_scale() { + local dev case "$1" in default) ios_size="large"; android_scale="1.0" ;; large) ios_size="accessibility-large"; android_scale="1.5" ;; *) echo "set_font_scale: unknown size '$1'" >&2; return 1 ;; esac - if [ "$PLATFORM" = ios ]; then - xcrun simctl ui "$DEVICE_ID" content_size "$ios_size" - else - adb -s "$DEVICE_ID" shell settings put system font_scale "$android_scale" - fi + for dev in "${DEVICES[@]}"; do + if [ "$PLATFORM" = ios ]; then + xcrun simctl ui "$dev" content_size "$ios_size" + else + adb -s "$dev" shell settings put system font_scale "$android_scale" + fi + done } # Guarantees the font scale is restored on any exit. @@ -90,18 +134,73 @@ set_font_scale() { # in a scaled-up state. trap 'set_font_scale default' EXIT -if [ -n "$REBUILD" ] || ! app_installed; then - [ -n "$REBUILD" ] && echo "=== rebuild requested, building and installing ===" - [ -z "$REBUILD" ] && echo "=== App ($BUNDLE_ID) not found, building and installing ===" +# Builds and installs on the given device via the react-native CLI. +build_and_install() { + local dev="$1" + if [ "$PLATFORM" = ios ]; then + yarn example ios --udid "$dev" + else + yarn example android --device "$dev" + fi +} + +# Installs the already-built app artifact on the given device. Tries the +# build output first, then the copy installed on the primary device, and +# only falls back to a full build if neither exists. +install_prebuilt() { + local dev="$1" app_path apk_path if [ "$PLATFORM" = ios ]; then - yarn example ios --udid "$DEVICE_ID" + app_path=$(ls -td "$EXAMPLE_DIR/ios/build/Build/Products/Debug-iphonesimulator"/*.app 2>/dev/null | head -1 || true) + if [ -z "$app_path" ]; then + app_path=$(xcrun simctl get_app_container "$DEVICE_ID" "$BUNDLE_ID" app 2>/dev/null || true) + fi + if [ -n "$app_path" ]; then + echo "=== Installing prebuilt app on $dev ===" + xcrun simctl install "$dev" "$app_path" + else + echo "warn: no prebuilt .app found, building for $dev" >&2 + build_and_install "$dev" + fi else - yarn example android --device "$DEVICE_ID" + apk_path=$(ls -t "$EXAMPLE_DIR/android/app/build/outputs/apk/debug"/*.apk 2>/dev/null | head -1 || true) + if [ -z "$apk_path" ]; then + local device_apk + device_apk=$(adb -s "$DEVICE_ID" shell pm path "$BUNDLE_ID" 2>/dev/null | head -1 | cut -d: -f2 | tr -d '\r' || true) + if [ -n "$device_apk" ]; then + apk_path=$(mktemp -d)/example.apk + adb -s "$DEVICE_ID" pull "$device_apk" "$apk_path" >/dev/null + fi + fi + if [ -n "$apk_path" ]; then + echo "=== Installing prebuilt app on $dev ===" + adb -s "$dev" install -r "$apk_path" + else + echo "warn: no prebuilt .apk found, building for $dev" >&2 + build_and_install "$dev" + fi fi +} + +if [ -n "$REBUILD" ] || ! app_installed "$DEVICE_ID"; then + [ -n "$REBUILD" ] && echo "=== rebuild requested, building and installing ===" + [ -z "$REBUILD" ] && echo "=== App ($BUNDLE_ID) not found, building and installing ===" + build_and_install "$DEVICE_ID" else - echo "=== App ($BUNDLE_ID) already installed, skipping build ===" + echo "=== App ($BUNDLE_ID) already installed on $DEVICE_ID, skipping build ===" fi +# Reuse the artifact built above for the remaining shard devices. +i=1 +while [ "$i" -lt "${#DEVICES[@]}" ]; do + dev="${DEVICES[$i]}" + if [ -n "$REBUILD" ] || ! app_installed "$dev"; then + install_prebuilt "$dev" + else + echo "=== App ($BUNDLE_ID) already installed on $dev, skipping install ===" + fi + i=$((i + 1)) +done + EXTRA="--env SCREENSHOT_ROOT=$SCREENSHOT_ROOT" [ -n "$UPDATE_SCREENSHOTS" ] && EXTRA="$EXTRA --env UPDATE_SCREENSHOTS=true" @@ -111,6 +210,11 @@ case "$PLATFORM" in android) EXTRA="$EXTRA --exclude-tags ios-only" ;; esac +# Split flows across all shard devices. With --shard-split each flow still +# runs exactly once, so shared screenshot baselines stay safe. +DEVICE_ARGS="--device $DEVICE_IDS_CSV" +[ "$SHARDS" -gt 1 ] && DEVICE_ARGS="$DEVICE_ARGS --shard-split $SHARDS" + # Maestro resolves addMedia paths by walking the workspace inputs. Since assets # live outside the flows directory, always include it so media files are found. ASSETS_DIR="$MAESTRO_ROOT/assets" @@ -127,8 +231,11 @@ run_maestro() { local tmp rc tmp=$(mktemp) # `script` allocates a pseudo-TTY so maestro keeps - # ANSI colors when piped through `tee`. - if [[ "$OSTYPE" == darwin* ]]; then + # ANSI colors when piped through `tee`. It needs a real + # terminal though, so skip it when there is none (e.g. CI). + if ! [ -t 1 ]; then + maestro test "$@" 2>&1 | tee "$tmp" + elif [[ "$OSTYPE" == darwin* ]]; then script -q /dev/null maestro test "$@" 2>&1 | tee "$tmp" else local cmd @@ -148,7 +255,7 @@ set +e echo "=== Running maestro tests ===" # shellcheck disable=SC2086 -run_maestro --device "$DEVICE_ID" --exclude-tags accessibility $EXTRA $FLOWS +run_maestro $DEVICE_ARGS --exclude-tags accessibility $EXTRA $FLOWS EXIT_REGULAR=$? # These are the tests that require changing the system's settings @@ -157,7 +264,7 @@ echo "=== Running maestro accessibility tests ===" set_font_scale large # shellcheck disable=SC2086 -run_maestro --device "$DEVICE_ID" --include-tags accessibility $EXTRA $FLOWS +run_maestro $DEVICE_ARGS --include-tags accessibility $EXTRA $FLOWS EXIT_A11Y=$? set -e diff --git a/.maestro/scripts/setup-android-emulator.sh b/.maestro/scripts/setup-android-emulator.sh index 605c8f6bd..59e02d425 100755 --- a/.maestro/scripts/setup-android-emulator.sh +++ b/.maestro/scripts/setup-android-emulator.sh @@ -1,6 +1,21 @@ #!/bin/bash +# setup-android-emulator.sh - boot one or more Android emulator instances. +# +# Usage: +# ./setup-android-emulator.sh [num_devices] +# +# Boots `num_devices` (default 1) independent emulators. +# Prints: +# DEVICE_ID= +# DEVICE_IDS= set -euo pipefail +NUM_DEVICES="${1:-1}" +if ! [[ "$NUM_DEVICES" =~ ^[0-9]+$ ]] || [ "$NUM_DEVICES" -lt 1 ]; then + echo "Error: num_devices must be a positive integer (got '$NUM_DEVICES')" >&2 + exit 1 +fi + API_LEVEL="36" DEVICE_ID="pixel_9" ARCH=$(uname -m) @@ -11,11 +26,10 @@ else fi TAG="google_apis_playstore" SYSTEM_IMAGE="system-images;android-${API_LEVEL};${TAG};${ABI}" -AVD_NAME="Pixel9-API${API_LEVEL}-Enriched" -PORT=5570 -SERIAL="emulator-${PORT}" +AVD_BASE="Pixel9-API${API_LEVEL}-Enriched" +BASE_PORT=5570 -if [ -z "$ANDROID_HOME" ]; then +if [ -z "${ANDROID_HOME:-}" ]; then echo "Error: ANDROID_HOME is not set. Set it to your Android SDK directory." exit 1 fi @@ -39,45 +53,78 @@ if ! avdmanager list device -c | grep -qx "$DEVICE_ID"; then exit 1 fi -if ! avdmanager list avd -c | grep -qx "${AVD_NAME}"; then - echo "Creating AVD '$AVD_NAME'..." - echo "no" | avdmanager create avd \ - --name "$AVD_NAME" \ - --device "$DEVICE_ID" \ - --package "$SYSTEM_IMAGE" \ - --skin "$DEVICE_ID" -fi +configure_avd() { + local avd_name="$1" + local avd_config="$HOME/.android/avd/${avd_name}.avd/config.ini" + if [ -f "$avd_config" ]; then + sed -i '' 's/^hw\.keyboard=.*/hw.keyboard=yes/' "$avd_config" + grep -q "^hw.keyboard=" "$avd_config" || echo "hw.keyboard=yes" >> "$avd_config" + sed -i '' 's/^hw\.mainKeys=.*/hw.mainKeys=yes/' "$avd_config" + grep -q "^hw.mainKeys=" "$avd_config" || echo "hw.mainKeys=yes" >> "$avd_config" + fi +} -AVD_CONFIG="$HOME/.android/avd/${AVD_NAME}.avd/config.ini" -if [ -f "$AVD_CONFIG" ]; then - sed -i '' 's/^hw\.keyboard=.*/hw.keyboard=yes/' "$AVD_CONFIG" - grep -q "^hw.keyboard=" "$AVD_CONFIG" || echo "hw.keyboard=yes" >> "$AVD_CONFIG" - sed -i '' 's/^hw\.mainKeys=.*/hw.mainKeys=yes/' "$AVD_CONFIG" - grep -q "^hw.mainKeys=" "$AVD_CONFIG" || echo "hw.mainKeys=yes" >> "$AVD_CONFIG" -fi +SERIALS=() +AVD_NAMES=() +i=0 +while [ "$i" -lt "$NUM_DEVICES" ]; do + # Each shard gets its own numbered AVD so they boot fully independently. + if [ "$i" -eq 0 ]; then + AVD_NAME="$AVD_BASE" + else + AVD_NAME="${AVD_BASE}-$((i + 1))" + fi + PORT=$((BASE_PORT + 2 * i)) + SERIAL="emulator-${PORT}" + SERIALS+=("$SERIAL") + AVD_NAMES+=("$AVD_NAME") -if pgrep -f "emulator.*${AVD_NAME}" > /dev/null 2>&1; then - echo "Emulator already running: $AVD_NAME ($SERIAL)" - echo "DEVICE_ID=$SERIAL" - exit 0 -fi + if ! avdmanager list avd -c | grep -qx "$AVD_NAME"; then + echo "Creating AVD '$AVD_NAME'..." + echo "no" | avdmanager create avd \ + --name "$AVD_NAME" \ + --device "$DEVICE_ID" \ + --package "$SYSTEM_IMAGE" \ + --skin "$DEVICE_ID" + fi + configure_avd "$AVD_NAME" + + if adb -s "$SERIAL" get-state >/dev/null 2>&1; then + echo "Emulator already running: $AVD_NAME ($SERIAL)" + else + echo "Starting emulator '$AVD_NAME' ($SERIAL)..." + emulator "@${AVD_NAME}" -port "$PORT" -no-boot-anim -no-snapshot-save > /dev/null 2>&1 & + fi + i=$((i + 1)) +done -echo "Starting emulator '$AVD_NAME'..." -emulator "@${AVD_NAME}" -port "$PORT" > /dev/null 2>&1 & +for idx in "${!SERIALS[@]}"; do + SERIAL="${SERIALS[$idx]}" + AVD_NAME="${AVD_NAMES[$idx]}" -echo "Waiting for emulator ($SERIAL) to connect to ADB..." -if ! timeout 120 adb -s "$SERIAL" wait-for-device; then - echo "Error: Emulator did not connect to ADB after 120s." - exit 1 -fi + echo "Waiting for emulator ($SERIAL) to connect to ADB..." + if ! timeout 120 adb -s "$SERIAL" wait-for-device; then + echo "Error: Emulator $SERIAL did not connect to ADB after 120s." + exit 1 + fi -echo "Waiting for emulator to finish booting..." -until adb -s "$SERIAL" shell getprop sys.boot_completed 2>/dev/null | grep -q "^1$"; do - sleep 2 -done + echo "Waiting for emulator ($SERIAL) to finish booting..." + until adb -s "$SERIAL" shell getprop sys.boot_completed 2>/dev/null | grep -q "^1$"; do + sleep 2 + done + + adb -s "$SERIAL" shell pm disable-user --user 0 com.google.android.inputmethod.latin + adb -s "$SERIAL" shell settings put secure spell_checker_enabled 0 -adb -s "$SERIAL" shell pm disable-user --user 0 com.google.android.inputmethod.latin -adb -s "$SERIAL" shell settings put secure spell_checker_enabled 0 + # Disable system animations so taps/transitions settle instantly and + # waitForAnimationToEnd doesn't pay real animation time. + adb -s "$SERIAL" shell settings put global window_animation_scale 0 + adb -s "$SERIAL" shell settings put global transition_animation_scale 0 + adb -s "$SERIAL" shell settings put global animator_duration_scale 0 + + echo "Emulator ready: $AVD_NAME ($SERIAL)" +done -echo "Emulator ready: $AVD_NAME ($SERIAL)" -echo "DEVICE_ID=$SERIAL" +DEVICE_IDS=$(IFS=,; echo "${SERIALS[*]}") +echo "DEVICE_ID=${SERIALS[0]}" +echo "DEVICE_IDS=$DEVICE_IDS" diff --git a/.maestro/scripts/setup-ios-simulator.sh b/.maestro/scripts/setup-ios-simulator.sh index 5f9d8ea23..6751875b3 100755 --- a/.maestro/scripts/setup-ios-simulator.sh +++ b/.maestro/scripts/setup-ios-simulator.sh @@ -1,11 +1,26 @@ #!/bin/bash +# setup-ios-simulator.sh - boot one or more iOS simulators. +# +# Usage: +# ./setup-ios-simulator.sh [num_devices] +# +# Boots `num_devices` (default 1) simulators. +# Prints: +# DEVICE_ID= +# DEVICE_IDS= set -euo pipefail +NUM_DEVICES="${1:-1}" +if ! [[ "$NUM_DEVICES" =~ ^[0-9]+$ ]] || [ "$NUM_DEVICES" -lt 1 ]; then + echo "Error: num_devices must be a positive integer (got '$NUM_DEVICES')" >&2 + exit 1 +fi + DEVICE_TYPE="com.apple.CoreSimulator.SimDeviceType.iPhone-17" IOS_VERSION="26.2" RUNTIME="com.apple.CoreSimulator.SimRuntime.iOS-$(echo "$IOS_VERSION" | tr '.' '-')" RUNTIME_LABEL="iOS $IOS_VERSION" -DEVICE_NAME="iPhone17-iOS${IOS_VERSION}-Enriched" +BASE_DEVICE_NAME="iPhone17-iOS${IOS_VERSION}-Enriched" if ! xcrun simctl list runtimes | grep -q "$RUNTIME"; then echo "Error: $RUNTIME_LABEL runtime not found." @@ -13,40 +28,55 @@ if ! xcrun simctl list runtimes | grep -q "$RUNTIME"; then exit 1 fi -if ! xcrun simctl list devices | grep -q "$DEVICE_NAME ("; then - echo "Creating simulator '$DEVICE_NAME'..." - xcrun simctl create "$DEVICE_NAME" "$DEVICE_TYPE" "$RUNTIME" -fi +UDIDS=() +i=1 +while [ "$i" -le "$NUM_DEVICES" ]; do + if [ "$i" -eq 1 ]; then + DEVICE_NAME="$BASE_DEVICE_NAME" + else + DEVICE_NAME="${BASE_DEVICE_NAME}-${i}" + fi -UDID=$(xcrun simctl list devices | grep "$DEVICE_NAME (" | head -1 | grep -oE '[A-F0-9-]{36}') + if ! xcrun simctl list devices | grep -q "$DEVICE_NAME ("; then + echo "Creating simulator '$DEVICE_NAME'..." + xcrun simctl create "$DEVICE_NAME" "$DEVICE_TYPE" "$RUNTIME" + fi -if [ -z "$UDID" ]; then - echo "Error: Could not find UDID for '$DEVICE_NAME'" - exit 1 -fi + UDID=$(xcrun simctl list devices | grep "$DEVICE_NAME (" | head -1 | grep -oE '[A-F0-9-]{36}') -# disable automatic text manipulation: auto-correction, spelling-check and auto-capitalization -SIM_PREFS_DIR="$HOME/Library/Developer/CoreSimulator/Devices/$UDID/data/Library/Preferences" -mkdir -p "$SIM_PREFS_DIR" + if [ -z "$UDID" ]; then + echo "Error: Could not find UDID for '$DEVICE_NAME'" + exit 1 + fi -PLIST="$SIM_PREFS_DIR/com.apple.keyboard.preferences.plist" + # disable automatic text manipulation: auto-correction, spelling-check and auto-capitalization + SIM_PREFS_DIR="$HOME/Library/Developer/CoreSimulator/Devices/$UDID/data/Library/Preferences" + mkdir -p "$SIM_PREFS_DIR" -/usr/libexec/PlistBuddy -c "Add :KeyboardAutocorrection bool false" "$PLIST" 2>/dev/null || \ -/usr/libexec/PlistBuddy -c "Set :KeyboardAutocorrection bool false" "$PLIST" + PLIST="$SIM_PREFS_DIR/com.apple.keyboard.preferences.plist" -/usr/libexec/PlistBuddy -c "Add :KeyboardCheckSpelling bool false" "$PLIST" 2>/dev/null || \ -/usr/libexec/PlistBuddy -c "Set :KeyboardCheckSpelling bool false" "$PLIST" + /usr/libexec/PlistBuddy -c "Add :KeyboardAutocorrection bool false" "$PLIST" 2>/dev/null || \ + /usr/libexec/PlistBuddy -c "Set :KeyboardAutocorrection bool false" "$PLIST" -/usr/libexec/PlistBuddy -c "Add :KeyboardAutocapitalization bool false" "$PLIST" 2>/dev/null || \ -/usr/libexec/PlistBuddy -c "Set :KeyboardAutocapitalization bool false" "$PLIST" + /usr/libexec/PlistBuddy -c "Add :KeyboardCheckSpelling bool false" "$PLIST" 2>/dev/null || \ + /usr/libexec/PlistBuddy -c "Set :KeyboardCheckSpelling bool false" "$PLIST" -STATE=$(xcrun simctl list devices | grep "$UDID" | grep -oE '\(Booted\)|\(Shutdown\)' || true) -if [ "$STATE" != "(Booted)" ]; then - echo "Booting '$DEVICE_NAME' ($UDID)..." - xcrun simctl boot "$UDID" -fi + /usr/libexec/PlistBuddy -c "Add :KeyboardAutocapitalization bool false" "$PLIST" 2>/dev/null || \ + /usr/libexec/PlistBuddy -c "Set :KeyboardAutocapitalization bool false" "$PLIST" + + STATE=$(xcrun simctl list devices | grep "$UDID" | grep -oE '\(Booted\)|\(Shutdown\)' || true) + if [ "$STATE" != "(Booted)" ]; then + echo "Booting '$DEVICE_NAME' ($UDID)..." + xcrun simctl boot "$UDID" + fi + + echo "Simulator ready: $DEVICE_NAME ($UDID)" + UDIDS+=("$UDID") + i=$((i + 1)) +done open -a Simulator -echo "Simulator ready: $DEVICE_NAME ($UDID)" -echo "DEVICE_ID=$UDID" +DEVICE_IDS=$(IFS=,; echo "${UDIDS[*]}") +echo "DEVICE_ID=${UDIDS[0]}" +echo "DEVICE_IDS=$DEVICE_IDS"