auto-capture.sh

#!/bin/bash

# Auto-capture script - Captures market data every 5 minutes
# Usage: ./auto-capture.sh [duration_in_minutes]
# Example: ./auto-capture.sh 60  (runs for 1 hour)

# Colors for output
GREEN='\033[0;32m'
BLUE='\033[0;34m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
NC='\033[0m' # No Color

# Configuration
MARKET_INTERVAL=60  # 1 minute in seconds (for market data)
OI_INTERVAL=180  # 3 minutes in seconds (for option chain OI data)
BREAKOUT_INTERVAL=60  # 1 minute in seconds (for breakout/breakdown checking)
PCR_INTERVAL=60  # 1 minute in seconds (for PCR calculation)
STOCK_SNAPSHOT_INTERVAL=60  # 1 minute in seconds (for stock snapshots)
BREAKOUT_SNAPSHOT_INTERVAL=180  # 3 minutes in seconds (for breakout snapshots)
CRON_SECRET="${CRON_SECRET:-9f3c1a7e4b2d8f0a6e9c3d1b5f7a2e4c8a6d0b9e3f2c1a4d7e8b5c6f0}"
MARKET_DATA_API_URL="http://localhost:3000/api/cron/capture-market-data"
OPTION_CHAIN_API_URL="http://localhost:3000/api/option-chain/cron"
DAILY_HIGH_LOW_API_URL="http://localhost:3000/api/cron/save-daily-high-low"
BREAKOUT_CHECK_API_URL="http://localhost:3000/api/cron/check-breakouts"
PCR_API_URL="http://localhost:3000/api/cron/calculate-pcr"
STOCK_SNAPSHOT_API_URL="http://localhost:3000/api/cron/update-stock-snapshots"
BREAKOUT_SNAPSHOT_API_URL="http://localhost:3000/api/cron/update-breakout-snapshots"

# Get duration from argument or default to infinite
DURATION_MINUTES=${1:-0}
if [ "$DURATION_MINUTES" -gt 0 ]; then
    END_TIME=$(($(date +%s) + DURATION_MINUTES * 60))
    echo -e "${BLUE}🕐 Auto-capture will run for ${DURATION_MINUTES} minutes${NC}"
else
    END_TIME=0
    echo -e "${BLUE}🕐 Auto-capture will run indefinitely (Ctrl+C to stop)${NC}"
fi

echo -e "${BLUE}📊 Capturing data:${NC}"
echo -e "${BLUE}   - Market data (sectors): every 1 minute${NC}"
echo -e "${BLUE}   - Stock snapshots (top movers): every 3 minutes (9:15 AM - 3:30 PM IST)${NC}"
echo -e "${BLUE}   - Breakout snapshots: every 3 minutes (9:15 AM - 3:30 PM IST)${NC}"
echo -e "${BLUE}   - Option chain OI data: every 3 minutes (9:15 AM - 3:30 PM IST)${NC}"
echo -e "${BLUE}   - Breakout/Breakdown check: every 1 minute (9:15 AM - 3:30 PM IST)${NC}"
echo -e "${BLUE}   - PCR calculation: every 1 minute (9:15 AM - 3:30 PM IST)${NC}"
echo -e "${BLUE}   - Daily High-Low (yfinance): at startup${NC}"
echo -e "${YELLOW}Press Ctrl+C to stop${NC}\n"

# Counter for successful captures
MARKET_CAPTURE_COUNT=0
OI_CAPTURE_COUNT=0
BREAKOUT_CHECK_COUNT=0
PCR_CAPTURE_COUNT=0
STOCK_SNAPSHOT_COUNT=0
BREAKOUT_SNAPSHOT_COUNT=0
DAILY_HIGH_LOW_CAPTURED=false
PYTHON_SCRIPT_RUN=false

# Function to call API endpoint with retry logic
call_api() {
    local api_url=$1
    local api_name=$2
    local max_retries=3
    local retry_count=0
    local wait_time=2
    
    while [ $retry_count -lt $max_retries ]; do
        response=$(curl -s -w "\nHTTP_STATUS:%{http_code}" \
            -X GET "$api_url" \
            -H "Authorization: Bearer $CRON_SECRET")
        
        http_status=$(echo "$response" | grep HTTP_STATUS | cut -d':' -f2)
        body=$(echo "$response" | sed '/HTTP_STATUS/d')
        
        if [ "$http_status" = "200" ]; then
            echo "$body"
            return 0
        elif [ "$http_status" = "404" ]; then
            retry_count=$((retry_count + 1))
            if [ $retry_count -lt $max_retries ]; then
                echo -e "${YELLOW}⚠️  $api_name API not ready (404), retrying in ${wait_time}s... (attempt $retry_count/$max_retries)${NC}"
                sleep $wait_time
                wait_time=$((wait_time * 2))  # Exponential backoff
            else
                echo -e "${RED}❌ Error: $api_name API still not available after $max_retries attempts${NC}"
                return 1
            fi
        else
            echo -e "${RED}❌ Error: $api_name API returned HTTP ${http_status}${NC}"
            echo -e "${RED}   Response: ${body:0:200}...${NC}"
            return 1
        fi
    done
}

# Cleanup function
cleanup_intraday_data() {
    echo -e "${BLUE}🧹 Cleaning up previous day's intraday data...${NC}"
    response=$(call_api "http://localhost:3000/api/cron/cleanup-intraday" "Cleanup")
    if [ $? -eq 0 ]; then
        echo -e "${GREEN}✅ Cleanup complete: Preserved daily_high_low, cleared live snapshots.${NC}"
    fi
}

# Initial cleanup
cleanup_intraday_data

# Run Python script to populate daily_high_low with yesterday's data from yfinance
populate_daily_high_low() {
    echo -e "${BLUE}🐍 Running Python script to fetch yesterday's OHLC data from yfinance...${NC}"
    
    # Check if python3 is available
    if ! command -v python3 &> /dev/null; then
        echo -e "${RED}❌ python3 not found. Please install Python 3.${NC}"
        return 1
    fi
    
    # Run the Python script
    python3 populate_daily_high_low.py
    
    if [ $? -eq 0 ]; then
        echo -e "${GREEN}✅ Python script completed successfully${NC}"
        PYTHON_SCRIPT_RUN=true
        return 0
    else
        echo -e "${RED}❌ Python script failed${NC}"
        return 1
    fi
}

# Run the Python script at startup
populate_daily_high_low

# Function to capture market data (sectors & stocks)
capture_market_data() {
    echo -e "${BLUE}[$(date '+%Y-%m-%d %H:%M:%S')] Capturing market data (sectors)...${NC}"
    
    market_response=$(call_api "$MARKET_DATA_API_URL" "Market Data")
    if [ $? -eq 0 ] && [ -n "$market_response" ]; then
        MARKET_CAPTURE_COUNT=$((MARKET_CAPTURE_COUNT + 1))
        sectors=$(echo "$market_response" | grep -o '"sectors_captured":[0-9]*' | cut -d':' -f2)
        # stocks=$(echo "$market_response" | grep -o '"stocks_captured":[0-9]*' | cut -d':' -f2)
        echo -e "${GREEN}✅ Market Data: ${sectors} sectors (total: ${MARKET_CAPTURE_COUNT})${NC}"
        return 0
    else
        echo -e "${RED}❌ Market Data capture failed${NC}"
        return 1
    fi
}



# Function to calculate PCR (runs every 1 minute during market hours)
calculate_pcr() {
    echo -e "${BLUE}[$(date '+%Y-%m-%d %H:%M:%S')] Calculating PCR...${NC}"
    
    response=$(call_api "$PCR_API_URL" "PCR Calculation")
    exit_code=$?
    if [ $exit_code -eq 0 ] && [ -n "$response" ]; then
        # Check if it was skipped
        message=$(echo "$response" | grep -o '"message":"[^"]*"' | cut -d'"' -f4)
        if [ -n "$message" ]; then
            echo -e "${YELLOW}⏭️  PCR: ${message}${NC}"
            return 0
        fi
        
        # Get counts
        PCR_CAPTURE_COUNT=$((PCR_CAPTURE_COUNT + 1))
        pcr_calculated=$(echo "$response" | grep -o '"pcr_calculated":[0-9]*' | cut -d':' -f2)
        echo -e "${GREEN}✅ PCR: ${pcr_calculated} indices calculated (total: ${PCR_CAPTURE_COUNT})${NC}"
        return 0
    else
        echo -e "${RED}❌ PCR calculation failed${NC}"
        return 1
    fi
}

# Function to check for breakout/breakdown stocks (runs every 1 minute during market hours)
check_breakouts_breakdowns() {
    echo -e "${BLUE}[$(date '+%Y-%m-%d %H:%M:%S')] Checking breakouts/breakdowns...${NC}"
    
    response=$(call_api "$BREAKOUT_CHECK_API_URL" "Breakout Check")
    exit_code=$?
    if [ $exit_code -eq 0 ] && [ -n "$response" ]; then
        # Check if it was skipped
        message=$(echo "$response" | grep -o '"message":"[^"]*"' | cut -d'"' -f4)
        if [ -n "$message" ]; then
            echo -e "${YELLOW}⏭️  Breakout Check: ${message}${NC}"
            return 0
        fi
        
        # Get counts
        BREAKOUT_CHECK_COUNT=$((BREAKOUT_CHECK_COUNT + 1))
        breakouts=$(echo "$response" | grep -o '"breakouts_detected":[0-9]*' | cut -d':' -f2)
        breakdowns=$(echo "$response" | grep -o '"breakdowns_detected":[0-9]*' | cut -d':' -f2)
        stocks_checked=$(echo "$response" | grep -o '"stocks_checked":[0-9]*' | cut -d':' -f2)
        echo -e "${GREEN}✅ Breakout Check: ${stocks_checked} stocks checked - ${breakouts} breakouts, ${breakdowns} breakdowns (total checks: ${BREAKOUT_CHECK_COUNT})${NC}"
        return 0
    else
        echo -e "${RED}❌ Breakout check failed${NC}"
        return 1
    fi
}

# Function to capture option chain OI data
capture_oi_data() {
    echo -e "${BLUE}[$(date '+%Y-%m-%d %H:%M:%S')] Capturing option chain OI data...${NC}"
    
    oi_response=$(call_api "$OPTION_CHAIN_API_URL" "Option Chain")
    oi_exit_code=$?
    if [ $oi_exit_code -eq 0 ] && [ -n "$oi_response" ]; then
        # Check if it was skipped due to market hours
        skipped=$(echo "$oi_response" | grep -o '"message":"[^"]*"' | cut -d'"' -f4)
        if [ "$skipped" = "Outside market hours" ]; then
            echo -e "${YELLOW}⏭️  Option Chain: Skipped (outside market hours)${NC}"
            return 0
        fi
        
        # Check if it was successful
        success=$(echo "$oi_response" | grep -o '"success":[^,}]*' | cut -d':' -f2 | tr -d ' ')
        if [ "$success" = "true" ]; then
            OI_CAPTURE_COUNT=$((OI_CAPTURE_COUNT + 1))
            symbol=$(echo "$oi_response" | grep -o '"symbol":"[^"]*"' | cut -d'"' -f4)
            expiry=$(echo "$oi_response" | grep -o '"expiry_date":"[^"]*"' | cut -d'"' -f4)
            putOI=$(echo "$oi_response" | grep -o '"total_put_oi":[0-9]*' | cut -d':' -f2)
            callOI=$(echo "$oi_response" | grep -o '"total_call_oi":[0-9]*' | cut -d':' -f2)
            echo -e "${GREEN}✅ Option Chain: ${symbol} (${expiry}) - Put OI: ${putOI}, Call OI: ${callOI} (total: ${OI_CAPTURE_COUNT})${NC}"
            return 0
        else
            # Check for error message
            error_msg=$(echo "$oi_response" | grep -o '"error":"[^"]*"' | cut -d'"' -f4 | head -1)
            error_details=$(echo "$oi_response" | grep -o '"details":"[^"]*"' | cut -d'"' -f4 | head -1)
            echo -e "${RED}❌ Option Chain capture failed${NC}"
            if [ -n "$error_msg" ]; then
                echo -e "${YELLOW}   Error: ${error_msg}${NC}"
            fi
            if [ -n "$error_details" ]; then
                echo -e "${YELLOW}   Details: ${error_details}${NC}"
            fi
            # Also show full response for debugging (first 200 chars)
            echo -e "${YELLOW}   Response: ${oi_response:0:200}...${NC}"
            return 1
        fi
    else
        echo -e "${RED}❌ Option Chain capture failed${NC}"
        if [ -n "$oi_response" ]; then
            error_msg=$(echo "$oi_response" | grep -o '"error":"[^"]*"' | cut -d'"' -f4 | head -1)
            if [ -n "$error_msg" ]; then
                echo -e "${YELLOW}   Error: ${error_msg}${NC}"
            fi
            # Show full response for debugging (first 200 chars)
            echo -e "${YELLOW}   Response: ${oi_response:0:200}...${NC}"
        fi
        return 1
    fi
}

# Function to calculate PCR (Put-Call Ratio)
calculate_pcr() {
    echo -e "${BLUE}[$(date '+%Y-%m-%d %H:%M:%S')] Calculating PCR...${NC}"
    
    response=$(call_api "$PCR_API_URL" "PCR Calculation")
    exit_code=$?
    if [ $exit_code -eq 0 ] && [ -n "$response" ]; then
        # Check if it was skipped
        message=$(echo "$response" | grep -o '"message":"[^"]*"' | cut -d'"' -f4)
        if [ -n "$message" ]; then
            echo -e "${YELLOW}⏭️  PCR Calculation: ${message}${NC}"
            return 0
        fi
        
        # Get counts
        PCR_CAPTURE_COUNT=$((PCR_CAPTURE_COUNT + 1))
        indices=$(echo "$response" | grep -o '"indices_processed":[0-9]*' | cut -d':' -f2)
        echo -e "${GREEN}✅ PCR Calculation: ${indices} indices processed (total: ${PCR_CAPTURE_COUNT})${NC}"
        return 0
    else
        echo -e "${RED}❌ PCR calculation failed${NC}"
        return 1
    fi
}

# Function to update stock snapshots (runs every 3 minutes during market hours)
update_stock_snapshots() {
    echo -e "${BLUE}[$(date '+%Y-%m-%d %H:%M:%S')] Updating stock snapshots...${NC}"
    
    response=$(call_api "$STOCK_SNAPSHOT_API_URL" "Stock Snapshots")
    exit_code=$?
    if [ $exit_code -eq 0 ] && [ -n "$response" ]; then
        # Check if it was skipped
        message=$(echo "$response" | grep -o '"message":"[^"]*"' | cut -d'"' -f4)
        if [ -n "$message" ]; then
            echo -e "${YELLOW}⏭️  Stock Snapshots: ${message}${NC}"
            return 0
        fi
        
        # Get counts
        STOCK_SNAPSHOT_COUNT=$((STOCK_SNAPSHOT_COUNT + 1))
        stocks_updated=$(echo "$response" | grep -o '"stocks_updated":[0-9]*' | cut -d':' -f2)
        stocks_processed=$(echo "$response" | grep -o '"stocks_processed":[0-9]*' | cut -d':' -f2)
        echo -e "${GREEN}✅ Stock Snapshots: ${stocks_updated}/${stocks_processed} stocks updated (total: ${STOCK_SNAPSHOT_COUNT})${NC}"
        return 0
    else
        echo -e "${RED}❌ Stock snapshots update failed${NC}"
        return 1
    fi
}

# Function to update breakout snapshots
update_breakout_snapshots() {
    echo -e "${BLUE}📈 Updating breakout snapshots...${NC}"
    response=$(call_api "$BREAKOUT_SNAPSHOT_API_URL")
    
    if [ $? -eq 0 ] && echo "$response" | grep -q '"success":true'; then
        BREAKOUT_SNAPSHOT_COUNT=$((BREAKOUT_SNAPSHOT_COUNT + 1))
        updated=$(echo "$response" | grep -o '"updated":[0-9]*' | cut -d':' -f2)
        breakouts=$(echo "$response" | grep -o '"breakouts":[0-9]*' | cut -d':' -f2)
        breakdowns=$(echo "$response" | grep -o '"breakdowns":[0-9]*' | cut -d':' -f2)
        echo -e "${GREEN}✅ Breakout Snapshots: ${updated} stocks (${breakouts} breakouts, ${breakdowns} breakdowns) (total: ${BREAKOUT_SNAPSHOT_COUNT})${NC}"
        return 0
    else
        echo -e "${RED}❌ Breakout snapshots update failed${NC}"
        return 1
    fi
}

# Trap Ctrl+C to show summary
trap 'echo -e "\n${YELLOW}Stopping auto-capture...${NC}"; echo -e "${GREEN}Market data captures: ${MARKET_CAPTURE_COUNT}${NC}"; echo -e "${GREEN}Stock snapshot updates: ${STOCK_SNAPSHOT_COUNT}${NC}"; echo -e "${GREEN}Breakout snapshot updates: ${BREAKOUT_SNAPSHOT_COUNT}${NC}"; echo -e "${GREEN}OI data captures: ${OI_CAPTURE_COUNT}${NC}"; echo -e "${GREEN}Breakout checks: ${BREAKOUT_CHECK_COUNT}${NC}"; echo -e "${GREEN}PCR calculations: ${PCR_CAPTURE_COUNT}${NC}"; echo -e "${GREEN}Python script run: ${PYTHON_SCRIPT_RUN}${NC}"; exit 0' INT

# Track last capture times
LAST_MARKET_CAPTURE=0
LAST_OI_CAPTURE=0
LAST_BREAKOUT_CHECK=0
LAST_PCR_CAPTURE=0
LAST_STOCK_SNAPSHOT=0
LAST_BREAKOUT_SNAPSHOT=0

# Initial captures
capture_market_data
LAST_MARKET_CAPTURE=$(date +%s)

capture_oi_data
LAST_OI_CAPTURE=$(date +%s)

# Main loop
while true; do
    # Check if we've reached the end time
    if [ "$END_TIME" -gt 0 ] && [ "$(date +%s)" -ge "$END_TIME" ]; then
        echo -e "${YELLOW}Duration completed!${NC}"
        echo -e "${GREEN}Market data captures: ${MARKET_CAPTURE_COUNT}${NC}"
        echo -e "${GREEN}OI data captures: ${OI_CAPTURE_COUNT}${NC}"
        break
    fi
    
    CURRENT_TIME=$(date +%s)
    
    # Check if it's time to capture market data (every 5 minutes)
    if [ $((CURRENT_TIME - LAST_MARKET_CAPTURE)) -ge $MARKET_INTERVAL ]; then
        capture_market_data
        LAST_MARKET_CAPTURE=$(date +%s)
    fi
    
    # Check if it's time to capture OI data (every 3 minutes)
    if [ $((CURRENT_TIME - LAST_OI_CAPTURE)) -ge $OI_INTERVAL ]; then
        capture_oi_data
        LAST_OI_CAPTURE=$(date +%s)
    fi
    
    # Check if it's time to check breakouts/breakdowns (every 1 minute)
    if [ $((CURRENT_TIME - LAST_BREAKOUT_CHECK)) -ge $BREAKOUT_INTERVAL ]; then
        check_breakouts_breakdowns
        LAST_BREAKOUT_CHECK=$(date +%s)
    fi
    
    # Check if it's time to calculate PCR (every 1 minute)
    if [ $((CURRENT_TIME - LAST_PCR_CAPTURE)) -ge $PCR_INTERVAL ]; then
        calculate_pcr
        LAST_PCR_CAPTURE=$(date +%s)
    fi
    
    # Check if it's time to update stock snapshots (every 3 minutes)
    if [ $((CURRENT_TIME - LAST_STOCK_SNAPSHOT)) -ge $STOCK_SNAPSHOT_INTERVAL ]; then
        update_stock_snapshots
        LAST_STOCK_SNAPSHOT=$(date +%s)
    fi
    
    # DISABLED: Redundant - check_breakouts_breakdowns already populates breakout_stocks/breakdown_stocks tables
    # Check if it's time to update breakout snapshots (every 5 minutes)
    # if [ $((CURRENT_TIME - LAST_BREAKOUT_SNAPSHOT)) -ge $BREAKOUT_SNAPSHOT_INTERVAL ]; then
    #     update_breakout_snapshots
    #     LAST_BREAKOUT_SNAPSHOT=$(date +%s)
    # fi
    
    
    # Wait 30 seconds before checking again (faster check for more precise timing)
    sleep 30
done