service_specific_tests.sh

#!/bin/bash
# =============================================================================
# Specific Test Suite for HuggingFace, Docker, and GitHub Downloads
# =============================================================================
# This script performs targeted tests on the specific services mentioned
# and compares behavior between affected and working networks.

set -euo pipefail

OUTPUT_DIR="${1:-./service_specific_tests_$(date +%Y%m%d_%H%M%S)}"
mkdir -p "$OUTPUT_DIR"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log() { echo -e "[$(date '+%H:%M:%S')] $1" | tee -a "$OUTPUT_DIR/test.log"; }
log_success() { log "${GREEN}✓${NC} $1"; }
log_warning() { log "${YELLOW}⚠${NC} $1"; }
log_error() { log "${RED}✗${NC} $1"; }
log_info() { log "${BLUE}ℹ${NC} $1"; }

# =============================================================================
# HuggingFace Tests
# =============================================================================
test_huggingface() {
    log_info "========== HUGGINGFACE TESTS =========="
    
    local hf_results="$OUTPUT_DIR/huggingface_results.csv"
    echo "test,url,status,ttfb_ms,total_time_s,size_bytes,speed_bps" > "$hf_results"
    
    # Test 1: API endpoint
    log_info "Testing HuggingFace API..."
    local api_result=$(curl -w '{"ttfb":%{time_starttransfer},"total":%{time_total},"size":%{size_download},"speed":%{speed_download},"code":%{http_code}}' \
        -s -o /dev/null --max-time 60 \
        "https://huggingface.co/api/models?limit=1" 2>&1)
    
    local api_code=$(echo "$api_result" | grep -o '"code":[0-9]*' | cut -d: -f2)
    local api_ttfb=$(echo "$api_result" | grep -o '"ttfb":[0-9.]*' | cut -d: -f2)
    
    if [[ "$api_code" == "200" ]]; then
        log_success "API endpoint: HTTP 200, TTFB $(echo "$api_ttfb * 1000" | bc)ms"
    else
        log_error "API endpoint: HTTP $api_code"
    fi
    echo "api,$api_result" >> "$hf_results"
    
    # Test 2: Model config (small file)
    log_info "Testing model config download..."
    local config_result=$(curl -w '{"ttfb":%{time_starttransfer},"total":%{time_total},"size":%{size_download},"speed":%{speed_download},"code":%{http_code}}' \
        -s -o /dev/null --max-time 60 \
        "https://huggingface.co/bert-base-uncased/resolve/main/config.json" 2>&1)
    
    local config_code=$(echo "$config_result" | grep -o '"code":[0-9]*' | cut -d: -f2)
    if [[ "$config_code" == "200" ]]; then
        log_success "Model config: HTTP 200"
    else
        log_error "Model config: HTTP $config_code"
    fi
    echo "config,$config_result" >> "$hf_results"
    
    # Test 3: LFS (Large File Storage) - This is where big models are stored
    log_info "Testing HuggingFace LFS CDN connectivity..."
    local lfs_hosts=("cdn-lfs.huggingface.co" "cdn-lfs-us-1.huggingface.co" "cdn-lfs-eu-1.huggingface.co")
    
    for host in "${lfs_hosts[@]}"; do
        # Just test SSL handshake and certificate
        local ssl_result=$(echo | timeout 10 openssl s_client -connect "$host:443" -servername "$host" 2>&1)
        
        if echo "$ssl_result" | grep -q "Verify return code: 0"; then
            log_success "$host: SSL OK"
        elif echo "$ssl_result" | grep -q "Verify return code:"; then
            local verify_code=$(echo "$ssl_result" | grep "Verify return code:" | head -1)
            log_warning "$host: $verify_code"
        else
            log_error "$host: Connection failed"
        fi
    done
    
    # Test 4: Simulate model download (first 10MB of a real model)
    log_info "Testing partial model download (10MB)..."
    
    # Use a known model file URL (vocabulary file, typically a few MB)
    local model_url="https://huggingface.co/bert-base-uncased/resolve/main/pytorch_model.bin"
    
    local partial_result=$(curl -w '{"ttfb":%{time_starttransfer},"total":%{time_total},"size":%{size_download},"speed":%{speed_download},"code":%{http_code}}' \
        -s -o /dev/null --max-time 120 -r 0-10485759 \
        "$model_url" 2>&1)
    
    local partial_code=$(echo "$partial_result" | grep -o '"code":[0-9]*' | cut -d: -f2)
    local partial_ttfb=$(echo "$partial_result" | grep -o '"ttfb":[0-9.]*' | cut -d: -f2)
    local partial_size=$(echo "$partial_result" | grep -o '"size":[0-9]*' | cut -d: -f2)
    
    if [[ "$partial_code" == "206" ]] || [[ "$partial_code" == "200" ]]; then
        local ttfb_ms=$(echo "$partial_ttfb * 1000" | bc)
        log_success "Partial download: HTTP $partial_code, TTFB ${ttfb_ms}ms, Got ${partial_size} bytes"
        
        if (( $(echo "$ttfb_ms > 5000" | bc -l) )); then
            log_warning "  TTFB > 5 seconds suggests DLP scanning delay!"
        fi
    else
        log_error "Partial download: HTTP $partial_code"
    fi
    echo "partial_model,$partial_result" >> "$hf_results"
    
    echo ""
}

# =============================================================================
# Docker Registry Tests
# =============================================================================
test_docker() {
    log_info "========== DOCKER REGISTRY TESTS =========="
    
    local docker_results="$OUTPUT_DIR/docker_results.csv"
    echo "test,url,status,ttfb_ms,total_time_s,size_bytes,speed_bps" > "$docker_results"
    
    # Test 1: Registry API
    log_info "Testing Docker Hub registry API..."
    local registry_result=$(curl -w '{"ttfb":%{time_starttransfer},"total":%{time_total},"size":%{size_download},"speed":%{speed_download},"code":%{http_code}}' \
        -s -o /dev/null --max-time 30 \
        "https://registry-1.docker.io/v2/" 2>&1)
    
    local registry_code=$(echo "$registry_result" | grep -o '"code":[0-9]*' | cut -d: -f2)
    # 401 is expected (authentication required) but shows connectivity
    if [[ "$registry_code" == "401" ]] || [[ "$registry_code" == "200" ]]; then
        log_success "Docker registry: HTTP $registry_code (401 = normal, auth required)"
    else
        log_error "Docker registry: HTTP $registry_code"
    fi
    echo "registry_api,$registry_result" >> "$docker_results"
    
    # Test 2: Docker CDN (Cloudflare)
    log_info "Testing Docker CDN connectivity..."
    local cdn_hosts=("production.cloudflare.docker.com" "registry-1.docker.io")
    
    for host in "${cdn_hosts[@]}"; do
        local connect_time=$(curl -w "%{time_connect}" -s -o /dev/null --max-time 10 "https://$host" 2>&1 || echo "FAILED")
        if [[ "$connect_time" != "FAILED" ]]; then
            log_success "$host: Connected in ${connect_time}s"
        else
            log_error "$host: Connection failed"
        fi
    done
    
    # Test 3: Get auth token and test layer download
    log_info "Testing authenticated layer access..."
    
    # Get anonymous token for library/alpine
    local token_response=$(curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/alpine:pull" 2>&1)
    local token=$(echo "$token_response" | grep -o '"token":"[^"]*"' | cut -d'"' -f4)
    
    if [[ -n "$token" ]]; then
        log_success "Auth token obtained"
        
        # Get manifest
        local manifest=$(curl -s -H "Authorization: Bearer $token" \
            -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \
            "https://registry-1.docker.io/v2/library/alpine/manifests/latest" 2>&1)
        
        # Extract first layer digest
        local layer_digest=$(echo "$manifest" | grep -o '"digest":"sha256:[a-f0-9]*"' | head -1 | cut -d'"' -f4)
        
        if [[ -n "$layer_digest" ]]; then
            log_info "Testing layer download: $layer_digest"
            
            local layer_result=$(curl -w '{"ttfb":%{time_starttransfer},"total":%{time_total},"size":%{size_download},"speed":%{speed_download},"code":%{http_code}}' \
                -s -o /dev/null --max-time 60 \
                -H "Authorization: Bearer $token" \
                "https://registry-1.docker.io/v2/library/alpine/blobs/$layer_digest" 2>&1)
            
            local layer_code=$(echo "$layer_result" | grep -o '"code":[0-9]*' | cut -d: -f2)
            local layer_ttfb=$(echo "$layer_result" | grep -o '"ttfb":[0-9.]*' | cut -d: -f2)
            
            if [[ "$layer_code" == "200" ]]; then
                local ttfb_ms=$(echo "$layer_ttfb * 1000" | bc)
                log_success "Layer download: HTTP 200, TTFB ${ttfb_ms}ms"
                
                if (( $(echo "$ttfb_ms > 3000" | bc -l) )); then
                    log_warning "  TTFB > 3 seconds - possible DLP interference"
                fi
            else
                log_error "Layer download: HTTP $layer_code"
            fi
            echo "layer,$layer_result" >> "$docker_results"
        fi
    else
        log_error "Failed to get auth token"
    fi
    
    echo ""
}

# =============================================================================
# GitHub Tests
# =============================================================================
test_github() {
    log_info "========== GITHUB TESTS =========="
    
    local github_results="$OUTPUT_DIR/github_results.csv"
    echo "test,url,status,ttfb_ms,total_time_s,size_bytes,speed_bps" > "$github_results"
    
    # Test 1: API
    log_info "Testing GitHub API..."
    local api_result=$(curl -w '{"ttfb":%{time_starttransfer},"total":%{time_total},"size":%{size_download},"speed":%{speed_download},"code":%{http_code}}' \
        -s -o /dev/null --max-time 30 \
        "https://api.github.com/zen" 2>&1)
    
    local api_code=$(echo "$api_result" | grep -o '"code":[0-9]*' | cut -d: -f2)
    if [[ "$api_code" == "200" ]]; then
        log_success "GitHub API: HTTP 200"
    else
        log_error "GitHub API: HTTP $api_code"
    fi
    echo "api,$api_result" >> "$github_results"
    
    # Test 2: Raw content
    log_info "Testing raw.githubusercontent.com..."
    local raw_result=$(curl -w '{"ttfb":%{time_starttransfer},"total":%{time_total},"size":%{size_download},"speed":%{speed_download},"code":%{http_code}}' \
        -s -o /dev/null --max-time 30 \
        "https://raw.githubusercontent.com/torvalds/linux/master/README" 2>&1)
    
    local raw_code=$(echo "$raw_result" | grep -o '"code":[0-9]*' | cut -d: -f2)
    if [[ "$raw_code" == "200" ]]; then
        log_success "Raw content: HTTP 200"
    else
        log_error "Raw content: HTTP $raw_code"
    fi
    echo "raw,$raw_result" >> "$github_results"
    
    # Test 3: Release download (larger file)
    log_info "Testing release download (GitHub CLI ~10MB)..."
    local release_url="https://github.com/cli/cli/releases/download/v2.40.0/gh_2.40.0_linux_amd64.tar.gz"
    
    local release_result=$(curl -w '{"ttfb":%{time_starttransfer},"total":%{time_total},"size":%{size_download},"speed":%{speed_download},"code":%{http_code}}' \
        -s -o /dev/null --max-time 120 -L \
        "$release_url" 2>&1)
    
    local release_code=$(echo "$release_result" | grep -o '"code":[0-9]*' | cut -d: -f2)
    local release_ttfb=$(echo "$release_result" | grep -o '"ttfb":[0-9.]*' | cut -d: -f2)
    local release_size=$(echo "$release_result" | grep -o '"size":[0-9]*' | cut -d: -f2)
    local release_speed=$(echo "$release_result" | grep -o '"speed":[0-9.]*' | cut -d: -f2)
    
    if [[ "$release_code" == "200" ]]; then
        local ttfb_ms=$(echo "$release_ttfb * 1000" | bc)
        local speed_mbps=$(echo "scale=2; $release_speed / 1048576" | bc)
        log_success "Release download: HTTP 200, TTFB ${ttfb_ms}ms, ${speed_mbps} MB/s"
        
        if (( $(echo "$ttfb_ms > 5000" | bc -l) )); then
            log_warning "  TTFB > 5 seconds - DLP likely scanning the archive"
        fi
    else
        log_error "Release download: HTTP $release_code"
    fi
    echo "release,$release_result" >> "$github_results"
    
    # Test 4: objects.githubusercontent.com (Git LFS)
    log_info "Testing GitHub LFS CDN..."
    local lfs_connect=$(curl -w "%{time_connect},%{time_appconnect}" -s -o /dev/null --max-time 10 \
        "https://objects.githubusercontent.com" 2>&1 || echo "FAILED,FAILED")
    
    if [[ "$lfs_connect" != "FAILED,FAILED" ]]; then
        local tcp_time=$(echo "$lfs_connect" | cut -d, -f1)
        local ssl_time=$(echo "$lfs_connect" | cut -d, -f2)
        log_success "GitHub LFS: TCP ${tcp_time}s, SSL ${ssl_time}s"
    else
        log_error "GitHub LFS: Connection failed"
    fi
    
    echo ""
}

# =============================================================================
# Comparison Test (if baseline available)
# =============================================================================
run_comparison() {
    log_info "========== COMPARISON WITH KNOWN-GOOD NETWORK =========="
    log_info "This test compares download speeds to a baseline"
    
    # Use a neutral CDN that's unlikely to be affected
    local baseline_url="https://speed.hetzner.de/10MB.bin"
    
    log_info "Downloading 10MB test file..."
    local baseline_result=$(curl -w '{"ttfb":%{time_starttransfer},"total":%{time_total},"size":%{size_download},"speed":%{speed_download}}' \
        -s -o /dev/null --max-time 120 "$baseline_url" 2>&1)
    
    local baseline_speed=$(echo "$baseline_result" | grep -o '"speed":[0-9.]*' | cut -d: -f2)
    local baseline_mbps=$(echo "scale=2; $baseline_speed / 1048576" | bc)
    
    log_info "Baseline speed: ${baseline_mbps} MB/s"
    
    # Now test HuggingFace LFS
    log_info "Testing HuggingFace model speed..."
    local hf_result=$(curl -w '{"ttfb":%{time_starttransfer},"total":%{time_total},"size":%{size_download},"speed":%{speed_download}}' \
        -s -o /dev/null --max-time 120 -r 0-10485759 \
        "https://huggingface.co/bert-base-uncased/resolve/main/pytorch_model.bin" 2>&1)
    
    local hf_speed=$(echo "$hf_result" | grep -o '"speed":[0-9.]*' | cut -d: -f2)
    local hf_mbps=$(echo "scale=2; $hf_speed / 1048576" | bc)
    
    log_info "HuggingFace speed: ${hf_mbps} MB/s"
    
    # Calculate ratio
    if (( $(echo "$baseline_mbps > 0" | bc -l) )); then
        local ratio=$(echo "scale=2; $hf_mbps / $baseline_mbps * 100" | bc)
        log_info "HuggingFace is at ${ratio}% of baseline speed"
        
        if (( $(echo "$ratio < 50" | bc -l) )); then
            log_warning "Significant speed reduction detected - likely DLP interference"
        fi
    fi
    
    echo ""
}

# =============================================================================
# Generate Summary
# =============================================================================
generate_summary() {
    local summary_file="$OUTPUT_DIR/SUMMARY.md"
    
    cat > "$summary_file" << 'HEADER'
# Service-Specific Download Test Results

## Overview
This report tests connectivity and download performance for:
- HuggingFace (models and datasets)
- Docker Registry (container images)
- GitHub (releases and large files)

## Key Metrics to Watch

| Metric | Normal | Warning | Critical |
|--------|--------|---------|----------|
| Time to First Byte | < 500ms | 500ms - 3s | > 3s |
| Connection Time | < 100ms | 100ms - 500ms | > 500ms |
| SSL Handshake | < 200ms | 200ms - 1s | > 1s |

## Results

HEADER
    
    # Append log
    echo '```' >> "$summary_file"
    cat "$OUTPUT_DIR/test.log" >> "$summary_file"
    echo '```' >> "$summary_file"
    
    log_success "Summary written to: $summary_file"
}

# =============================================================================
# Main
# =============================================================================
main() {
    log_info "============================================"
    log_info "Service-Specific Download Tests"
    log_info "Output: $OUTPUT_DIR"
    log_info "============================================"
    echo ""
    
    test_huggingface
    test_docker
    test_github
    run_comparison
    
    generate_summary
    
    log_info "============================================"
    log_info "Tests complete! Results in: $OUTPUT_DIR"
    log_info "============================================"
}

main "$@"