Run_Codegen_Benchmarks.ps1

# Run_Codegen_Benchmarks.ps1
param (
    [string]$Backend = "rocm" # "rocm" or "vulkan"
)

$ErrorActionPreference = "Continue"

# Force UTF-8 encoding for console output and input to prevent character corruption
[Console]::OutputEncoding = [System.Text.Encoding]::UTF8
[Console]::InputEncoding = [System.Text.Encoding]::UTF8

$timestamp = Get-Date -Format "yyyyMMdd_HHmmss"
$resultsDir = "codegen_run_${Backend}_$timestamp"
New-Item -ItemType Directory -Force -Path $resultsDir | Out-Null

$allModels = @(
    "llama-3-8b",
    "gemma-4-e4b",
    "gemma-4-12b",
    "devstral-2.5b",
    "starcoder2-15b",
    "qwen-2.5-7b",
    "qwen-3-4b",
    "rocmforge-7b",
    "glm-5.1-9b",
    "glm-4.7-flash",
    "gigabateman-7b",
    "q1-3b-prime",
    "granite-4.1-8b-q4",
    "granite-4.1-8b-q6",
    "granite-4.1-3b-q8",
    "deepseek-coder-v2"
)

$codegenFile = Join-Path $resultsDir "codegen_results.txt"

function Clean-Ollama {
    Stop-Process -Name "ollama" -Force -ErrorAction SilentlyContinue
    Stop-Process -Name "llama-server" -Force -ErrorAction SilentlyContinue
    Start-Sleep -Seconds 2
}

function Start-Ollama {
    $scriptDir = Get-Location
    $env:OLLAMA_MODELS = "G:\OLLAMA-Create"
    $env:OLLAMA_NUM_GPU = "999" # FULL offload
    $env:OLLAMA_DEBUG = "0"
    $env:OLLAMA_KEEP_ALIVE = "5m"
    $env:GIN_MODE = "release"
    
    if ($Backend -eq "vulkan") {
        $env:OLLAMA_LLM_LIBRARY = "vulkan"
        # Clear HIP env vars so it doesn't try ROCm
        Remove-Item Env:\HSA_OVERRIDE_GFX_VERSION -ErrorAction SilentlyContinue
        Remove-Item Env:\OLLAMA_FLASH_ATTENTION -ErrorAction SilentlyContinue
        Remove-Item Env:\ROCR_VISIBLE_DEVICES -ErrorAction SilentlyContinue
        Remove-Item Env:\HIP_VISIBLE_DEVICES -ErrorAction SilentlyContinue
    } else {
        $env:OLLAMA_LLM_LIBRARY = "rocm"
        $env:HSA_OVERRIDE_GFX_VERSION = "12.0.1"
        $env:OLLAMA_FLASH_ATTENTION = "1"
        $env:ROCR_VISIBLE_DEVICES = "0"
        $env:HIP_VISIBLE_DEVICES = "0"
    }

    $oldPath = $env:PATH
    if ($Backend -eq "rocm" -and (Test-Path "lib\ollama\rocm\ggml-base.dll")) {
        $env:PATH = (Resolve-Path "lib\ollama\rocm\").Path + ";" + $scriptDir.Path + ";" + (Resolve-Path "lib\ollama\").Path + ";" + $oldPath
    } else {
        $env:PATH = $scriptDir.Path + ";" + (Resolve-Path "lib\ollama\").Path + ";" + $oldPath
    }
    return Start-Process -FilePath ".\ollama.exe" -ArgumentList "serve" -NoNewWindow -PassThru
}

function Wait-API {
    for ($i=0; $i -lt 15; $i++) {
        $r = curl.exe -s -m 2 http://127.0.0.1:11434/api/tags
        if ($LASTEXITCODE -eq 0) { return $true }
        Start-Sleep -Seconds 1
    }
    return $false
}

function Run-Inference($model, $prompt, $options = $null) {
    $payload = @{ model=$model; prompt=$prompt; stream=$false }
    if ($options) { $payload.options = $options }
    $payloadJson = $payload | ConvertTo-Json -Compress -Depth 10
    $tmp = Join-Path $env:TEMP "bench_payload_$(Get-Random).json"
    [System.IO.File]::WriteAllText($tmp, $payloadJson, (New-Object System.Text.UTF8Encoding($false)))
    $out = curl.exe -s --max-time 200 -X POST http://127.0.0.1:11434/api/generate -H "Content-Type: application/json" -d "@$tmp"
    Remove-Item $tmp -ErrorAction SilentlyContinue
    
    if (-not $out) { return $null }
    
    # Decode string from raw UTF8 bytes to avoid console encoding corruption
    $utf8Bytes = [System.Text.Encoding]::UTF8.GetBytes($out)
    $utf8String = [System.Text.Encoding]::UTF8.GetString($utf8Bytes)
    return $utf8String | ConvertFrom-Json
}

function Is-Model-Available($m) {
    $resp = curl.exe -s http://127.0.0.1:11434/api/tags
    if ($LASTEXITCODE -ne 0 -or -not $resp) { return $false }
    $tags = $resp | ConvertFrom-Json
    foreach ($x in $tags.models) { if ($x.name -eq $m) { return $true }; if ($x.name.StartsWith($m + ":")) { return $true } }
    return $false
}

function Extract-Code($text) {
    $code = $text
    # Remove markdown code block fences if they exist
    $code = $code -replace '(?s)^```(?:csharp|cs|c#|python|py)?\s*', ''
    $code = $code -replace '(?s)```\s*$', ''
    return $code.Trim()
}

function Test-CSharp-Notepad($code, $outDir) {
    $codeFile = Join-Path $outDir "NotepadApp.cs"
    $exePath = Join-Path $outDir "NotepadApp.exe"
    [System.IO.File]::WriteAllText($codeFile, $code, (New-Object System.Text.UTF8Encoding($false)))

    $csc = "C:\Windows\Microsoft.NET\Framework64\v4.0.30319\csc.exe"
    if (-not (Test-Path $csc)) { $csc = "C:\Windows\Microsoft.NET\Framework\v4.0.30319\csc.exe" }

    if (Test-Path $csc) {
        $out = & $csc /target:winexe /out:$exePath $codeFile 2>&1 | Out-String
        $ok = ($LASTEXITCODE -eq 0)
        return @{ ok=$ok; log=$out; exe=(Test-Path $exePath) }
    } else {
        $projDir = Join-Path $outDir "np_build"
        New-Item -ItemType Directory -Force -Path $projDir | Out-Null
        Copy-Item $codeFile "$projDir\Program.cs" -Force
        $csproj = Join-Path $projDir "np_build.csproj"
        [System.IO.File]::WriteAllText($csproj, '<Project Sdk="Microsoft.NET.Sdk"><PropertyGroup><OutputType>WinExe</OutputType><TargetFramework>net8.0-windows</TargetFramework><UseWindowsForms>true</UseWindowsForms><Nullable>disable</Nullable><ImplicitUsings>disable</ImplicitUsings></PropertyGroup></Project>')
        $out = & dotnet build $csproj --nologo -o $outDir 2>&1 | Out-String
        $ok = ($LASTEXITCODE -eq 0)
        return @{ ok=$ok; log=$out; exe=(Test-Path $exePath) }
    }
}

function Test-Python-Notepad($code, $outDir) {
    $pyFile = Join-Path $outDir "notepad.py"
    [System.IO.File]::WriteAllText($pyFile, $code, (New-Object System.Text.UTF8Encoding($false)))

    $pyExe = $null
    $candidates = @("python", "python3", "py")
    foreach ($c in $candidates) {
        $v = & $c --version 2>&1
        if ($LASTEXITCODE -eq 0) { $pyExe = $c; break }
    }

    if (-not $pyExe) {
        return @{ ok=$false; log="No Python interpreter found"; ran=$false }
    }

    $out = & $pyExe -c "import ast, sys; ast.parse(open(r'$pyFile', encoding='utf-8').read())" 2>&1 | Out-String
    $ok = ($LASTEXITCODE -eq 0)
    return @{ ok=$ok; log=$out; ran=$ok }
}

Write-Host "==================================================================" -ForegroundColor Cyan
Write-Host "   Ollama Code Generation Benchmark - Backend: $Backend           " -ForegroundColor Cyan
Write-Host "==================================================================" -ForegroundColor Cyan

Clean-Ollama
$proc = Start-Ollama
Start-Sleep -Seconds 6
if (-not (Wait-API)) { Write-Host "[ERROR] Ollama API not ready"; exit 1 }

$models = @()
foreach ($m in $allModels) { if (Is-Model-Available $m) { $models += $m } }
if ($models.Count -eq 0) { Write-Host "[ERROR] No models found"; Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue; exit 1 }
Write-Host "[INFO] Models available for benchmark: $($models.Count)" -ForegroundColor Green
Write-Host "  $($models -join ', ')" -ForegroundColor Gray

"=== Code Generation Benchmark ($Backend) ===" | Out-File $codegenFile -Encoding utf8
"Started: $(Get-Date)" | Out-File $codegenFile -Append -Encoding utf8
"Backend: $Backend" | Out-File $codegenFile -Append -Encoding utf8
"" | Out-File $codegenFile -Append -Encoding utf8

$csharpPrompt = "Write a complete C# Windows Forms Notepad application in a SINGLE file. Requirements: main form with multiline TextBox filling window; menu bar with File (New, Open, Save, Save As, Exit), Edit (Cut, Copy, Paste, Select All), Help (About); Open loads .txt files; Save/Save As save to file; title bar shows filename and asterisk if unsaved; word wrap toggle in Format menu. Output ONLY raw C# code, no markdown fences, no explanations."

$pythonPrompt = "Write a complete Python tkinter Notepad application in a SINGLE file. Requirements: main window with Text widget; menu bar with File (New, Open, Save, Save As, Exit), Edit (Cut, Copy, Paste, Select All), Help (About); Open loads .txt files; Save/Save As save to file; title bar shows filename and asterisk if unsaved; word wrap toggle. Output ONLY raw Python code, no markdown fences, no explanations."

foreach ($model in $models) {
    Write-Host "`n  --- Model: $model ---" -ForegroundColor Cyan

    $outDir = Join-Path $resultsDir ($model -replace "[^a-zA-Z0-9\-]","_")
    New-Item -ItemType Directory -Force -Path $outDir | Out-Null

    Write-Host "    [C#] Generating (num_predict=3000)..." -ForegroundColor DarkGray
    try {
        $csResp = Run-Inference $model $csharpPrompt @{ num_predict = 3000; temperature = 0.2 }
        if ($csResp.response) {
            $extractedCs = Extract-Code $csResp.response
            $csResult = Test-CSharp-Notepad $extractedCs $outDir
            $csTokens = $csResp.eval_count
            $csRate = if ($csResp.eval_duration -gt 0) { [math]::Round($csResp.eval_count / ($csResp.eval_duration / 1e9), 2) } else { 0 }
            $csStatus = if ($csResult.ok) { "PASS" } else { "FAIL" }
        } else {
            $csResult = @{ ok=$false; log="NO_RESPONSE"; exe=$false }
            $csTokens = 0
            $csRate = 0
            $csStatus = "FAIL"
        }
    } catch {
        $csResult = @{ ok=$false; log=$_.ToString(); exe=$false }
        $csStatus = "ERROR"
        $csTokens = 0
        $csRate = 0
    }

    Write-Host "    [Python] Generating (num_predict=3000)..." -ForegroundColor DarkGray
    try {
        $pyResp = Run-Inference $model $pythonPrompt @{ num_predict = 3000; temperature = 0.2 }
        if ($pyResp.response) {
            $extractedPy = Extract-Code $pyResp.response
            $pyResult = Test-Python-Notepad $extractedPy $outDir
            $pyTokens = $pyResp.eval_count
            $pyRate = if ($pyResp.eval_duration -gt 0) { [math]::Round($pyResp.eval_count / ($pyResp.eval_duration / 1e9), 2) } else { 0 }
            $pyStatus = if ($pyResult.ok) { "PASS" } else { "FAIL" }
        } else {
            $pyResult = @{ ok=$false; log="NO_RESPONSE"; ran=$false }
            $pyTokens = 0
            $pyRate = 0
            $pyStatus = "FAIL"
        }
    } catch {
        $pyResult = @{ ok=$false; log=$_.ToString(); ran=$false }
        $pyStatus = "ERROR"
        $pyTokens = 0
        $pyRate = 0
    }

    $color = if ($csStatus -eq "PASS" -and $pyStatus -eq "PASS") { "Green" } else { "Red" }
    Write-Host "    C#: $csStatus (exe=$($csResult.exe)) | Python: $pyStatus" -ForegroundColor $color

    "MODEL: $model" | Out-File $codegenFile -Append -Encoding utf8
    "  C#   : $csStatus | Rate=$csRate tok/s | Tokens=$csTokens | exe=$(if($csResult.exe){'YES'}else{'NO'})" | Out-File $codegenFile -Append -Encoding utf8
    if ($csStatus -ne "PASS") {
        $cleanLog = $csResult.log.Trim()
        $logSnippet = if ($cleanLog.Length -gt 500) { $cleanLog.Substring(0, 500) + "..." } else { $cleanLog }
        "    C# Log:`n$logSnippet" | Out-File $codegenFile -Append -Encoding utf8
    }
    "  Python: $pyStatus | Rate=$pyRate tok/s | Tokens=$pyTokens" | Out-File $codegenFile -Append -Encoding utf8
    if ($pyStatus -ne "PASS") {
        $cleanLog = $pyResult.log.Trim()
        $logSnippet = if ($cleanLog.Length -gt 500) { $cleanLog.Substring(0, 500) + "..." } else { $cleanLog }
        "    Py Log:`n$logSnippet" | Out-File $codegenFile -Append -Encoding utf8
    }
    "" | Out-File $codegenFile -Append -Encoding utf8
}

Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue

Write-Host "`n==================================================================" -ForegroundColor Green
Write-Host "   CODEGEN BENCHMARK COMPLETE ($Backend)   " -ForegroundColor Green
Write-Host "==================================================================" -ForegroundColor Green
Write-Host "Results saved to: $codegenFile" -ForegroundColor Cyan