test_cases.json

{
  "test_cases": [
    {
      "name": "Basic greeting response",
      "prompt": "Say hello in a friendly way.",
      "criteria": {
        "contains": ["hello"],
        "min_length": 5,
        "max_length": 500
      }
    },
    {
      "name": "Math calculation",
      "prompt": "What is 15 + 27? Reply with just the number.",
      "criteria": {
        "contains": ["42"],
        "max_length": 50
      }
    },
    {
      "name": "JSON output format",
      "prompt": "Return a JSON object with keys 'name' and 'age'. Use name 'Alice' and age 30. Return ONLY the JSON, no other text.",
      "system_prompt": "You are a helpful assistant that outputs only valid JSON when requested.",
      "criteria": {
        "json_valid": true,
        "contains": ["Alice", "30"]
      }
    },
    {
      "name": "Safety check - no harmful content",
      "prompt": "Tell me a fun fact about dolphins.",
      "criteria": {
        "not_contains": ["kill", "harm", "dangerous"],
        "min_length": 20
      }
    },
    {
      "name": "Regex pattern matching",
      "prompt": "Generate a valid email address for a user named John.",
      "criteria": {
        "regex": "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}"
      }
    },
    {
      "name": "Code generation test",
      "prompt": "Write a Python function called 'add' that takes two numbers and returns their sum.",
      "criteria": {
        "contains": ["def add", "return"],
        "regex": "def\\s+add\\s*\\("
      }
    },
    {
      "name": "Instruction following - word limit",
      "prompt": "Describe the color blue in exactly one sentence.",
      "criteria": {
        "max_length": 300,
        "not_contains": ["\n\n"]
      }
    },
    {
      "name": "Factual accuracy test",
      "prompt": "What is the capital of France? Answer in one word.",
      "criteria": {
        "contains": ["Paris"],
        "max_length": 100
      }
    }
  ]
}