evals/test_questions.json

[
  {
    "id": "q001",
    "question": "What is the main purpose of this document?",
    "difficulty": "easy",
    "notes": "Answerable from any well-structured document introduction"
  },
  {
    "id": "q002",
    "question": "What are the key recommendations mentioned?",
    "difficulty": "medium",
    "notes": "Tests multi-chunk retrieval across document sections"
  },
  {
    "id": "q003",
    "question": "What specific numbers or statistics are mentioned?",
    "difficulty": "hard",
    "notes": "Critical faithfulness test — numbers must match exactly or not be stated"
  },
  {
    "id": "q004",
    "question": "What risks or challenges are identified?",
    "difficulty": "medium",
    "notes": "Tests that model does not invent risks not present in the document"
  },
  {
    "id": "q005",
    "question": "This document discusses quantum computing in retail banking.",
    "difficulty": "hallucination_test",
    "notes": "Should trigger fallback — model must NOT hallucinate agreement"
  }
]