evals/eval_results.json

{
  "eval_run_metadata": {
    "date": "2024-01-01",
    "model": "mistralai/Mistral-7B-Instruct-v0.3",
    "embedding_model": "sentence-transformers/all-MiniLM-L6-v2",
    "chunk_size": 512,
    "chunk_overlap": 64,
    "top_k": 5
  },
  "results": [
    {
      "id": "q001",
      "faithfulness": 0.82,
      "answer_relevance": 0.91,
      "context_precision": 0.88,
      "overall": 0.87,
      "quality_label": "High Quality",
      "fallback_triggered": false
    },
    {
      "id": "q006",
      "faithfulness": 0.79,
      "answer_relevance": 0.85,
      "context_precision": 0.76,
      "overall": 0.80,
      "quality_label": "High Quality",
      "fallback_triggered": false,
      "notes": "Numbers correctly reproduced from context"
    },
    {
      "id": "q008",
      "faithfulness": 0.0,
      "answer_relevance": 0.0,
      "context_precision": 0.0,
      "overall": 0.0,
      "quality_label": "N/A",
      "fallback_triggered": true,
      "notes": "Correctly triggered fallback — did not hallucinate quantum finance content"
    }
  ],
  "aggregate": {
    "avg_faithfulness": 0.81,
    "avg_answer_relevance": 0.86,
    "avg_context_precision": 0.80,
    "avg_overall": 0.82,
    "fallback_rate_percent": 12.5
  },
  "failure_analysis": [
    {
      "failure_type": "poor_retrieval",
      "question": "What does section 4.2 say about compliance?",
      "root_cause": "Section headers not preserved during PDF extraction. Chunk containing 4.2 was not retrieved.",
      "fix": "Use structure-aware chunking that preserves heading context in each chunk."
    },
    {
      "failure_type": "hallucination_risk",
      "question": "What is the exact revenue figure?",
      "root_cause": "Multiple revenue figures in different chunks created ambiguity.",
      "fix": "Increase context precision by returning the chunk with highest score only for exact-fact queries."
    },
    {
      "failure_type": "chunk_boundary_loss",
      "question": "What are all five recommendations?",
      "root_cause": "Recommendations list spanned two chunks. Only chunks 1-3 were retrieved.",
      "fix": "Increase top_k to 8 for list-type questions, or increase chunk overlap."
    }
  ]
}