{
  "cli_version": "1.11.0",
  "baseline": {
    "command_flag_accuracy": {
      "mean_score": 0.6667,
      "stdev_score": 0.5774,
      "num_cases_total": 3,
      "num_cases_valid": 3,
      "num_cases_error": 0
    },
    "multi_turn_task_success": {
      "mean_score": 3.6667,
      "stdev_score": 1.1547,
      "num_cases_total": 3,
      "num_cases_valid": 3,
      "num_cases_error": 0
    },
    "multi_turn_tool_use_quality": {
      "mean_score": 0.6667,
      "stdev_score": 0.5774,
      "num_cases_total": 3,
      "num_cases_valid": 3,
      "num_cases_error": 0
    },
    "agent_turn_count": {
      "mean_score": 2.3333,
      "stdev_score": 0.5774,
      "num_cases_total": 3,
      "num_cases_valid": 3,
      "num_cases_error": 0
    }
  },
  "candidate": {
    "command_flag_accuracy": {
      "mean_score": 1.0,
      "stdev_score": 0.0,
      "num_cases_total": 3,
      "num_cases_valid": 3,
      "num_cases_error": 0
    },
    "multi_turn_task_success": {
      "mean_score": 4.6667,
      "stdev_score": 0.5774,
      "num_cases_total": 3,
      "num_cases_valid": 3,
      "num_cases_error": 0
    },
    "multi_turn_tool_use_quality": {
      "mean_score": 1.0,
      "stdev_score": 0.0,
      "num_cases_total": 3,
      "num_cases_valid": 3,
      "num_cases_error": 0
    },
    "agent_turn_count": {
      "mean_score": 2.0,
      "stdev_score": 0.0,
      "num_cases_total": 3,
      "num_cases_valid": 3,
      "num_cases_error": 0
    }
  },
  "compare_output": {
    "command": "agents-cli eval compare artifacts/eval_results/baseline.json artifacts/eval_results/candidate.json",
    "exit_code": 0,
    "differences": {
      "agent_turn_count.mean_score": {
        "baseline": 2.3333,
        "candidate": 2.0,
        "delta": "-0.3333"
      },
      "agent_turn_count.stdev_score": {
        "baseline": 0.5774,
        "candidate": 0.0,
        "delta": "-0.5774"
      },
      "command_flag_accuracy.mean_score": {
        "baseline": 0.6667,
        "candidate": 1.0,
        "delta": "+0.3333"
      },
      "command_flag_accuracy.stdev_score": {
        "baseline": 0.5774,
        "candidate": 0.0,
        "delta": "-0.5774"
      },
      "multi_turn_task_success.mean_score": {
        "baseline": 3.6667,
        "candidate": 4.6667,
        "delta": "+1.0"
      },
      "multi_turn_task_success.stdev_score": {
        "baseline": 1.1547,
        "candidate": 0.5774,
        "delta": "-0.5773"
      },
      "multi_turn_tool_use_quality.mean_score": {
        "baseline": 0.6667,
        "candidate": 1.0,
        "delta": "+0.3333"
      },
      "multi_turn_tool_use_quality.stdev_score": {
        "baseline": 0.5774,
        "candidate": 0.0,
        "delta": "-0.5774"
      }
    }
  },
  "loss_cluster": {
    "command": "agents-cli eval analyze --eval-result artifacts/eval_results/baseline_full.json --metric multi_turn_tool_use_quality --top-k 2",
    "top_cluster": "Redundant read_page calls across unrelated pages when a sub-question is not covered in the target page",
    "fix_applied": "Updated INSTRUCTION to stop after reading the relevant topic page and state when a sub-question is undocumented instead of scanning all pages."
  },
  "failures": {
    "unsupported_analyze_metric": {
      "metric": "command_flag_accuracy",
      "error": "Unsupported metric 'command_flag_accuracy' for 'agents-cli eval analyze'. Currently supported: multi_turn_task_success, multi_turn_tool_use_quality."
    },
    "regressed_candidate": {
      "differences": {
        "multi_turn_tool_use_quality.mean_score": {
          "baseline": 0.6667,
          "candidate": 1.0,
          "delta": "+0.3333"
        },
        "command_flag_accuracy.mean_score": {
          "baseline": 1.0,
          "candidate": 0.6667,
          "delta": "-0.3333"
        }
      },
      "error": "Regression detected: command_flag_accuracy.mean_score regressed from 1.0 to 0.6667 (-0.3333)"
    }
  }
}