{
  "cli_version": "1.11.0",
  "generate": {
    "command": "agents-cli eval generate --dataset tests/eval/datasets/shipctl-eval.json --output artifacts/traces/shipctl-traces.json",
    "server_url": "http://127.0.0.1:18080",
    "discovered_root_agent_name": "shipctl_docs",
    "cases_populated": 3
  },
  "grade": {
    "command": "agents-cli eval grade --traces artifacts/traces/shipctl-traces.json --config tests/eval/eval_config.yaml --output artifacts/eval_results --project vertex-ai-experminent",
    "exit_code": 0,
    "qps": 15.0,
    "summary_metrics": [
      {
        "metric_name": "command_flag_accuracy",
        "num_cases_total": 3,
        "num_cases_valid": 3,
        "num_cases_error": 0,
        "mean_score": 1.0,
        "stdev_score": 0.0
      },
      {
        "metric_name": "agent_turn_count",
        "num_cases_total": 3,
        "num_cases_valid": 3,
        "num_cases_error": 0,
        "mean_score": 1.3333,
        "stdev_score": 0.5774
      }
    ],
    "case_results": [
      {
        "eval_case_id": "rollback_prod_r41",
        "command_flag_accuracy": 1.0,
        "agent_turn_count": 1.0,
        "tool_called": "read_page(page='rollback')"
      },
      {
        "eval_case_id": "create_ci_token",
        "command_flag_accuracy": 1.0,
        "agent_turn_count": 1.0,
        "tool_called": "read_page(page='tokens')"
      },
      {
        "eval_case_id": "continued_staging_rollback",
        "command_flag_accuracy": 1.0,
        "agent_turn_count": 2.0,
        "tool_called": null
      }
    ]
  },
  "failures": {
    "ambiguous_prompt_and_agent_data": {
      "eval_case_id": "broken_ambiguous_case",
      "error": "Case has both top-level 'prompt' and agent_data.turns; ambiguous."
    },
    "dropped_case_silent_exit_zero": {
      "cli_exit_code": 0,
      "warning": "Inference summary: 2/3 succeeded, 1 failed. Warning: Artifact contains only the 2 successful case(s); 1 failed case(s) were dropped.",
      "summary_metrics_with_drop": [
        {
          "metric_name": "command_flag_accuracy",
          "num_cases_total": 2,
          "num_cases_valid": 2,
          "num_cases_error": 0,
          "mean_score": 1.0,
          "stdev_score": 0.0
        }
      ]
    }
  }
}