{
  "execution_mode": "fresh-gemini-versus-frozen-scripted-policies",
  "model": "gemini-3.1-flash-lite",
  "heldout_sha256": "831748d6fd8906e3626b1a94c6f498ec5c7cc3e934469429b0ce676677f8291e",
  "cases": [
    {
      "case_id": "heldout-encoding",
      "question": "Which pathlib writing operation lets me set an explicit text encoding?",
      "version": "3.12",
      "target_sources": [
        "path-write"
      ]
    },
    {
      "case_id": "heldout-existing",
      "question": "If the destination already exists, does pathlib write_text append or overwrite its contents?",
      "version": "3.12",
      "target_sources": [
        "path-write"
      ]
    },
    {
      "case_id": "heldout-json",
      "question": "What Python type does JSON serialization with dumps return?",
      "version": "3.12",
      "target_sources": [
        "json-text"
      ]
    },
    {
      "case_id": "heldout-unsupported",
      "question": "How often should our team create a backup of files saved with pathlib?",
      "version": "3.12",
      "target_sources": []
    }
  ],
  "trials": [
    {
      "case_id": "heldout-encoding",
      "question": "Which pathlib writing operation lets me set an explicit text encoding?",
      "version": "3.12",
      "policy": "fixed_pipeline",
      "policy_terminal_status": "completed",
      "policy_reason": "fixed_pipeline_done",
      "runtime": {
        "controller_status": "completed",
        "terminal_reason": "policy_complete",
        "limits": {
          "max_steps": 6,
          "max_searches": 2,
          "max_reads": 2,
          "max_source_bytes": 4096
        },
        "usage": {
          "steps": 2,
          "searches": 1,
          "reads": 1,
          "reserved_source_bytes": 154,
          "delivered_source_bytes": 154
        },
        "trace": [
          {
            "action": {
              "tool": "search",
              "arguments": {
                "query": "text",
                "version": "3.12",
                "k": 2
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 0,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "query": "text",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                },
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                  "source_id": "path-write",
                  "version": "3.12",
                  "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                  "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "byte_count": 224,
                  "native_score": -1.3203883495145633e-06,
                  "rank": 2,
                  "read_token": "189bf56802f958924691f452106802696dcd22186df4ee9c03bf0efecf1e1f81"
                }
              ]
            },
            "usage_after": {
              "steps": 1,
              "searches": 1,
              "reads": 0,
              "reserved_source_bytes": 0,
              "delivered_source_bytes": 0
            }
          },
          {
            "action": {
              "tool": "read",
              "arguments": {
                "handle": {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                }
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 154,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
              "source_id": "path-read",
              "version": "3.12",
              "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
              "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "byte_count": 154,
              "native_score": -1.4220532319391636e-06,
              "rank": 1,
              "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124",
              "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
              "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
              "read_request_id": "b2ce4d1e809a197df6c864261264b583737b1f41c59f8a4b6e353911b84c5bfe",
              "result_sha256": "2d855cfa0b4f7e90c3711783b7779709f3b964760fd789d0d65b11f9b3604ac5"
            },
            "usage_after": {
              "steps": 2,
              "searches": 1,
              "reads": 1,
              "reserved_source_bytes": 154,
              "delivered_source_bytes": 154
            }
          }
        ],
        "observations": [
          {
            "tool": "search",
            "arguments": {
              "query": "text",
              "version": "3.12",
              "k": 2
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "query": "text",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                },
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                  "source_id": "path-write",
                  "version": "3.12",
                  "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                  "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "byte_count": 224,
                  "native_score": -1.3203883495145633e-06,
                  "rank": 2,
                  "read_token": "189bf56802f958924691f452106802696dcd22186df4ee9c03bf0efecf1e1f81"
                }
              ]
            }
          },
          {
            "tool": "read",
            "arguments": {
              "handle": {
                "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                "source_id": "path-read",
                "version": "3.12",
                "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                "start_line": 1,
                "end_line": 1,
                "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                "byte_count": 154,
                "native_score": -1.4220532319391636e-06,
                "rank": 1,
                "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
              }
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
              "source_id": "path-read",
              "version": "3.12",
              "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
              "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "byte_count": 154,
              "native_score": -1.4220532319391636e-06,
              "rank": 1,
              "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124",
              "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
              "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
              "read_request_id": "b2ce4d1e809a197df6c864261264b583737b1f41c59f8a4b6e353911b84c5bfe",
              "result_sha256": "2d855cfa0b4f7e90c3711783b7779709f3b964760fd789d0d65b11f9b3604ac5"
            }
          }
        ],
        "scope": "Host validates and admits bounded documentation actions; Gemini calls have a separate shared request budget."
      },
      "outcome_check": {
        "status": "fail",
        "reason": "target_passage_not_read",
        "read_passage_ids": [
          "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900"
        ],
        "target_passage_ids": [
          "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7"
        ]
      }
    },
    {
      "case_id": "heldout-encoding",
      "question": "Which pathlib writing operation lets me set an explicit text encoding?",
      "version": "3.12",
      "policy": "scripted_refinement",
      "policy_terminal_status": "incomplete",
      "policy_reason": "round_limit",
      "runtime": {
        "controller_status": "completed",
        "terminal_reason": "policy_complete",
        "limits": {
          "max_steps": 6,
          "max_searches": 2,
          "max_reads": 2,
          "max_source_bytes": 4096
        },
        "usage": {
          "steps": 4,
          "searches": 2,
          "reads": 2,
          "reserved_source_bytes": 353,
          "delivered_source_bytes": 353
        },
        "trace": [
          {
            "action": {
              "tool": "search",
              "arguments": {
                "query": "text",
                "version": "3.12",
                "k": 2
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 0,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "query": "text",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                },
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                  "source_id": "path-write",
                  "version": "3.12",
                  "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                  "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "byte_count": 224,
                  "native_score": -1.3203883495145633e-06,
                  "rank": 2,
                  "read_token": "189bf56802f958924691f452106802696dcd22186df4ee9c03bf0efecf1e1f81"
                }
              ]
            },
            "usage_after": {
              "steps": 1,
              "searches": 1,
              "reads": 0,
              "reserved_source_bytes": 0,
              "delivered_source_bytes": 0
            }
          },
          {
            "action": {
              "tool": "read",
              "arguments": {
                "handle": {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                }
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 154,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
              "source_id": "path-read",
              "version": "3.12",
              "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
              "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "byte_count": 154,
              "native_score": -1.4220532319391636e-06,
              "rank": 1,
              "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124",
              "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
              "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
              "read_request_id": "b2ce4d1e809a197df6c864261264b583737b1f41c59f8a4b6e353911b84c5bfe",
              "result_sha256": "2d855cfa0b4f7e90c3711783b7779709f3b964760fd789d0d65b11f9b3604ac5"
            },
            "usage_after": {
              "steps": 2,
              "searches": 1,
              "reads": 1,
              "reserved_source_bytes": 154,
              "delivered_source_bytes": 154
            }
          },
          {
            "action": {
              "tool": "search",
              "arguments": {
                "query": "write",
                "version": "3.12",
                "k": 2
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 0,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "ce0ea8d504d083c0a75c9ba19cfa89f88985c889f4e863067efc6a313a49094d",
              "query": "write",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "ce0ea8d504d083c0a75c9ba19cfa89f88985c889f4e863067efc6a313a49094d",
                  "passage_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
                  "source_id": "json-text",
                  "version": "3.12",
                  "document_revision": "0c4b565b2b1900787d6da9f8539c471f8cb2245c00abf887a57d460364abc57a",
                  "source_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                  "byte_count": 199,
                  "native_score": -9.765013054830289e-07,
                  "rank": 1,
                  "read_token": "d8a24ee4e5d00a42b3bd417be07632c7a329c7e2cf39a0dd3ccfdea64badaaa0"
                },
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "ce0ea8d504d083c0a75c9ba19cfa89f88985c889f4e863067efc6a313a49094d",
                  "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                  "source_id": "path-write",
                  "version": "3.12",
                  "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                  "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "byte_count": 224,
                  "native_score": -9.432534678436319e-07,
                  "rank": 2,
                  "read_token": "1bd370d49cf0504dd50e14f4a9e27a9eb6440b789a66ecb0c857d432e025510f"
                }
              ]
            },
            "usage_after": {
              "steps": 3,
              "searches": 2,
              "reads": 1,
              "reserved_source_bytes": 154,
              "delivered_source_bytes": 154
            }
          },
          {
            "action": {
              "tool": "read",
              "arguments": {
                "handle": {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "ce0ea8d504d083c0a75c9ba19cfa89f88985c889f4e863067efc6a313a49094d",
                  "passage_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
                  "source_id": "json-text",
                  "version": "3.12",
                  "document_revision": "0c4b565b2b1900787d6da9f8539c471f8cb2245c00abf887a57d460364abc57a",
                  "source_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                  "byte_count": 199,
                  "native_score": -9.765013054830289e-07,
                  "rank": 1,
                  "read_token": "d8a24ee4e5d00a42b3bd417be07632c7a329c7e2cf39a0dd3ccfdea64badaaa0"
                }
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 199,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "ce0ea8d504d083c0a75c9ba19cfa89f88985c889f4e863067efc6a313a49094d",
              "passage_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
              "source_id": "json-text",
              "version": "3.12",
              "document_revision": "0c4b565b2b1900787d6da9f8539c471f8cb2245c00abf887a57d460364abc57a",
              "source_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
              "byte_count": 199,
              "native_score": -9.765013054830289e-07,
              "rank": 1,
              "read_token": "d8a24ee4e5d00a42b3bd417be07632c7a329c7e2cf39a0dd3ccfdea64badaaa0",
              "text": "json.dumps(obj, ensure_ascii=True) returns a JSON string. Setting ensure_ascii=False leaves non-ASCII characters in that string. It does not write a file or select the encoding used by a file writer.",
              "url": "https://docs.python.org/3.12/library/json.html#json.dumps",
              "read_request_id": "9a1ee0faadc084cb2c51e02f9efb21e61aa11cc069904f9e84a8c43056a188d5",
              "result_sha256": "661fa082d4ddae14438f0f872ae166fdeddb09a04a8be1e622082e376cb25168"
            },
            "usage_after": {
              "steps": 4,
              "searches": 2,
              "reads": 2,
              "reserved_source_bytes": 353,
              "delivered_source_bytes": 353
            }
          }
        ],
        "observations": [
          {
            "tool": "search",
            "arguments": {
              "query": "text",
              "version": "3.12",
              "k": 2
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "query": "text",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                },
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                  "source_id": "path-write",
                  "version": "3.12",
                  "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                  "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "byte_count": 224,
                  "native_score": -1.3203883495145633e-06,
                  "rank": 2,
                  "read_token": "189bf56802f958924691f452106802696dcd22186df4ee9c03bf0efecf1e1f81"
                }
              ]
            }
          },
          {
            "tool": "read",
            "arguments": {
              "handle": {
                "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                "source_id": "path-read",
                "version": "3.12",
                "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                "start_line": 1,
                "end_line": 1,
                "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                "byte_count": 154,
                "native_score": -1.4220532319391636e-06,
                "rank": 1,
                "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
              }
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
              "source_id": "path-read",
              "version": "3.12",
              "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
              "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "byte_count": 154,
              "native_score": -1.4220532319391636e-06,
              "rank": 1,
              "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124",
              "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
              "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
              "read_request_id": "b2ce4d1e809a197df6c864261264b583737b1f41c59f8a4b6e353911b84c5bfe",
              "result_sha256": "2d855cfa0b4f7e90c3711783b7779709f3b964760fd789d0d65b11f9b3604ac5"
            }
          },
          {
            "tool": "search",
            "arguments": {
              "query": "write",
              "version": "3.12",
              "k": 2
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "ce0ea8d504d083c0a75c9ba19cfa89f88985c889f4e863067efc6a313a49094d",
              "query": "write",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "ce0ea8d504d083c0a75c9ba19cfa89f88985c889f4e863067efc6a313a49094d",
                  "passage_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
                  "source_id": "json-text",
                  "version": "3.12",
                  "document_revision": "0c4b565b2b1900787d6da9f8539c471f8cb2245c00abf887a57d460364abc57a",
                  "source_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                  "byte_count": 199,
                  "native_score": -9.765013054830289e-07,
                  "rank": 1,
                  "read_token": "d8a24ee4e5d00a42b3bd417be07632c7a329c7e2cf39a0dd3ccfdea64badaaa0"
                },
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "ce0ea8d504d083c0a75c9ba19cfa89f88985c889f4e863067efc6a313a49094d",
                  "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                  "source_id": "path-write",
                  "version": "3.12",
                  "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                  "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "byte_count": 224,
                  "native_score": -9.432534678436319e-07,
                  "rank": 2,
                  "read_token": "1bd370d49cf0504dd50e14f4a9e27a9eb6440b789a66ecb0c857d432e025510f"
                }
              ]
            }
          },
          {
            "tool": "read",
            "arguments": {
              "handle": {
                "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                "search_request_id": "ce0ea8d504d083c0a75c9ba19cfa89f88985c889f4e863067efc6a313a49094d",
                "passage_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
                "source_id": "json-text",
                "version": "3.12",
                "document_revision": "0c4b565b2b1900787d6da9f8539c471f8cb2245c00abf887a57d460364abc57a",
                "source_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                "start_line": 1,
                "end_line": 1,
                "text_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                "byte_count": 199,
                "native_score": -9.765013054830289e-07,
                "rank": 1,
                "read_token": "d8a24ee4e5d00a42b3bd417be07632c7a329c7e2cf39a0dd3ccfdea64badaaa0"
              }
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "ce0ea8d504d083c0a75c9ba19cfa89f88985c889f4e863067efc6a313a49094d",
              "passage_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
              "source_id": "json-text",
              "version": "3.12",
              "document_revision": "0c4b565b2b1900787d6da9f8539c471f8cb2245c00abf887a57d460364abc57a",
              "source_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
              "byte_count": 199,
              "native_score": -9.765013054830289e-07,
              "rank": 1,
              "read_token": "d8a24ee4e5d00a42b3bd417be07632c7a329c7e2cf39a0dd3ccfdea64badaaa0",
              "text": "json.dumps(obj, ensure_ascii=True) returns a JSON string. Setting ensure_ascii=False leaves non-ASCII characters in that string. It does not write a file or select the encoding used by a file writer.",
              "url": "https://docs.python.org/3.12/library/json.html#json.dumps",
              "read_request_id": "9a1ee0faadc084cb2c51e02f9efb21e61aa11cc069904f9e84a8c43056a188d5",
              "result_sha256": "661fa082d4ddae14438f0f872ae166fdeddb09a04a8be1e622082e376cb25168"
            }
          }
        ],
        "scope": "Host validates and admits bounded documentation actions; Gemini calls have a separate shared request budget."
      },
      "outcome_check": {
        "status": "incomplete",
        "reason": "policy_stopped_without_target",
        "read_passage_ids": [
          "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
          "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c"
        ],
        "target_passage_ids": [
          "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7"
        ]
      }
    },
    {
      "case_id": "heldout-encoding",
      "question": "Which pathlib writing operation lets me set an explicit text encoding?",
      "version": "3.12",
      "policy": "gemini",
      "policy_terminal_status": null,
      "policy_reason": null,
      "runtime": {
        "controller_status": "failed",
        "terminal_reason": "ValueError: invalid_search_query",
        "limits": {
          "max_steps": 6,
          "max_searches": 2,
          "max_reads": 2,
          "max_source_bytes": 4096
        },
        "usage": {
          "steps": 1,
          "searches": 1,
          "reads": 0,
          "reserved_source_bytes": 0,
          "delivered_source_bytes": 0
        },
        "trace": [
          {
            "action": {
              "tool": "search",
              "arguments": {
                "query": "pathlib",
                "version": "3.12",
                "k": 2
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 0,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "99d39479617c633ab856eaba7df76831e1ac9eed8c58608de2c59734fd6efb51",
              "query": "pathlib",
              "version": "3.12",
              "k": 2,
              "results": []
            },
            "usage_after": {
              "steps": 1,
              "searches": 1,
              "reads": 0,
              "reserved_source_bytes": 0,
              "delivered_source_bytes": 0
            }
          },
          {
            "decision": "refused",
            "reason": "ValueError: invalid_search_query",
            "invoked": false
          }
        ],
        "observations": [
          {
            "tool": "search",
            "arguments": {
              "query": "pathlib",
              "version": "3.12",
              "k": 2
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "99d39479617c633ab856eaba7df76831e1ac9eed8c58608de2c59734fd6efb51",
              "query": "pathlib",
              "version": "3.12",
              "k": 2,
              "results": []
            }
          }
        ],
        "scope": "Host validates and admits bounded documentation actions; Gemini calls have a separate shared request budget."
      },
      "outcome_check": {
        "status": "fail",
        "reason": "controller_or_tool_failure",
        "read_passage_ids": [],
        "target_passage_ids": [
          "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7"
        ]
      }
    },
    {
      "case_id": "heldout-existing",
      "question": "If the destination already exists, does pathlib write_text append or overwrite its contents?",
      "version": "3.12",
      "policy": "fixed_pipeline",
      "policy_terminal_status": "completed",
      "policy_reason": "fixed_pipeline_done",
      "runtime": {
        "controller_status": "completed",
        "terminal_reason": "policy_complete",
        "limits": {
          "max_steps": 6,
          "max_searches": 2,
          "max_reads": 2,
          "max_source_bytes": 4096
        },
        "usage": {
          "steps": 2,
          "searches": 1,
          "reads": 1,
          "reserved_source_bytes": 154,
          "delivered_source_bytes": 154
        },
        "trace": [
          {
            "action": {
              "tool": "search",
              "arguments": {
                "query": "text",
                "version": "3.12",
                "k": 2
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 0,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "query": "text",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                },
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                  "source_id": "path-write",
                  "version": "3.12",
                  "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                  "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "byte_count": 224,
                  "native_score": -1.3203883495145633e-06,
                  "rank": 2,
                  "read_token": "189bf56802f958924691f452106802696dcd22186df4ee9c03bf0efecf1e1f81"
                }
              ]
            },
            "usage_after": {
              "steps": 1,
              "searches": 1,
              "reads": 0,
              "reserved_source_bytes": 0,
              "delivered_source_bytes": 0
            }
          },
          {
            "action": {
              "tool": "read",
              "arguments": {
                "handle": {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                }
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 154,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
              "source_id": "path-read",
              "version": "3.12",
              "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
              "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "byte_count": 154,
              "native_score": -1.4220532319391636e-06,
              "rank": 1,
              "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124",
              "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
              "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
              "read_request_id": "b2ce4d1e809a197df6c864261264b583737b1f41c59f8a4b6e353911b84c5bfe",
              "result_sha256": "2d855cfa0b4f7e90c3711783b7779709f3b964760fd789d0d65b11f9b3604ac5"
            },
            "usage_after": {
              "steps": 2,
              "searches": 1,
              "reads": 1,
              "reserved_source_bytes": 154,
              "delivered_source_bytes": 154
            }
          }
        ],
        "observations": [
          {
            "tool": "search",
            "arguments": {
              "query": "text",
              "version": "3.12",
              "k": 2
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "query": "text",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                },
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                  "source_id": "path-write",
                  "version": "3.12",
                  "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                  "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "byte_count": 224,
                  "native_score": -1.3203883495145633e-06,
                  "rank": 2,
                  "read_token": "189bf56802f958924691f452106802696dcd22186df4ee9c03bf0efecf1e1f81"
                }
              ]
            }
          },
          {
            "tool": "read",
            "arguments": {
              "handle": {
                "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                "source_id": "path-read",
                "version": "3.12",
                "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                "start_line": 1,
                "end_line": 1,
                "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                "byte_count": 154,
                "native_score": -1.4220532319391636e-06,
                "rank": 1,
                "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
              }
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
              "source_id": "path-read",
              "version": "3.12",
              "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
              "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "byte_count": 154,
              "native_score": -1.4220532319391636e-06,
              "rank": 1,
              "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124",
              "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
              "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
              "read_request_id": "b2ce4d1e809a197df6c864261264b583737b1f41c59f8a4b6e353911b84c5bfe",
              "result_sha256": "2d855cfa0b4f7e90c3711783b7779709f3b964760fd789d0d65b11f9b3604ac5"
            }
          }
        ],
        "scope": "Host validates and admits bounded documentation actions; Gemini calls have a separate shared request budget."
      },
      "outcome_check": {
        "status": "fail",
        "reason": "target_passage_not_read",
        "read_passage_ids": [
          "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900"
        ],
        "target_passage_ids": [
          "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7"
        ]
      }
    },
    {
      "case_id": "heldout-existing",
      "question": "If the destination already exists, does pathlib write_text append or overwrite its contents?",
      "version": "3.12",
      "policy": "scripted_refinement",
      "policy_terminal_status": "completed",
      "policy_reason": "configured_context_marker_found",
      "runtime": {
        "controller_status": "completed",
        "terminal_reason": "policy_complete",
        "limits": {
          "max_steps": 6,
          "max_searches": 2,
          "max_reads": 2,
          "max_source_bytes": 4096
        },
        "usage": {
          "steps": 4,
          "searches": 2,
          "reads": 2,
          "reserved_source_bytes": 378,
          "delivered_source_bytes": 378
        },
        "trace": [
          {
            "action": {
              "tool": "search",
              "arguments": {
                "query": "text",
                "version": "3.12",
                "k": 2
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 0,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "query": "text",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                },
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                  "source_id": "path-write",
                  "version": "3.12",
                  "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                  "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "byte_count": 224,
                  "native_score": -1.3203883495145633e-06,
                  "rank": 2,
                  "read_token": "189bf56802f958924691f452106802696dcd22186df4ee9c03bf0efecf1e1f81"
                }
              ]
            },
            "usage_after": {
              "steps": 1,
              "searches": 1,
              "reads": 0,
              "reserved_source_bytes": 0,
              "delivered_source_bytes": 0
            }
          },
          {
            "action": {
              "tool": "read",
              "arguments": {
                "handle": {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                }
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 154,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
              "source_id": "path-read",
              "version": "3.12",
              "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
              "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "byte_count": 154,
              "native_score": -1.4220532319391636e-06,
              "rank": 1,
              "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124",
              "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
              "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
              "read_request_id": "b2ce4d1e809a197df6c864261264b583737b1f41c59f8a4b6e353911b84c5bfe",
              "result_sha256": "2d855cfa0b4f7e90c3711783b7779709f3b964760fd789d0d65b11f9b3604ac5"
            },
            "usage_after": {
              "steps": 2,
              "searches": 1,
              "reads": 1,
              "reserved_source_bytes": 154,
              "delivered_source_bytes": 154
            }
          },
          {
            "action": {
              "tool": "search",
              "arguments": {
                "query": "overwritten",
                "version": "3.12",
                "k": 2
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 0,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "aeffc68efdee46b8d23c597f651143424c575c0c1a4699ab08a48734d011f127",
              "query": "overwritten",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "aeffc68efdee46b8d23c597f651143424c575c0c1a4699ab08a48734d011f127",
                  "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                  "source_id": "path-write",
                  "version": "3.12",
                  "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                  "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "byte_count": 224,
                  "native_score": -0.7992166451067194,
                  "rank": 1,
                  "read_token": "93594ba3ddd6fd296ba70304fa0bd5bf8c8c5d39e1d4d3b97db54d31c512fb84"
                }
              ]
            },
            "usage_after": {
              "steps": 3,
              "searches": 2,
              "reads": 1,
              "reserved_source_bytes": 154,
              "delivered_source_bytes": 154
            }
          },
          {
            "action": {
              "tool": "read",
              "arguments": {
                "handle": {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "aeffc68efdee46b8d23c597f651143424c575c0c1a4699ab08a48734d011f127",
                  "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                  "source_id": "path-write",
                  "version": "3.12",
                  "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                  "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "byte_count": 224,
                  "native_score": -0.7992166451067194,
                  "rank": 1,
                  "read_token": "93594ba3ddd6fd296ba70304fa0bd5bf8c8c5d39e1d4d3b97db54d31c512fb84"
                }
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 224,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "aeffc68efdee46b8d23c597f651143424c575c0c1a4699ab08a48734d011f127",
              "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
              "source_id": "path-write",
              "version": "3.12",
              "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
              "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
              "byte_count": 224,
              "native_score": -0.7992166451067194,
              "rank": 1,
              "read_token": "93594ba3ddd6fd296ba70304fa0bd5bf8c8c5d39e1d4d3b97db54d31c512fb84",
              "text": "Path.write_text(data, encoding=None, errors=None, newline=None) writes text to a file and closes it. Set encoding=\"utf-8\" to select UTF-8. An existing file is overwritten. The method returns the number of characters written.",
              "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.write_text",
              "read_request_id": "ebff16e2002855a51b3a7f1ce690d01e0e29c9bc77e4e87001979c0a88be7db4",
              "result_sha256": "b6df6bc29cc62ee7c203fe983839c14991b707c9c2d312b3f5561b40769b2c38"
            },
            "usage_after": {
              "steps": 4,
              "searches": 2,
              "reads": 2,
              "reserved_source_bytes": 378,
              "delivered_source_bytes": 378
            }
          }
        ],
        "observations": [
          {
            "tool": "search",
            "arguments": {
              "query": "text",
              "version": "3.12",
              "k": 2
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "query": "text",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                },
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                  "source_id": "path-write",
                  "version": "3.12",
                  "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                  "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "byte_count": 224,
                  "native_score": -1.3203883495145633e-06,
                  "rank": 2,
                  "read_token": "189bf56802f958924691f452106802696dcd22186df4ee9c03bf0efecf1e1f81"
                }
              ]
            }
          },
          {
            "tool": "read",
            "arguments": {
              "handle": {
                "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                "source_id": "path-read",
                "version": "3.12",
                "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                "start_line": 1,
                "end_line": 1,
                "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                "byte_count": 154,
                "native_score": -1.4220532319391636e-06,
                "rank": 1,
                "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
              }
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
              "source_id": "path-read",
              "version": "3.12",
              "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
              "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "byte_count": 154,
              "native_score": -1.4220532319391636e-06,
              "rank": 1,
              "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124",
              "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
              "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
              "read_request_id": "b2ce4d1e809a197df6c864261264b583737b1f41c59f8a4b6e353911b84c5bfe",
              "result_sha256": "2d855cfa0b4f7e90c3711783b7779709f3b964760fd789d0d65b11f9b3604ac5"
            }
          },
          {
            "tool": "search",
            "arguments": {
              "query": "overwritten",
              "version": "3.12",
              "k": 2
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "aeffc68efdee46b8d23c597f651143424c575c0c1a4699ab08a48734d011f127",
              "query": "overwritten",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "aeffc68efdee46b8d23c597f651143424c575c0c1a4699ab08a48734d011f127",
                  "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                  "source_id": "path-write",
                  "version": "3.12",
                  "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                  "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "byte_count": 224,
                  "native_score": -0.7992166451067194,
                  "rank": 1,
                  "read_token": "93594ba3ddd6fd296ba70304fa0bd5bf8c8c5d39e1d4d3b97db54d31c512fb84"
                }
              ]
            }
          },
          {
            "tool": "read",
            "arguments": {
              "handle": {
                "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                "search_request_id": "aeffc68efdee46b8d23c597f651143424c575c0c1a4699ab08a48734d011f127",
                "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                "source_id": "path-write",
                "version": "3.12",
                "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                "start_line": 1,
                "end_line": 1,
                "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                "byte_count": 224,
                "native_score": -0.7992166451067194,
                "rank": 1,
                "read_token": "93594ba3ddd6fd296ba70304fa0bd5bf8c8c5d39e1d4d3b97db54d31c512fb84"
              }
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "aeffc68efdee46b8d23c597f651143424c575c0c1a4699ab08a48734d011f127",
              "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
              "source_id": "path-write",
              "version": "3.12",
              "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
              "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
              "byte_count": 224,
              "native_score": -0.7992166451067194,
              "rank": 1,
              "read_token": "93594ba3ddd6fd296ba70304fa0bd5bf8c8c5d39e1d4d3b97db54d31c512fb84",
              "text": "Path.write_text(data, encoding=None, errors=None, newline=None) writes text to a file and closes it. Set encoding=\"utf-8\" to select UTF-8. An existing file is overwritten. The method returns the number of characters written.",
              "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.write_text",
              "read_request_id": "ebff16e2002855a51b3a7f1ce690d01e0e29c9bc77e4e87001979c0a88be7db4",
              "result_sha256": "b6df6bc29cc62ee7c203fe983839c14991b707c9c2d312b3f5561b40769b2c38"
            }
          }
        ],
        "scope": "Host validates and admits bounded documentation actions; Gemini calls have a separate shared request budget."
      },
      "outcome_check": {
        "status": "pass",
        "reason": "all_bound_target_passages_read",
        "read_passage_ids": [
          "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
          "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7"
        ],
        "target_passage_ids": [
          "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7"
        ]
      }
    },
    {
      "case_id": "heldout-existing",
      "question": "If the destination already exists, does pathlib write_text append or overwrite its contents?",
      "version": "3.12",
      "policy": "gemini",
      "policy_terminal_status": null,
      "policy_reason": null,
      "runtime": {
        "controller_status": "failed",
        "terminal_reason": "ValueError: invalid_search_query",
        "limits": {
          "max_steps": 6,
          "max_searches": 2,
          "max_reads": 2,
          "max_source_bytes": 4096
        },
        "usage": {
          "steps": 0,
          "searches": 0,
          "reads": 0,
          "reserved_source_bytes": 0,
          "delivered_source_bytes": 0
        },
        "trace": [
          {
            "decision": "refused",
            "reason": "ValueError: invalid_search_query",
            "invoked": false
          }
        ],
        "observations": [],
        "scope": "Host validates and admits bounded documentation actions; Gemini calls have a separate shared request budget."
      },
      "outcome_check": {
        "status": "fail",
        "reason": "controller_or_tool_failure",
        "read_passage_ids": [],
        "target_passage_ids": [
          "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7"
        ]
      }
    },
    {
      "case_id": "heldout-json",
      "question": "What Python type does JSON serialization with dumps return?",
      "version": "3.12",
      "policy": "fixed_pipeline",
      "policy_terminal_status": "completed",
      "policy_reason": "fixed_pipeline_done",
      "runtime": {
        "controller_status": "completed",
        "terminal_reason": "policy_complete",
        "limits": {
          "max_steps": 6,
          "max_searches": 2,
          "max_reads": 2,
          "max_source_bytes": 4096
        },
        "usage": {
          "steps": 2,
          "searches": 1,
          "reads": 1,
          "reserved_source_bytes": 154,
          "delivered_source_bytes": 154
        },
        "trace": [
          {
            "action": {
              "tool": "search",
              "arguments": {
                "query": "text",
                "version": "3.12",
                "k": 2
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 0,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "query": "text",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                },
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                  "source_id": "path-write",
                  "version": "3.12",
                  "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                  "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "byte_count": 224,
                  "native_score": -1.3203883495145633e-06,
                  "rank": 2,
                  "read_token": "189bf56802f958924691f452106802696dcd22186df4ee9c03bf0efecf1e1f81"
                }
              ]
            },
            "usage_after": {
              "steps": 1,
              "searches": 1,
              "reads": 0,
              "reserved_source_bytes": 0,
              "delivered_source_bytes": 0
            }
          },
          {
            "action": {
              "tool": "read",
              "arguments": {
                "handle": {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                }
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 154,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
              "source_id": "path-read",
              "version": "3.12",
              "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
              "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "byte_count": 154,
              "native_score": -1.4220532319391636e-06,
              "rank": 1,
              "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124",
              "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
              "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
              "read_request_id": "b2ce4d1e809a197df6c864261264b583737b1f41c59f8a4b6e353911b84c5bfe",
              "result_sha256": "2d855cfa0b4f7e90c3711783b7779709f3b964760fd789d0d65b11f9b3604ac5"
            },
            "usage_after": {
              "steps": 2,
              "searches": 1,
              "reads": 1,
              "reserved_source_bytes": 154,
              "delivered_source_bytes": 154
            }
          }
        ],
        "observations": [
          {
            "tool": "search",
            "arguments": {
              "query": "text",
              "version": "3.12",
              "k": 2
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "query": "text",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                },
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                  "source_id": "path-write",
                  "version": "3.12",
                  "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                  "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "byte_count": 224,
                  "native_score": -1.3203883495145633e-06,
                  "rank": 2,
                  "read_token": "189bf56802f958924691f452106802696dcd22186df4ee9c03bf0efecf1e1f81"
                }
              ]
            }
          },
          {
            "tool": "read",
            "arguments": {
              "handle": {
                "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                "source_id": "path-read",
                "version": "3.12",
                "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                "start_line": 1,
                "end_line": 1,
                "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                "byte_count": 154,
                "native_score": -1.4220532319391636e-06,
                "rank": 1,
                "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
              }
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
              "source_id": "path-read",
              "version": "3.12",
              "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
              "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "byte_count": 154,
              "native_score": -1.4220532319391636e-06,
              "rank": 1,
              "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124",
              "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
              "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
              "read_request_id": "b2ce4d1e809a197df6c864261264b583737b1f41c59f8a4b6e353911b84c5bfe",
              "result_sha256": "2d855cfa0b4f7e90c3711783b7779709f3b964760fd789d0d65b11f9b3604ac5"
            }
          }
        ],
        "scope": "Host validates and admits bounded documentation actions; Gemini calls have a separate shared request budget."
      },
      "outcome_check": {
        "status": "fail",
        "reason": "target_passage_not_read",
        "read_passage_ids": [
          "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900"
        ],
        "target_passage_ids": [
          "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c"
        ]
      }
    },
    {
      "case_id": "heldout-json",
      "question": "What Python type does JSON serialization with dumps return?",
      "version": "3.12",
      "policy": "scripted_refinement",
      "policy_terminal_status": "completed",
      "policy_reason": "configured_context_marker_found",
      "runtime": {
        "controller_status": "completed",
        "terminal_reason": "policy_complete",
        "limits": {
          "max_steps": 6,
          "max_searches": 2,
          "max_reads": 2,
          "max_source_bytes": 4096
        },
        "usage": {
          "steps": 4,
          "searches": 2,
          "reads": 2,
          "reserved_source_bytes": 353,
          "delivered_source_bytes": 353
        },
        "trace": [
          {
            "action": {
              "tool": "search",
              "arguments": {
                "query": "text",
                "version": "3.12",
                "k": 2
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 0,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "query": "text",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                },
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                  "source_id": "path-write",
                  "version": "3.12",
                  "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                  "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "byte_count": 224,
                  "native_score": -1.3203883495145633e-06,
                  "rank": 2,
                  "read_token": "189bf56802f958924691f452106802696dcd22186df4ee9c03bf0efecf1e1f81"
                }
              ]
            },
            "usage_after": {
              "steps": 1,
              "searches": 1,
              "reads": 0,
              "reserved_source_bytes": 0,
              "delivered_source_bytes": 0
            }
          },
          {
            "action": {
              "tool": "read",
              "arguments": {
                "handle": {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                }
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 154,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
              "source_id": "path-read",
              "version": "3.12",
              "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
              "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "byte_count": 154,
              "native_score": -1.4220532319391636e-06,
              "rank": 1,
              "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124",
              "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
              "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
              "read_request_id": "b2ce4d1e809a197df6c864261264b583737b1f41c59f8a4b6e353911b84c5bfe",
              "result_sha256": "2d855cfa0b4f7e90c3711783b7779709f3b964760fd789d0d65b11f9b3604ac5"
            },
            "usage_after": {
              "steps": 2,
              "searches": 1,
              "reads": 1,
              "reserved_source_bytes": 154,
              "delivered_source_bytes": 154
            }
          },
          {
            "action": {
              "tool": "search",
              "arguments": {
                "query": "json",
                "version": "3.12",
                "k": 2
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 0,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "ba197ec4bb02ae4b95e6809131447d75281efd2d2c61c98552c2c34989050d8d",
              "query": "json",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "ba197ec4bb02ae4b95e6809131447d75281efd2d2c61c98552c2c34989050d8d",
                  "passage_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
                  "source_id": "json-text",
                  "version": "3.12",
                  "document_revision": "0c4b565b2b1900787d6da9f8539c471f8cb2245c00abf887a57d460364abc57a",
                  "source_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                  "byte_count": 199,
                  "native_score": -1.1460737786069228,
                  "rank": 1,
                  "read_token": "9cf3e73f825e876ae4d4ff2f5f40debb78f766a711502b807aeff90d29a9a384"
                }
              ]
            },
            "usage_after": {
              "steps": 3,
              "searches": 2,
              "reads": 1,
              "reserved_source_bytes": 154,
              "delivered_source_bytes": 154
            }
          },
          {
            "action": {
              "tool": "read",
              "arguments": {
                "handle": {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "ba197ec4bb02ae4b95e6809131447d75281efd2d2c61c98552c2c34989050d8d",
                  "passage_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
                  "source_id": "json-text",
                  "version": "3.12",
                  "document_revision": "0c4b565b2b1900787d6da9f8539c471f8cb2245c00abf887a57d460364abc57a",
                  "source_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                  "byte_count": 199,
                  "native_score": -1.1460737786069228,
                  "rank": 1,
                  "read_token": "9cf3e73f825e876ae4d4ff2f5f40debb78f766a711502b807aeff90d29a9a384"
                }
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 199,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "ba197ec4bb02ae4b95e6809131447d75281efd2d2c61c98552c2c34989050d8d",
              "passage_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
              "source_id": "json-text",
              "version": "3.12",
              "document_revision": "0c4b565b2b1900787d6da9f8539c471f8cb2245c00abf887a57d460364abc57a",
              "source_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
              "byte_count": 199,
              "native_score": -1.1460737786069228,
              "rank": 1,
              "read_token": "9cf3e73f825e876ae4d4ff2f5f40debb78f766a711502b807aeff90d29a9a384",
              "text": "json.dumps(obj, ensure_ascii=True) returns a JSON string. Setting ensure_ascii=False leaves non-ASCII characters in that string. It does not write a file or select the encoding used by a file writer.",
              "url": "https://docs.python.org/3.12/library/json.html#json.dumps",
              "read_request_id": "9b9badb1f56b1a51417aaab51e514a7b2370172285a7057fb146368c786ea0d2",
              "result_sha256": "a8db9780486beab6fdc9f18328b643dcdb61dda11290bdacf22db7ead00ec13e"
            },
            "usage_after": {
              "steps": 4,
              "searches": 2,
              "reads": 2,
              "reserved_source_bytes": 353,
              "delivered_source_bytes": 353
            }
          }
        ],
        "observations": [
          {
            "tool": "search",
            "arguments": {
              "query": "text",
              "version": "3.12",
              "k": 2
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "query": "text",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                },
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                  "source_id": "path-write",
                  "version": "3.12",
                  "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                  "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "byte_count": 224,
                  "native_score": -1.3203883495145633e-06,
                  "rank": 2,
                  "read_token": "189bf56802f958924691f452106802696dcd22186df4ee9c03bf0efecf1e1f81"
                }
              ]
            }
          },
          {
            "tool": "read",
            "arguments": {
              "handle": {
                "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                "source_id": "path-read",
                "version": "3.12",
                "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                "start_line": 1,
                "end_line": 1,
                "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                "byte_count": 154,
                "native_score": -1.4220532319391636e-06,
                "rank": 1,
                "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
              }
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
              "source_id": "path-read",
              "version": "3.12",
              "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
              "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "byte_count": 154,
              "native_score": -1.4220532319391636e-06,
              "rank": 1,
              "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124",
              "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
              "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
              "read_request_id": "b2ce4d1e809a197df6c864261264b583737b1f41c59f8a4b6e353911b84c5bfe",
              "result_sha256": "2d855cfa0b4f7e90c3711783b7779709f3b964760fd789d0d65b11f9b3604ac5"
            }
          },
          {
            "tool": "search",
            "arguments": {
              "query": "json",
              "version": "3.12",
              "k": 2
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "ba197ec4bb02ae4b95e6809131447d75281efd2d2c61c98552c2c34989050d8d",
              "query": "json",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "ba197ec4bb02ae4b95e6809131447d75281efd2d2c61c98552c2c34989050d8d",
                  "passage_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
                  "source_id": "json-text",
                  "version": "3.12",
                  "document_revision": "0c4b565b2b1900787d6da9f8539c471f8cb2245c00abf887a57d460364abc57a",
                  "source_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                  "byte_count": 199,
                  "native_score": -1.1460737786069228,
                  "rank": 1,
                  "read_token": "9cf3e73f825e876ae4d4ff2f5f40debb78f766a711502b807aeff90d29a9a384"
                }
              ]
            }
          },
          {
            "tool": "read",
            "arguments": {
              "handle": {
                "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                "search_request_id": "ba197ec4bb02ae4b95e6809131447d75281efd2d2c61c98552c2c34989050d8d",
                "passage_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
                "source_id": "json-text",
                "version": "3.12",
                "document_revision": "0c4b565b2b1900787d6da9f8539c471f8cb2245c00abf887a57d460364abc57a",
                "source_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                "start_line": 1,
                "end_line": 1,
                "text_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                "byte_count": 199,
                "native_score": -1.1460737786069228,
                "rank": 1,
                "read_token": "9cf3e73f825e876ae4d4ff2f5f40debb78f766a711502b807aeff90d29a9a384"
              }
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "ba197ec4bb02ae4b95e6809131447d75281efd2d2c61c98552c2c34989050d8d",
              "passage_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
              "source_id": "json-text",
              "version": "3.12",
              "document_revision": "0c4b565b2b1900787d6da9f8539c471f8cb2245c00abf887a57d460364abc57a",
              "source_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
              "byte_count": 199,
              "native_score": -1.1460737786069228,
              "rank": 1,
              "read_token": "9cf3e73f825e876ae4d4ff2f5f40debb78f766a711502b807aeff90d29a9a384",
              "text": "json.dumps(obj, ensure_ascii=True) returns a JSON string. Setting ensure_ascii=False leaves non-ASCII characters in that string. It does not write a file or select the encoding used by a file writer.",
              "url": "https://docs.python.org/3.12/library/json.html#json.dumps",
              "read_request_id": "9b9badb1f56b1a51417aaab51e514a7b2370172285a7057fb146368c786ea0d2",
              "result_sha256": "a8db9780486beab6fdc9f18328b643dcdb61dda11290bdacf22db7ead00ec13e"
            }
          }
        ],
        "scope": "Host validates and admits bounded documentation actions; Gemini calls have a separate shared request budget."
      },
      "outcome_check": {
        "status": "pass",
        "reason": "all_bound_target_passages_read",
        "read_passage_ids": [
          "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
          "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c"
        ],
        "target_passage_ids": [
          "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c"
        ]
      }
    },
    {
      "case_id": "heldout-json",
      "question": "What Python type does JSON serialization with dumps return?",
      "version": "3.12",
      "policy": "gemini",
      "policy_terminal_status": null,
      "policy_reason": null,
      "runtime": {
        "controller_status": "failed",
        "terminal_reason": "ValueError: invalid_read_arguments",
        "limits": {
          "max_steps": 6,
          "max_searches": 2,
          "max_reads": 2,
          "max_source_bytes": 4096
        },
        "usage": {
          "steps": 1,
          "searches": 1,
          "reads": 0,
          "reserved_source_bytes": 0,
          "delivered_source_bytes": 0
        },
        "trace": [
          {
            "action": {
              "tool": "search",
              "arguments": {
                "query": "json",
                "version": "3.12",
                "k": 2
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 0,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "ba197ec4bb02ae4b95e6809131447d75281efd2d2c61c98552c2c34989050d8d",
              "query": "json",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "ba197ec4bb02ae4b95e6809131447d75281efd2d2c61c98552c2c34989050d8d",
                  "passage_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
                  "source_id": "json-text",
                  "version": "3.12",
                  "document_revision": "0c4b565b2b1900787d6da9f8539c471f8cb2245c00abf887a57d460364abc57a",
                  "source_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                  "byte_count": 199,
                  "native_score": -1.1460737786069228,
                  "rank": 1,
                  "read_token": "9cf3e73f825e876ae4d4ff2f5f40debb78f766a711502b807aeff90d29a9a384"
                }
              ]
            },
            "usage_after": {
              "steps": 1,
              "searches": 1,
              "reads": 0,
              "reserved_source_bytes": 0,
              "delivered_source_bytes": 0
            }
          },
          {
            "decision": "refused",
            "reason": "ValueError: invalid_read_arguments",
            "invoked": false
          }
        ],
        "observations": [
          {
            "tool": "search",
            "arguments": {
              "query": "json",
              "version": "3.12",
              "k": 2
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "ba197ec4bb02ae4b95e6809131447d75281efd2d2c61c98552c2c34989050d8d",
              "query": "json",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "ba197ec4bb02ae4b95e6809131447d75281efd2d2c61c98552c2c34989050d8d",
                  "passage_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
                  "source_id": "json-text",
                  "version": "3.12",
                  "document_revision": "0c4b565b2b1900787d6da9f8539c471f8cb2245c00abf887a57d460364abc57a",
                  "source_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                  "byte_count": 199,
                  "native_score": -1.1460737786069228,
                  "rank": 1,
                  "read_token": "9cf3e73f825e876ae4d4ff2f5f40debb78f766a711502b807aeff90d29a9a384"
                }
              ]
            }
          }
        ],
        "scope": "Host validates and admits bounded documentation actions; Gemini calls have a separate shared request budget."
      },
      "outcome_check": {
        "status": "fail",
        "reason": "controller_or_tool_failure",
        "read_passage_ids": [],
        "target_passage_ids": [
          "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c"
        ]
      }
    },
    {
      "case_id": "heldout-unsupported",
      "question": "How often should our team create a backup of files saved with pathlib?",
      "version": "3.12",
      "policy": "fixed_pipeline",
      "policy_terminal_status": "completed",
      "policy_reason": "fixed_pipeline_done",
      "runtime": {
        "controller_status": "completed",
        "terminal_reason": "policy_complete",
        "limits": {
          "max_steps": 6,
          "max_searches": 2,
          "max_reads": 2,
          "max_source_bytes": 4096
        },
        "usage": {
          "steps": 2,
          "searches": 1,
          "reads": 1,
          "reserved_source_bytes": 154,
          "delivered_source_bytes": 154
        },
        "trace": [
          {
            "action": {
              "tool": "search",
              "arguments": {
                "query": "text",
                "version": "3.12",
                "k": 2
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 0,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "query": "text",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                },
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                  "source_id": "path-write",
                  "version": "3.12",
                  "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                  "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "byte_count": 224,
                  "native_score": -1.3203883495145633e-06,
                  "rank": 2,
                  "read_token": "189bf56802f958924691f452106802696dcd22186df4ee9c03bf0efecf1e1f81"
                }
              ]
            },
            "usage_after": {
              "steps": 1,
              "searches": 1,
              "reads": 0,
              "reserved_source_bytes": 0,
              "delivered_source_bytes": 0
            }
          },
          {
            "action": {
              "tool": "read",
              "arguments": {
                "handle": {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                }
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 154,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
              "source_id": "path-read",
              "version": "3.12",
              "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
              "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "byte_count": 154,
              "native_score": -1.4220532319391636e-06,
              "rank": 1,
              "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124",
              "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
              "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
              "read_request_id": "b2ce4d1e809a197df6c864261264b583737b1f41c59f8a4b6e353911b84c5bfe",
              "result_sha256": "2d855cfa0b4f7e90c3711783b7779709f3b964760fd789d0d65b11f9b3604ac5"
            },
            "usage_after": {
              "steps": 2,
              "searches": 1,
              "reads": 1,
              "reserved_source_bytes": 154,
              "delivered_source_bytes": 154
            }
          }
        ],
        "observations": [
          {
            "tool": "search",
            "arguments": {
              "query": "text",
              "version": "3.12",
              "k": 2
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "query": "text",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                },
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                  "source_id": "path-write",
                  "version": "3.12",
                  "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                  "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "byte_count": 224,
                  "native_score": -1.3203883495145633e-06,
                  "rank": 2,
                  "read_token": "189bf56802f958924691f452106802696dcd22186df4ee9c03bf0efecf1e1f81"
                }
              ]
            }
          },
          {
            "tool": "read",
            "arguments": {
              "handle": {
                "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                "source_id": "path-read",
                "version": "3.12",
                "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                "start_line": 1,
                "end_line": 1,
                "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                "byte_count": 154,
                "native_score": -1.4220532319391636e-06,
                "rank": 1,
                "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
              }
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
              "source_id": "path-read",
              "version": "3.12",
              "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
              "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "byte_count": 154,
              "native_score": -1.4220532319391636e-06,
              "rank": 1,
              "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124",
              "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
              "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
              "read_request_id": "b2ce4d1e809a197df6c864261264b583737b1f41c59f8a4b6e353911b84c5bfe",
              "result_sha256": "2d855cfa0b4f7e90c3711783b7779709f3b964760fd789d0d65b11f9b3604ac5"
            }
          }
        ],
        "scope": "Host validates and admits bounded documentation actions; Gemini calls have a separate shared request budget."
      },
      "outcome_check": {
        "status": "incomplete",
        "reason": "unsupported_fixture_has_no_retrieval_target",
        "read_passage_ids": [
          "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900"
        ],
        "target_passage_ids": []
      }
    },
    {
      "case_id": "heldout-unsupported",
      "question": "How often should our team create a backup of files saved with pathlib?",
      "version": "3.12",
      "policy": "scripted_refinement",
      "policy_terminal_status": "incomplete",
      "policy_reason": "round_limit",
      "runtime": {
        "controller_status": "completed",
        "terminal_reason": "policy_complete",
        "limits": {
          "max_steps": 6,
          "max_searches": 2,
          "max_reads": 2,
          "max_source_bytes": 4096
        },
        "usage": {
          "steps": 4,
          "searches": 2,
          "reads": 2,
          "reserved_source_bytes": 353,
          "delivered_source_bytes": 353
        },
        "trace": [
          {
            "action": {
              "tool": "search",
              "arguments": {
                "query": "text",
                "version": "3.12",
                "k": 2
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 0,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "query": "text",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                },
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                  "source_id": "path-write",
                  "version": "3.12",
                  "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                  "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "byte_count": 224,
                  "native_score": -1.3203883495145633e-06,
                  "rank": 2,
                  "read_token": "189bf56802f958924691f452106802696dcd22186df4ee9c03bf0efecf1e1f81"
                }
              ]
            },
            "usage_after": {
              "steps": 1,
              "searches": 1,
              "reads": 0,
              "reserved_source_bytes": 0,
              "delivered_source_bytes": 0
            }
          },
          {
            "action": {
              "tool": "read",
              "arguments": {
                "handle": {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                }
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 154,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
              "source_id": "path-read",
              "version": "3.12",
              "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
              "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "byte_count": 154,
              "native_score": -1.4220532319391636e-06,
              "rank": 1,
              "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124",
              "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
              "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
              "read_request_id": "b2ce4d1e809a197df6c864261264b583737b1f41c59f8a4b6e353911b84c5bfe",
              "result_sha256": "2d855cfa0b4f7e90c3711783b7779709f3b964760fd789d0d65b11f9b3604ac5"
            },
            "usage_after": {
              "steps": 2,
              "searches": 1,
              "reads": 1,
              "reserved_source_bytes": 154,
              "delivered_source_bytes": 154
            }
          },
          {
            "action": {
              "tool": "search",
              "arguments": {
                "query": "write",
                "version": "3.12",
                "k": 2
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 0,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "ce0ea8d504d083c0a75c9ba19cfa89f88985c889f4e863067efc6a313a49094d",
              "query": "write",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "ce0ea8d504d083c0a75c9ba19cfa89f88985c889f4e863067efc6a313a49094d",
                  "passage_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
                  "source_id": "json-text",
                  "version": "3.12",
                  "document_revision": "0c4b565b2b1900787d6da9f8539c471f8cb2245c00abf887a57d460364abc57a",
                  "source_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                  "byte_count": 199,
                  "native_score": -9.765013054830289e-07,
                  "rank": 1,
                  "read_token": "d8a24ee4e5d00a42b3bd417be07632c7a329c7e2cf39a0dd3ccfdea64badaaa0"
                },
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "ce0ea8d504d083c0a75c9ba19cfa89f88985c889f4e863067efc6a313a49094d",
                  "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                  "source_id": "path-write",
                  "version": "3.12",
                  "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                  "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "byte_count": 224,
                  "native_score": -9.432534678436319e-07,
                  "rank": 2,
                  "read_token": "1bd370d49cf0504dd50e14f4a9e27a9eb6440b789a66ecb0c857d432e025510f"
                }
              ]
            },
            "usage_after": {
              "steps": 3,
              "searches": 2,
              "reads": 1,
              "reserved_source_bytes": 154,
              "delivered_source_bytes": 154
            }
          },
          {
            "action": {
              "tool": "read",
              "arguments": {
                "handle": {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "ce0ea8d504d083c0a75c9ba19cfa89f88985c889f4e863067efc6a313a49094d",
                  "passage_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
                  "source_id": "json-text",
                  "version": "3.12",
                  "document_revision": "0c4b565b2b1900787d6da9f8539c471f8cb2245c00abf887a57d460364abc57a",
                  "source_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                  "byte_count": 199,
                  "native_score": -9.765013054830289e-07,
                  "rank": 1,
                  "read_token": "d8a24ee4e5d00a42b3bd417be07632c7a329c7e2cf39a0dd3ccfdea64badaaa0"
                }
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 199,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "ce0ea8d504d083c0a75c9ba19cfa89f88985c889f4e863067efc6a313a49094d",
              "passage_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
              "source_id": "json-text",
              "version": "3.12",
              "document_revision": "0c4b565b2b1900787d6da9f8539c471f8cb2245c00abf887a57d460364abc57a",
              "source_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
              "byte_count": 199,
              "native_score": -9.765013054830289e-07,
              "rank": 1,
              "read_token": "d8a24ee4e5d00a42b3bd417be07632c7a329c7e2cf39a0dd3ccfdea64badaaa0",
              "text": "json.dumps(obj, ensure_ascii=True) returns a JSON string. Setting ensure_ascii=False leaves non-ASCII characters in that string. It does not write a file or select the encoding used by a file writer.",
              "url": "https://docs.python.org/3.12/library/json.html#json.dumps",
              "read_request_id": "9a1ee0faadc084cb2c51e02f9efb21e61aa11cc069904f9e84a8c43056a188d5",
              "result_sha256": "661fa082d4ddae14438f0f872ae166fdeddb09a04a8be1e622082e376cb25168"
            },
            "usage_after": {
              "steps": 4,
              "searches": 2,
              "reads": 2,
              "reserved_source_bytes": 353,
              "delivered_source_bytes": 353
            }
          }
        ],
        "observations": [
          {
            "tool": "search",
            "arguments": {
              "query": "text",
              "version": "3.12",
              "k": 2
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "query": "text",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                  "source_id": "path-read",
                  "version": "3.12",
                  "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                  "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                  "byte_count": 154,
                  "native_score": -1.4220532319391636e-06,
                  "rank": 1,
                  "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
                },
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                  "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                  "source_id": "path-write",
                  "version": "3.12",
                  "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                  "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "byte_count": 224,
                  "native_score": -1.3203883495145633e-06,
                  "rank": 2,
                  "read_token": "189bf56802f958924691f452106802696dcd22186df4ee9c03bf0efecf1e1f81"
                }
              ]
            }
          },
          {
            "tool": "read",
            "arguments": {
              "handle": {
                "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
                "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
                "source_id": "path-read",
                "version": "3.12",
                "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
                "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                "start_line": 1,
                "end_line": 1,
                "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
                "byte_count": 154,
                "native_score": -1.4220532319391636e-06,
                "rank": 1,
                "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124"
              }
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "3f33383e706e36449e22c380704d880d5e2b8a4b1f6b779b8aa54206e072ad5f",
              "passage_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
              "source_id": "path-read",
              "version": "3.12",
              "document_revision": "b58385b9bf597a951c16c035653d639929924e53eae3e5ada255afd6ee1592f2",
              "source_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "dc097ca63486ace45279f412ea266bf754fcd2f0b1153f5d763ce905dafbc2c0",
              "byte_count": 154,
              "native_score": -1.4220532319391636e-06,
              "rank": 1,
              "read_token": "72abbf5e0022ef780447ed8c2322b711cac8b4971d817a440bc02dfa28b07124",
              "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
              "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
              "read_request_id": "b2ce4d1e809a197df6c864261264b583737b1f41c59f8a4b6e353911b84c5bfe",
              "result_sha256": "2d855cfa0b4f7e90c3711783b7779709f3b964760fd789d0d65b11f9b3604ac5"
            }
          },
          {
            "tool": "search",
            "arguments": {
              "query": "write",
              "version": "3.12",
              "k": 2
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "ce0ea8d504d083c0a75c9ba19cfa89f88985c889f4e863067efc6a313a49094d",
              "query": "write",
              "version": "3.12",
              "k": 2,
              "results": [
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "ce0ea8d504d083c0a75c9ba19cfa89f88985c889f4e863067efc6a313a49094d",
                  "passage_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
                  "source_id": "json-text",
                  "version": "3.12",
                  "document_revision": "0c4b565b2b1900787d6da9f8539c471f8cb2245c00abf887a57d460364abc57a",
                  "source_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                  "byte_count": 199,
                  "native_score": -9.765013054830289e-07,
                  "rank": 1,
                  "read_token": "d8a24ee4e5d00a42b3bd417be07632c7a329c7e2cf39a0dd3ccfdea64badaaa0"
                },
                {
                  "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                  "search_request_id": "ce0ea8d504d083c0a75c9ba19cfa89f88985c889f4e863067efc6a313a49094d",
                  "passage_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
                  "source_id": "path-write",
                  "version": "3.12",
                  "document_revision": "28c6349eaf80b629322edb310bb871dc45a04af9fad3b752bc75d12f0d7314e7",
                  "source_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "start_line": 1,
                  "end_line": 1,
                  "text_sha256": "a48b4b9f763cd6309ebe32af19c5a814adcf810c2259683da9f328ccee46948e",
                  "byte_count": 224,
                  "native_score": -9.432534678436319e-07,
                  "rank": 2,
                  "read_token": "1bd370d49cf0504dd50e14f4a9e27a9eb6440b789a66ecb0c857d432e025510f"
                }
              ]
            }
          },
          {
            "tool": "read",
            "arguments": {
              "handle": {
                "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
                "search_request_id": "ce0ea8d504d083c0a75c9ba19cfa89f88985c889f4e863067efc6a313a49094d",
                "passage_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
                "source_id": "json-text",
                "version": "3.12",
                "document_revision": "0c4b565b2b1900787d6da9f8539c471f8cb2245c00abf887a57d460364abc57a",
                "source_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                "start_line": 1,
                "end_line": 1,
                "text_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
                "byte_count": 199,
                "native_score": -9.765013054830289e-07,
                "rank": 1,
                "read_token": "d8a24ee4e5d00a42b3bd417be07632c7a329c7e2cf39a0dd3ccfdea64badaaa0"
              }
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "ce0ea8d504d083c0a75c9ba19cfa89f88985c889f4e863067efc6a313a49094d",
              "passage_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
              "source_id": "json-text",
              "version": "3.12",
              "document_revision": "0c4b565b2b1900787d6da9f8539c471f8cb2245c00abf887a57d460364abc57a",
              "source_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
              "start_line": 1,
              "end_line": 1,
              "text_sha256": "0f5efa9669a97f93d5e7e8fe91de80a0d954643947ac9619dba41d3904166288",
              "byte_count": 199,
              "native_score": -9.765013054830289e-07,
              "rank": 1,
              "read_token": "d8a24ee4e5d00a42b3bd417be07632c7a329c7e2cf39a0dd3ccfdea64badaaa0",
              "text": "json.dumps(obj, ensure_ascii=True) returns a JSON string. Setting ensure_ascii=False leaves non-ASCII characters in that string. It does not write a file or select the encoding used by a file writer.",
              "url": "https://docs.python.org/3.12/library/json.html#json.dumps",
              "read_request_id": "9a1ee0faadc084cb2c51e02f9efb21e61aa11cc069904f9e84a8c43056a188d5",
              "result_sha256": "661fa082d4ddae14438f0f872ae166fdeddb09a04a8be1e622082e376cb25168"
            }
          }
        ],
        "scope": "Host validates and admits bounded documentation actions; Gemini calls have a separate shared request budget."
      },
      "outcome_check": {
        "status": "incomplete",
        "reason": "unsupported_fixture_has_no_retrieval_target",
        "read_passage_ids": [
          "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
          "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c"
        ],
        "target_passage_ids": []
      }
    },
    {
      "case_id": "heldout-unsupported",
      "question": "How often should our team create a backup of files saved with pathlib?",
      "version": "3.12",
      "policy": "gemini",
      "policy_terminal_status": "completed",
      "policy_reason": "The pathlib module in Python is a library for filesystem path manipulation and does not contain information or policies regarding file backup frequency.",
      "runtime": {
        "controller_status": "completed",
        "terminal_reason": "policy_complete",
        "limits": {
          "max_steps": 6,
          "max_searches": 2,
          "max_reads": 2,
          "max_source_bytes": 4096
        },
        "usage": {
          "steps": 1,
          "searches": 1,
          "reads": 0,
          "reserved_source_bytes": 0,
          "delivered_source_bytes": 0
        },
        "trace": [
          {
            "action": {
              "tool": "search",
              "arguments": {
                "query": "pathlib",
                "version": "3.12",
                "k": 2
              }
            },
            "decision": "executed",
            "invoked": true,
            "reserved_source_bytes": 0,
            "status": "succeeded",
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "99d39479617c633ab856eaba7df76831e1ac9eed8c58608de2c59734fd6efb51",
              "query": "pathlib",
              "version": "3.12",
              "k": 2,
              "results": []
            },
            "usage_after": {
              "steps": 1,
              "searches": 1,
              "reads": 0,
              "reserved_source_bytes": 0,
              "delivered_source_bytes": 0
            }
          }
        ],
        "observations": [
          {
            "tool": "search",
            "arguments": {
              "query": "pathlib",
              "version": "3.12",
              "k": 2
            },
            "result": {
              "corpus_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
              "search_request_id": "99d39479617c633ab856eaba7df76831e1ac9eed8c58608de2c59734fd6efb51",
              "query": "pathlib",
              "version": "3.12",
              "k": 2,
              "results": []
            }
          }
        ],
        "scope": "Host validates and admits bounded documentation actions; Gemini calls have a separate shared request budget."
      },
      "outcome_check": {
        "status": "incomplete",
        "reason": "unsupported_fixture_has_no_retrieval_target",
        "read_passage_ids": [],
        "target_passage_ids": []
      }
    }
  ],
  "calls": [
    {
      "file": "request-007.json",
      "request_sha256": "685d3688081a7b632c1545ce57b91def79788460a73eb1884184fb06f6558502",
      "estimated_usd": 9.5e-05
    },
    {
      "file": "request-009.json",
      "request_sha256": "b77f5685c7ee8132b09e3ae4c2a54106017168399be20fa9496507a6359a1794",
      "estimated_usd": 0.0001445
    },
    {
      "file": "request-011.json",
      "request_sha256": "cd411448106eee127a4d008252b1a0525713ccf45804825f9f5b7a5dfc6de1e7",
      "estimated_usd": 9.775e-05
    },
    {
      "file": "request-013.json",
      "request_sha256": "a8f46181b5fe32b2be9319421999689059b813c9c5645a05d7ac5a1a0e73b722",
      "estimated_usd": 9.3e-05
    },
    {
      "file": "request-014.json",
      "request_sha256": "94ac1a52a222a95057d7fd6826d211681e1a963ef65920c678591273e0f3ec39",
      "estimated_usd": 0.00033825
    },
    {
      "file": "request-015.json",
      "request_sha256": "a6dc3d2c113e8b6ae5a58f0c9ba394c7eb60ba541a2fa7d4925bb5adebf922dd",
      "estimated_usd": 9.55e-05
    },
    {
      "file": "request-016.json",
      "request_sha256": "e4214977d825a699cf0a39684e7bd8a6946bb79b026e02fac639a93cbdd84e3b",
      "estimated_usd": 0.000163
    }
  ],
  "scope": "Four new questions frozen before Gemini execution, unseen by its prompt. These reuse development intent families and are not an independent benchmark. Same snapshot, tool limits and questions; policies differ in domain rules, actual effort and model cost. Outcomes check exact target reads, not generated answers. Unsupported cases remain incomplete."
}