{
  "execution_mode": "offline-frozen-development-comparison",
  "model_called": false,
  "snapshot_id": "0615d22a3315e97c0a88c6c566c546877304f5a28a97e83f68dd36881d4bbd0e",
  "frozen_configuration": {
    "names": [
      "lexical",
      "vector",
      "rrf",
      "rrf+coverage"
    ],
    "candidate_budget_per_branch": 3,
    "rank_constant": 60,
    "cutoffs": [
      1,
      2
    ],
    "scope": "four original development cases; no held-out queries; no configuration tuning"
  },
  "label_binding_receipts": [
    {
      "case_id": "utf8-write",
      "legacy_chunk_id": "path-write:b71a2d18a53d86bf",
      "current_chunk_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
      "binding": [
        "source_id",
        "source_version",
        "source_text_sha256",
        "chunk_text_sha256"
      ]
    },
    {
      "case_id": "replace-file",
      "legacy_chunk_id": "path-write:b71a2d18a53d86bf",
      "current_chunk_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
      "binding": [
        "source_id",
        "source_version",
        "source_text_sha256",
        "chunk_text_sha256"
      ]
    },
    {
      "case_id": "json-result-type",
      "legacy_chunk_id": "json-text:8a17e251a41ccb3b",
      "current_chunk_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
      "binding": [
        "source_id",
        "source_version",
        "source_text_sha256",
        "chunk_text_sha256"
      ]
    }
  ],
  "retrieval_runs": [
    {
      "case_id": "utf8-write",
      "query": "How do I explicitly select UTF-8 when saving text with pathlib?",
      "version": "3.12",
      "lexical_query_terms": [
        "how",
        "do",
        "i",
        "explicitly",
        "select",
        "utf",
        "8",
        "when",
        "saving",
        "text",
        "with",
        "pathlib"
      ],
      "rankings": {
        "lexical": [
          {
            "chunk_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
            "source_id": "path-read",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
            "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
            "native_score": -0.8901430296001853
          },
          {
            "chunk_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
            "source_id": "path-write",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.write_text",
            "text": "Path.write_text(data, encoding=None, errors=None, newline=None) writes text to a file and closes it. Set encoding=\"utf-8\" to select UTF-8. An existing file is overwritten. The method returns the number of characters written.",
            "native_score": -4.904418516387322e-06
          },
          {
            "chunk_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
            "source_id": "json-text",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/json.html#json.dumps",
            "text": "json.dumps(obj, ensure_ascii=True) returns a JSON string. Setting ensure_ascii=False leaves non-ASCII characters in that string. It does not write a file or select the encoding used by a file writer.",
            "native_score": -9.765013054830289e-07
          }
        ],
        "vector": [
          {
            "chunk_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
            "source_id": "path-read",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
            "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
            "score": 0.637539995393079
          },
          {
            "chunk_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
            "source_id": "path-write",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.write_text",
            "text": "Path.write_text(data, encoding=None, errors=None, newline=None) writes text to a file and closes it. Set encoding=\"utf-8\" to select UTF-8. An existing file is overwritten. The method returns the number of characters written.",
            "score": 0.5762678845903342
          },
          {
            "chunk_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
            "source_id": "json-text",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/json.html#json.dumps",
            "text": "json.dumps(obj, ensure_ascii=True) returns a JSON string. Setting ensure_ascii=False leaves non-ASCII characters in that string. It does not write a file or select the encoding used by a file writer.",
            "score": 0.4948350231504922
          }
        ],
        "rrf": [
          {
            "chunk_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
            "source_id": "path-read",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
            "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
            "lexical_score": -0.8901430296001853,
            "vector_score": 0.637539995393079,
            "lexical_rank": 1,
            "vector_rank": 1,
            "lexical_contribution": 0.01639344262295082,
            "vector_contribution": 0.01639344262295082,
            "fused_score": 0.03278688524590164
          },
          {
            "chunk_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
            "source_id": "path-write",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.write_text",
            "text": "Path.write_text(data, encoding=None, errors=None, newline=None) writes text to a file and closes it. Set encoding=\"utf-8\" to select UTF-8. An existing file is overwritten. The method returns the number of characters written.",
            "lexical_score": -4.904418516387322e-06,
            "vector_score": 0.5762678845903342,
            "lexical_rank": 2,
            "vector_rank": 2,
            "lexical_contribution": 0.016129032258064516,
            "vector_contribution": 0.016129032258064516,
            "fused_score": 0.03225806451612903
          },
          {
            "chunk_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
            "source_id": "json-text",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/json.html#json.dumps",
            "text": "json.dumps(obj, ensure_ascii=True) returns a JSON string. Setting ensure_ascii=False leaves non-ASCII characters in that string. It does not write a file or select the encoding used by a file writer.",
            "lexical_score": -9.765013054830289e-07,
            "vector_score": 0.4948350231504922,
            "lexical_rank": 3,
            "vector_rank": 3,
            "lexical_contribution": 0.015873015873015872,
            "vector_contribution": 0.015873015873015872,
            "fused_score": 0.031746031746031744
          }
        ],
        "rrf+coverage": [
          {
            "chunk_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
            "source_id": "path-read",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
            "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
            "lexical_score": -0.8901430296001853,
            "vector_score": 0.637539995393079,
            "lexical_rank": 1,
            "vector_rank": 1,
            "lexical_contribution": 0.01639344262295082,
            "vector_contribution": 0.01639344262295082,
            "fused_score": 0.03278688524590164,
            "incoming_rank": 1,
            "rerank_features": {
              "query_tokens": [
                "how",
                "do",
                "i",
                "explicitly",
                "select",
                "utf",
                "8",
                "when",
                "saving",
                "text",
                "with",
                "pathlib"
              ],
              "matched_tokens": [
                "8",
                "text",
                "utf",
                "when"
              ],
              "matched_count": 4,
              "query_token_count": 12
            },
            "coverage_score": 0.3333333333333333,
            "rerank_rank": 1
          },
          {
            "chunk_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
            "source_id": "path-write",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.write_text",
            "text": "Path.write_text(data, encoding=None, errors=None, newline=None) writes text to a file and closes it. Set encoding=\"utf-8\" to select UTF-8. An existing file is overwritten. The method returns the number of characters written.",
            "lexical_score": -4.904418516387322e-06,
            "vector_score": 0.5762678845903342,
            "lexical_rank": 2,
            "vector_rank": 2,
            "lexical_contribution": 0.016129032258064516,
            "vector_contribution": 0.016129032258064516,
            "fused_score": 0.03225806451612903,
            "incoming_rank": 2,
            "rerank_features": {
              "query_tokens": [
                "how",
                "do",
                "i",
                "explicitly",
                "select",
                "utf",
                "8",
                "when",
                "saving",
                "text",
                "with",
                "pathlib"
              ],
              "matched_tokens": [
                "8",
                "select",
                "text",
                "utf"
              ],
              "matched_count": 4,
              "query_token_count": 12
            },
            "coverage_score": 0.3333333333333333,
            "rerank_rank": 2
          },
          {
            "chunk_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
            "source_id": "json-text",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/json.html#json.dumps",
            "text": "json.dumps(obj, ensure_ascii=True) returns a JSON string. Setting ensure_ascii=False leaves non-ASCII characters in that string. It does not write a file or select the encoding used by a file writer.",
            "lexical_score": -9.765013054830289e-07,
            "vector_score": 0.4948350231504922,
            "lexical_rank": 3,
            "vector_rank": 3,
            "lexical_contribution": 0.015873015873015872,
            "vector_contribution": 0.015873015873015872,
            "fused_score": 0.031746031746031744,
            "incoming_rank": 3,
            "rerank_features": {
              "query_tokens": [
                "how",
                "do",
                "i",
                "explicitly",
                "select",
                "utf",
                "8",
                "when",
                "saving",
                "text",
                "with",
                "pathlib"
              ],
              "matched_tokens": [
                "select"
              ],
              "matched_count": 1,
              "query_token_count": 12
            },
            "coverage_score": 0.08333333333333333,
            "rerank_rank": 3
          }
        ]
      }
    },
    {
      "case_id": "replace-file",
      "query": "Will write_text replace the contents of an existing file?",
      "version": "3.12",
      "lexical_query_terms": [
        "will",
        "write",
        "text",
        "replace",
        "the",
        "contents",
        "of",
        "an",
        "existing",
        "file"
      ],
      "rankings": {
        "lexical": [
          {
            "chunk_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
            "source_id": "path-write",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.write_text",
            "text": "Path.write_text(data, encoding=None, errors=None, newline=None) writes text to a file and closes it. Set encoding=\"utf-8\" to select UTF-8. An existing file is overwritten. The method returns the number of characters written.",
            "native_score": -2.3976548397386743
          },
          {
            "chunk_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
            "source_id": "path-read",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
            "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
            "native_score": -4.084683995209214e-06
          },
          {
            "chunk_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
            "source_id": "json-text",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/json.html#json.dumps",
            "text": "json.dumps(obj, ensure_ascii=True) returns a JSON string. Setting ensure_ascii=False leaves non-ASCII characters in that string. It does not write a file or select the encoding used by a file writer.",
            "native_score": -3.3056246724488786e-06
          }
        ],
        "vector": [
          {
            "chunk_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
            "source_id": "path-write",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.write_text",
            "text": "Path.write_text(data, encoding=None, errors=None, newline=None) writes text to a file and closes it. Set encoding=\"utf-8\" to select UTF-8. An existing file is overwritten. The method returns the number of characters written.",
            "score": 0.6357498076122466
          },
          {
            "chunk_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
            "source_id": "path-read",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
            "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
            "score": 0.5547892797395033
          },
          {
            "chunk_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
            "source_id": "json-text",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/json.html#json.dumps",
            "text": "json.dumps(obj, ensure_ascii=True) returns a JSON string. Setting ensure_ascii=False leaves non-ASCII characters in that string. It does not write a file or select the encoding used by a file writer.",
            "score": 0.36746132784169144
          }
        ],
        "rrf": [
          {
            "chunk_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
            "source_id": "path-write",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.write_text",
            "text": "Path.write_text(data, encoding=None, errors=None, newline=None) writes text to a file and closes it. Set encoding=\"utf-8\" to select UTF-8. An existing file is overwritten. The method returns the number of characters written.",
            "lexical_score": -2.3976548397386743,
            "vector_score": 0.6357498076122466,
            "lexical_rank": 1,
            "vector_rank": 1,
            "lexical_contribution": 0.01639344262295082,
            "vector_contribution": 0.01639344262295082,
            "fused_score": 0.03278688524590164
          },
          {
            "chunk_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
            "source_id": "path-read",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
            "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
            "lexical_score": -4.084683995209214e-06,
            "vector_score": 0.5547892797395033,
            "lexical_rank": 2,
            "vector_rank": 2,
            "lexical_contribution": 0.016129032258064516,
            "vector_contribution": 0.016129032258064516,
            "fused_score": 0.03225806451612903
          },
          {
            "chunk_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
            "source_id": "json-text",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/json.html#json.dumps",
            "text": "json.dumps(obj, ensure_ascii=True) returns a JSON string. Setting ensure_ascii=False leaves non-ASCII characters in that string. It does not write a file or select the encoding used by a file writer.",
            "lexical_score": -3.3056246724488786e-06,
            "vector_score": 0.36746132784169144,
            "lexical_rank": 3,
            "vector_rank": 3,
            "lexical_contribution": 0.015873015873015872,
            "vector_contribution": 0.015873015873015872,
            "fused_score": 0.031746031746031744
          }
        ],
        "rrf+coverage": [
          {
            "chunk_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
            "source_id": "path-write",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.write_text",
            "text": "Path.write_text(data, encoding=None, errors=None, newline=None) writes text to a file and closes it. Set encoding=\"utf-8\" to select UTF-8. An existing file is overwritten. The method returns the number of characters written.",
            "lexical_score": -2.3976548397386743,
            "vector_score": 0.6357498076122466,
            "lexical_rank": 1,
            "vector_rank": 1,
            "lexical_contribution": 0.01639344262295082,
            "vector_contribution": 0.01639344262295082,
            "fused_score": 0.03278688524590164,
            "incoming_rank": 1,
            "rerank_features": {
              "query_tokens": [
                "will",
                "write",
                "text",
                "replace",
                "the",
                "contents",
                "of",
                "an",
                "existing",
                "file"
              ],
              "matched_tokens": [
                "an",
                "existing",
                "file",
                "of",
                "text",
                "the",
                "write"
              ],
              "matched_count": 7,
              "query_token_count": 10
            },
            "coverage_score": 0.7,
            "rerank_rank": 1
          },
          {
            "chunk_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
            "source_id": "path-read",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
            "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
            "lexical_score": -4.084683995209214e-06,
            "vector_score": 0.5547892797395033,
            "lexical_rank": 2,
            "vector_rank": 2,
            "lexical_contribution": 0.016129032258064516,
            "vector_contribution": 0.016129032258064516,
            "fused_score": 0.03225806451612903,
            "incoming_rank": 2,
            "rerank_features": {
              "query_tokens": [
                "will",
                "write",
                "text",
                "replace",
                "the",
                "contents",
                "of",
                "an",
                "existing",
                "file"
              ],
              "matched_tokens": [
                "file",
                "text",
                "the"
              ],
              "matched_count": 3,
              "query_token_count": 10
            },
            "coverage_score": 0.3,
            "rerank_rank": 2
          },
          {
            "chunk_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
            "source_id": "json-text",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/json.html#json.dumps",
            "text": "json.dumps(obj, ensure_ascii=True) returns a JSON string. Setting ensure_ascii=False leaves non-ASCII characters in that string. It does not write a file or select the encoding used by a file writer.",
            "lexical_score": -3.3056246724488786e-06,
            "vector_score": 0.36746132784169144,
            "lexical_rank": 3,
            "vector_rank": 3,
            "lexical_contribution": 0.015873015873015872,
            "vector_contribution": 0.015873015873015872,
            "fused_score": 0.031746031746031744,
            "incoming_rank": 3,
            "rerank_features": {
              "query_tokens": [
                "will",
                "write",
                "text",
                "replace",
                "the",
                "contents",
                "of",
                "an",
                "existing",
                "file"
              ],
              "matched_tokens": [
                "file",
                "the",
                "write"
              ],
              "matched_count": 3,
              "query_token_count": 10
            },
            "coverage_score": 0.3,
            "rerank_rank": 3
          }
        ]
      }
    },
    {
      "case_id": "json-result-type",
      "query": "Does json.dumps produce text or a bytes object?",
      "version": "3.12",
      "lexical_query_terms": [
        "does",
        "json",
        "dumps",
        "produce",
        "text",
        "or",
        "a",
        "bytes",
        "object"
      ],
      "rankings": {
        "lexical": [
          {
            "chunk_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
            "source_id": "json-text",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/json.html#json.dumps",
            "text": "json.dumps(obj, ensure_ascii=True) returns a JSON string. Setting ensure_ascii=False leaves non-ASCII characters in that string. It does not write a file or select the encoding used by a file writer.",
            "native_score": -3.628237730877387
          },
          {
            "chunk_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
            "source_id": "path-read",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
            "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
            "native_score": -3.0341221974564053e-06
          },
          {
            "chunk_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
            "source_id": "path-write",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.write_text",
            "text": "Path.write_text(data, encoding=None, errors=None, newline=None) writes text to a file and closes it. Set encoding=\"utf-8\" to select UTF-8. An existing file is overwritten. The method returns the number of characters written.",
            "native_score": -2.263641817358195e-06
          }
        ],
        "vector": [
          {
            "chunk_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
            "source_id": "json-text",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/json.html#json.dumps",
            "text": "json.dumps(obj, ensure_ascii=True) returns a JSON string. Setting ensure_ascii=False leaves non-ASCII characters in that string. It does not write a file or select the encoding used by a file writer.",
            "score": 0.652333570145617
          },
          {
            "chunk_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
            "source_id": "path-read",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
            "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
            "score": 0.42563890525675496
          },
          {
            "chunk_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
            "source_id": "path-write",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.write_text",
            "text": "Path.write_text(data, encoding=None, errors=None, newline=None) writes text to a file and closes it. Set encoding=\"utf-8\" to select UTF-8. An existing file is overwritten. The method returns the number of characters written.",
            "score": 0.4145266500154773
          }
        ],
        "rrf": [
          {
            "chunk_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
            "source_id": "json-text",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/json.html#json.dumps",
            "text": "json.dumps(obj, ensure_ascii=True) returns a JSON string. Setting ensure_ascii=False leaves non-ASCII characters in that string. It does not write a file or select the encoding used by a file writer.",
            "lexical_score": -3.628237730877387,
            "vector_score": 0.652333570145617,
            "lexical_rank": 1,
            "vector_rank": 1,
            "lexical_contribution": 0.01639344262295082,
            "vector_contribution": 0.01639344262295082,
            "fused_score": 0.03278688524590164
          },
          {
            "chunk_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
            "source_id": "path-read",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
            "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
            "lexical_score": -3.0341221974564053e-06,
            "vector_score": 0.42563890525675496,
            "lexical_rank": 2,
            "vector_rank": 2,
            "lexical_contribution": 0.016129032258064516,
            "vector_contribution": 0.016129032258064516,
            "fused_score": 0.03225806451612903
          },
          {
            "chunk_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
            "source_id": "path-write",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.write_text",
            "text": "Path.write_text(data, encoding=None, errors=None, newline=None) writes text to a file and closes it. Set encoding=\"utf-8\" to select UTF-8. An existing file is overwritten. The method returns the number of characters written.",
            "lexical_score": -2.263641817358195e-06,
            "vector_score": 0.4145266500154773,
            "lexical_rank": 3,
            "vector_rank": 3,
            "lexical_contribution": 0.015873015873015872,
            "vector_contribution": 0.015873015873015872,
            "fused_score": 0.031746031746031744
          }
        ],
        "rrf+coverage": [
          {
            "chunk_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
            "source_id": "json-text",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/json.html#json.dumps",
            "text": "json.dumps(obj, ensure_ascii=True) returns a JSON string. Setting ensure_ascii=False leaves non-ASCII characters in that string. It does not write a file or select the encoding used by a file writer.",
            "lexical_score": -3.628237730877387,
            "vector_score": 0.652333570145617,
            "lexical_rank": 1,
            "vector_rank": 1,
            "lexical_contribution": 0.01639344262295082,
            "vector_contribution": 0.01639344262295082,
            "fused_score": 0.03278688524590164,
            "incoming_rank": 1,
            "rerank_features": {
              "query_tokens": [
                "does",
                "json",
                "dumps",
                "produce",
                "text",
                "or",
                "a",
                "bytes",
                "object"
              ],
              "matched_tokens": [
                "a",
                "does",
                "dumps",
                "json",
                "or"
              ],
              "matched_count": 5,
              "query_token_count": 9
            },
            "coverage_score": 0.5555555555555556,
            "rerank_rank": 1
          },
          {
            "chunk_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
            "source_id": "path-read",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
            "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
            "lexical_score": -3.0341221974564053e-06,
            "vector_score": 0.42563890525675496,
            "lexical_rank": 2,
            "vector_rank": 2,
            "lexical_contribution": 0.016129032258064516,
            "vector_contribution": 0.016129032258064516,
            "fused_score": 0.03225806451612903,
            "incoming_rank": 2,
            "rerank_features": {
              "query_tokens": [
                "does",
                "json",
                "dumps",
                "produce",
                "text",
                "or",
                "a",
                "bytes",
                "object"
              ],
              "matched_tokens": [
                "a",
                "text"
              ],
              "matched_count": 2,
              "query_token_count": 9
            },
            "coverage_score": 0.2222222222222222,
            "rerank_rank": 2
          },
          {
            "chunk_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
            "source_id": "path-write",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.write_text",
            "text": "Path.write_text(data, encoding=None, errors=None, newline=None) writes text to a file and closes it. Set encoding=\"utf-8\" to select UTF-8. An existing file is overwritten. The method returns the number of characters written.",
            "lexical_score": -2.263641817358195e-06,
            "vector_score": 0.4145266500154773,
            "lexical_rank": 3,
            "vector_rank": 3,
            "lexical_contribution": 0.015873015873015872,
            "vector_contribution": 0.015873015873015872,
            "fused_score": 0.031746031746031744,
            "incoming_rank": 3,
            "rerank_features": {
              "query_tokens": [
                "does",
                "json",
                "dumps",
                "produce",
                "text",
                "or",
                "a",
                "bytes",
                "object"
              ],
              "matched_tokens": [
                "a",
                "text"
              ],
              "matched_count": 2,
              "query_token_count": 9
            },
            "coverage_score": 0.2222222222222222,
            "rerank_rank": 3
          }
        ]
      }
    },
    {
      "case_id": "backup-schedule",
      "query": "How frequently does our cloud provider back up these files?",
      "version": "3.12",
      "lexical_query_terms": [
        "how",
        "frequently",
        "does",
        "our",
        "cloud",
        "provider",
        "back",
        "up",
        "these",
        "files"
      ],
      "rankings": {
        "lexical": [
          {
            "chunk_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
            "source_id": "json-text",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/json.html#json.dumps",
            "text": "json.dumps(obj, ensure_ascii=True) returns a JSON string. Setting ensure_ascii=False leaves non-ASCII characters in that string. It does not write a file or select the encoding used by a file writer.",
            "native_score": -0.8273874668010814
          }
        ],
        "vector": [
          {
            "chunk_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
            "source_id": "path-read",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
            "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
            "score": 0.3202785226491292
          },
          {
            "chunk_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
            "source_id": "path-write",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.write_text",
            "text": "Path.write_text(data, encoding=None, errors=None, newline=None) writes text to a file and closes it. Set encoding=\"utf-8\" to select UTF-8. An existing file is overwritten. The method returns the number of characters written.",
            "score": 0.26563513588901855
          },
          {
            "chunk_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
            "source_id": "json-text",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/json.html#json.dumps",
            "text": "json.dumps(obj, ensure_ascii=True) returns a JSON string. Setting ensure_ascii=False leaves non-ASCII characters in that string. It does not write a file or select the encoding used by a file writer.",
            "score": 0.2488070454064458
          }
        ],
        "rrf": [
          {
            "chunk_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
            "source_id": "json-text",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/json.html#json.dumps",
            "text": "json.dumps(obj, ensure_ascii=True) returns a JSON string. Setting ensure_ascii=False leaves non-ASCII characters in that string. It does not write a file or select the encoding used by a file writer.",
            "lexical_score": -0.8273874668010814,
            "vector_score": 0.2488070454064458,
            "lexical_rank": 1,
            "vector_rank": 3,
            "lexical_contribution": 0.01639344262295082,
            "vector_contribution": 0.015873015873015872,
            "fused_score": 0.032266458495966696
          },
          {
            "chunk_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
            "source_id": "path-read",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
            "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
            "lexical_score": null,
            "vector_score": 0.3202785226491292,
            "lexical_rank": null,
            "vector_rank": 1,
            "lexical_contribution": 0.0,
            "vector_contribution": 0.01639344262295082,
            "fused_score": 0.01639344262295082
          },
          {
            "chunk_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
            "source_id": "path-write",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.write_text",
            "text": "Path.write_text(data, encoding=None, errors=None, newline=None) writes text to a file and closes it. Set encoding=\"utf-8\" to select UTF-8. An existing file is overwritten. The method returns the number of characters written.",
            "lexical_score": null,
            "vector_score": 0.26563513588901855,
            "lexical_rank": null,
            "vector_rank": 2,
            "lexical_contribution": 0.0,
            "vector_contribution": 0.016129032258064516,
            "fused_score": 0.016129032258064516
          }
        ],
        "rrf+coverage": [
          {
            "chunk_id": "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c",
            "source_id": "json-text",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/json.html#json.dumps",
            "text": "json.dumps(obj, ensure_ascii=True) returns a JSON string. Setting ensure_ascii=False leaves non-ASCII characters in that string. It does not write a file or select the encoding used by a file writer.",
            "lexical_score": -0.8273874668010814,
            "vector_score": 0.2488070454064458,
            "lexical_rank": 1,
            "vector_rank": 3,
            "lexical_contribution": 0.01639344262295082,
            "vector_contribution": 0.015873015873015872,
            "fused_score": 0.032266458495966696,
            "incoming_rank": 1,
            "rerank_features": {
              "query_tokens": [
                "how",
                "frequently",
                "does",
                "our",
                "cloud",
                "provider",
                "back",
                "up",
                "these",
                "files"
              ],
              "matched_tokens": [
                "does"
              ],
              "matched_count": 1,
              "query_token_count": 10
            },
            "coverage_score": 0.1,
            "rerank_rank": 1
          },
          {
            "chunk_id": "c737db948e8111ae00e3442d8a22c80c876e43446dc1ee176ffe8ce142a41900",
            "source_id": "path-read",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.read_text",
            "text": "Path.read_text(encoding=None, errors=None) opens a file, returns its text as a string and closes the file. Set encoding=\"utf-8\" when reading a UTF-8 file.",
            "lexical_score": null,
            "vector_score": 0.3202785226491292,
            "lexical_rank": null,
            "vector_rank": 1,
            "lexical_contribution": 0.0,
            "vector_contribution": 0.01639344262295082,
            "fused_score": 0.01639344262295082,
            "incoming_rank": 2,
            "rerank_features": {
              "query_tokens": [
                "how",
                "frequently",
                "does",
                "our",
                "cloud",
                "provider",
                "back",
                "up",
                "these",
                "files"
              ],
              "matched_tokens": [],
              "matched_count": 0,
              "query_token_count": 10
            },
            "coverage_score": 0.0,
            "rerank_rank": 2
          },
          {
            "chunk_id": "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7",
            "source_id": "path-write",
            "version": "3.12",
            "url": "https://docs.python.org/3.12/library/pathlib.html#pathlib.Path.write_text",
            "text": "Path.write_text(data, encoding=None, errors=None, newline=None) writes text to a file and closes it. Set encoding=\"utf-8\" to select UTF-8. An existing file is overwritten. The method returns the number of characters written.",
            "lexical_score": null,
            "vector_score": 0.26563513588901855,
            "lexical_rank": null,
            "vector_rank": 2,
            "lexical_contribution": 0.0,
            "vector_contribution": 0.016129032258064516,
            "fused_score": 0.016129032258064516,
            "incoming_rank": 3,
            "rerank_features": {
              "query_tokens": [
                "how",
                "frequently",
                "does",
                "our",
                "cloud",
                "provider",
                "back",
                "up",
                "these",
                "files"
              ],
              "matched_tokens": [],
              "matched_count": 0,
              "query_token_count": 10
            },
            "coverage_score": 0.0,
            "rerank_rank": 3
          }
        ]
      }
    }
  ],
  "supported_misses": {
    "lexical": {
      "1": [
        "utf8-write"
      ],
      "2": []
    },
    "vector": {
      "1": [
        "utf8-write"
      ],
      "2": []
    },
    "rrf": {
      "1": [
        "utf8-write"
      ],
      "2": []
    },
    "rrf+coverage": {
      "1": [
        "utf8-write"
      ],
      "2": []
    }
  },
  "per_query": [
    {
      "case_id": "utf8-write",
      "case_class": "answerable",
      "target_chunk_ids": [
        "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7"
      ],
      "configurations": {
        "lexical": {
          "1": {
            "target_ranks": [
              2
            ],
            "recall_at_k": 0.0,
            "reciprocal_rank_at_k": 0.0,
            "status": "supported"
          },
          "2": {
            "target_ranks": [
              2
            ],
            "recall_at_k": 1.0,
            "reciprocal_rank_at_k": 0.5,
            "status": "supported"
          }
        },
        "vector": {
          "1": {
            "target_ranks": [
              2
            ],
            "recall_at_k": 0.0,
            "reciprocal_rank_at_k": 0.0,
            "status": "supported"
          },
          "2": {
            "target_ranks": [
              2
            ],
            "recall_at_k": 1.0,
            "reciprocal_rank_at_k": 0.5,
            "status": "supported"
          }
        },
        "rrf": {
          "1": {
            "target_ranks": [
              2
            ],
            "recall_at_k": 0.0,
            "reciprocal_rank_at_k": 0.0,
            "status": "supported"
          },
          "2": {
            "target_ranks": [
              2
            ],
            "recall_at_k": 1.0,
            "reciprocal_rank_at_k": 0.5,
            "status": "supported"
          }
        },
        "rrf+coverage": {
          "1": {
            "target_ranks": [
              2
            ],
            "recall_at_k": 0.0,
            "reciprocal_rank_at_k": 0.0,
            "status": "supported"
          },
          "2": {
            "target_ranks": [
              2
            ],
            "recall_at_k": 1.0,
            "reciprocal_rank_at_k": 0.5,
            "status": "supported"
          }
        }
      }
    },
    {
      "case_id": "replace-file",
      "case_class": "answerable",
      "target_chunk_ids": [
        "c66643a4609c6ee157458bf4353f7446b9c6a200a3ef453846d6a18d5b5069a7"
      ],
      "configurations": {
        "lexical": {
          "1": {
            "target_ranks": [
              1
            ],
            "recall_at_k": 1.0,
            "reciprocal_rank_at_k": 1.0,
            "status": "supported"
          },
          "2": {
            "target_ranks": [
              1
            ],
            "recall_at_k": 1.0,
            "reciprocal_rank_at_k": 1.0,
            "status": "supported"
          }
        },
        "vector": {
          "1": {
            "target_ranks": [
              1
            ],
            "recall_at_k": 1.0,
            "reciprocal_rank_at_k": 1.0,
            "status": "supported"
          },
          "2": {
            "target_ranks": [
              1
            ],
            "recall_at_k": 1.0,
            "reciprocal_rank_at_k": 1.0,
            "status": "supported"
          }
        },
        "rrf": {
          "1": {
            "target_ranks": [
              1
            ],
            "recall_at_k": 1.0,
            "reciprocal_rank_at_k": 1.0,
            "status": "supported"
          },
          "2": {
            "target_ranks": [
              1
            ],
            "recall_at_k": 1.0,
            "reciprocal_rank_at_k": 1.0,
            "status": "supported"
          }
        },
        "rrf+coverage": {
          "1": {
            "target_ranks": [
              1
            ],
            "recall_at_k": 1.0,
            "reciprocal_rank_at_k": 1.0,
            "status": "supported"
          },
          "2": {
            "target_ranks": [
              1
            ],
            "recall_at_k": 1.0,
            "reciprocal_rank_at_k": 1.0,
            "status": "supported"
          }
        }
      }
    },
    {
      "case_id": "json-result-type",
      "case_class": "answerable",
      "target_chunk_ids": [
        "e5455737b11664449f653280d89dfc2faaa9a14de16e9c3eb24296ca2ceea51c"
      ],
      "configurations": {
        "lexical": {
          "1": {
            "target_ranks": [
              1
            ],
            "recall_at_k": 1.0,
            "reciprocal_rank_at_k": 1.0,
            "status": "supported"
          },
          "2": {
            "target_ranks": [
              1
            ],
            "recall_at_k": 1.0,
            "reciprocal_rank_at_k": 1.0,
            "status": "supported"
          }
        },
        "vector": {
          "1": {
            "target_ranks": [
              1
            ],
            "recall_at_k": 1.0,
            "reciprocal_rank_at_k": 1.0,
            "status": "supported"
          },
          "2": {
            "target_ranks": [
              1
            ],
            "recall_at_k": 1.0,
            "reciprocal_rank_at_k": 1.0,
            "status": "supported"
          }
        },
        "rrf": {
          "1": {
            "target_ranks": [
              1
            ],
            "recall_at_k": 1.0,
            "reciprocal_rank_at_k": 1.0,
            "status": "supported"
          },
          "2": {
            "target_ranks": [
              1
            ],
            "recall_at_k": 1.0,
            "reciprocal_rank_at_k": 1.0,
            "status": "supported"
          }
        },
        "rrf+coverage": {
          "1": {
            "target_ranks": [
              1
            ],
            "recall_at_k": 1.0,
            "reciprocal_rank_at_k": 1.0,
            "status": "supported"
          },
          "2": {
            "target_ranks": [
              1
            ],
            "recall_at_k": 1.0,
            "reciprocal_rank_at_k": 1.0,
            "status": "supported"
          }
        }
      }
    },
    {
      "case_id": "backup-schedule",
      "case_class": "unsupported",
      "target_chunk_ids": [],
      "configurations": {
        "lexical": {
          "1": {
            "target_ranks": [],
            "recall_at_k": null,
            "reciprocal_rank_at_k": null,
            "status": "unsupported-excluded"
          },
          "2": {
            "target_ranks": [],
            "recall_at_k": null,
            "reciprocal_rank_at_k": null,
            "status": "unsupported-excluded"
          }
        },
        "vector": {
          "1": {
            "target_ranks": [],
            "recall_at_k": null,
            "reciprocal_rank_at_k": null,
            "status": "unsupported-excluded"
          },
          "2": {
            "target_ranks": [],
            "recall_at_k": null,
            "reciprocal_rank_at_k": null,
            "status": "unsupported-excluded"
          }
        },
        "rrf": {
          "1": {
            "target_ranks": [],
            "recall_at_k": null,
            "reciprocal_rank_at_k": null,
            "status": "unsupported-excluded"
          },
          "2": {
            "target_ranks": [],
            "recall_at_k": null,
            "reciprocal_rank_at_k": null,
            "status": "unsupported-excluded"
          }
        },
        "rrf+coverage": {
          "1": {
            "target_ranks": [],
            "recall_at_k": null,
            "reciprocal_rank_at_k": null,
            "status": "unsupported-excluded"
          },
          "2": {
            "target_ranks": [],
            "recall_at_k": null,
            "reciprocal_rank_at_k": null,
            "status": "unsupported-excluded"
          }
        }
      }
    }
  ],
  "aggregate": {
    "lexical": {
      "1": {
        "supported_case_count": 3,
        "mean_recall_at_k": 0.6666666666666666,
        "mean_reciprocal_rank_at_k": 0.6666666666666666
      },
      "2": {
        "supported_case_count": 3,
        "mean_recall_at_k": 1.0,
        "mean_reciprocal_rank_at_k": 0.8333333333333334
      }
    },
    "vector": {
      "1": {
        "supported_case_count": 3,
        "mean_recall_at_k": 0.6666666666666666,
        "mean_reciprocal_rank_at_k": 0.6666666666666666
      },
      "2": {
        "supported_case_count": 3,
        "mean_recall_at_k": 1.0,
        "mean_reciprocal_rank_at_k": 0.8333333333333334
      }
    },
    "rrf": {
      "1": {
        "supported_case_count": 3,
        "mean_recall_at_k": 0.6666666666666666,
        "mean_reciprocal_rank_at_k": 0.6666666666666666
      },
      "2": {
        "supported_case_count": 3,
        "mean_recall_at_k": 1.0,
        "mean_reciprocal_rank_at_k": 0.8333333333333334
      }
    },
    "rrf+coverage": {
      "1": {
        "supported_case_count": 3,
        "mean_recall_at_k": 0.6666666666666666,
        "mean_reciprocal_rank_at_k": 0.6666666666666666
      },
      "2": {
        "supported_case_count": 3,
        "mean_recall_at_k": 1.0,
        "mean_reciprocal_rank_at_k": 0.8333333333333334
      }
    }
  },
  "unsupported_candidate_behavior": [
    {
      "case_id": "backup-schedule",
      "returned_candidate_counts": {
        "lexical": 1,
        "vector": 3,
        "rrf": 3,
        "rrf+coverage": 3
      }
    }
  ],
  "supported_case_count": 3,
  "unsupported_case_count": 1,
  "limit": "Descriptive results on three supported development questions, not a general search-quality estimate or model comparison. Unsupported questions have no retrieval target and are excluded from metric means."
}
