{
  "execution_mode": "fresh local inference; no simulated latency",
  "environment": {
    "python": "3.12.8",
    "platform": "macOS-27.0.1-arm64-arm-64bit",
    "torch": "2.14.1",
    "transformers": "5.20.2",
    "device": "mps"
  },
  "model": {
    "id": "Qwen/Qwen2.5-0.5B-Instruct",
    "revision": "7ae557604adf67be50417f59c2c2f167def9a775",
    "license": "Apache-2.0",
    "dtype": "float32",
    "attention_backend": "eager",
    "parameters": 494032768,
    "weight_bytes": 1976131072,
    "layers": 24,
    "kv_heads": 2,
    "head_dim": 64,
    "kv_bytes_per_token": 24576
  },
  "generation": {
    "strategy": "greedy argmax",
    "max_new_tokens": 24,
    "do_sample": false,
    "warmup_tokens": 4,
    "thinking": "not a configurable reasoning mode in this model"
  },
  "source_sha256": "754d9bf81d04694541e53d5c0472f984c9f046886070c33b376ca071c81c8086",
  "warmup": {
    "prompt": "Explain a cache in one sentence.",
    "prompt_tokens": 37,
    "generated_token_ids": [
      32,
      6500,
      374,
      264
    ],
    "generated_tokens": 4,
    "text": "A cache is a",
    "use_cache": true,
    "chunk_size": null,
    "stop_reason": "length",
    "tokenization_ms": 6.358,
    "ttft_ms": 138.267,
    "tpot_ms": 25.619,
    "itl_ms": [
      27.98,
      23.916,
      24.96
    ],
    "e2e_ms": 215.123,
    "cache_tokens_before_release": 40
  },
  "runs": [
    {
      "name": "short_cached",
      "samples": [
        {
          "prompt": "Explain why a code assistant should stop generating when its user cancels.",
          "prompt_tokens": 44,
          "generated_token_ids": [
            4498,
            264,
            2038,
            17847,
            33906,
            264,
            1196,
            879,
            702,
            33446,
            862,
            1681,
            11,
            432,
            374,
            16587,
            311,
            3535,
            279,
            2266,
            323,
            279,
            6993,
            315
          ],
          "generated_tokens": 24,
          "text": "When a code assistant encounters a user who has canceled their request, it is crucial to understand the context and the nature of",
          "use_cache": true,
          "chunk_size": null,
          "stop_reason": "length",
          "tokenization_ms": 0.42,
          "ttft_ms": 36.971,
          "tpot_ms": 37.841,
          "itl_ms": [
            24.717,
            24.26,
            24.26,
            23.673,
            23.945,
            24.013,
            23.746,
            23.935,
            24.656,
            24.232,
            24.202,
            23.957,
            24.223,
            24.584,
            24.632,
            24.522,
            25.532,
            25.876,
            25.605,
            201.281,
            148.208,
            27.729,
            28.559
          ],
          "e2e_ms": 907.318,
          "cache_tokens_before_release": 67
        },
        {
          "prompt": "Explain why a code assistant should stop generating when its user cancels.",
          "prompt_tokens": 44,
          "generated_token_ids": [
            4498,
            264,
            2038,
            17847,
            33906,
            264,
            1196,
            879,
            702,
            33446,
            862,
            1681,
            11,
            432,
            374,
            16587,
            311,
            3535,
            279,
            2266,
            323,
            279,
            6993,
            315
          ],
          "generated_tokens": 24,
          "text": "When a code assistant encounters a user who has canceled their request, it is crucial to understand the context and the nature of",
          "use_cache": true,
          "chunk_size": null,
          "stop_reason": "length",
          "tokenization_ms": 0.508,
          "ttft_ms": 36.399,
          "tpot_ms": 24.093,
          "itl_ms": [
            24.36,
            24.582,
            23.627,
            23.961,
            24.925,
            24.601,
            24.032,
            24.172,
            24.462,
            24.122,
            24.799,
            24.095,
            23.782,
            23.391,
            23.578,
            24.265,
            23.98,
            24.264,
            23.391,
            22.59,
            24.304,
            24.778,
            24.084
          ],
          "e2e_ms": 590.543,
          "cache_tokens_before_release": 67
        },
        {
          "prompt": "Explain why a code assistant should stop generating when its user cancels.",
          "prompt_tokens": 44,
          "generated_token_ids": [
            4498,
            264,
            2038,
            17847,
            33906,
            264,
            1196,
            879,
            702,
            33446,
            862,
            1681,
            11,
            432,
            374,
            16587,
            311,
            3535,
            279,
            2266,
            323,
            279,
            6993,
            315
          ],
          "generated_tokens": 24,
          "text": "When a code assistant encounters a user who has canceled their request, it is crucial to understand the context and the nature of",
          "use_cache": true,
          "chunk_size": null,
          "stop_reason": "length",
          "tokenization_ms": 0.538,
          "ttft_ms": 36.34,
          "tpot_ms": 23.944,
          "itl_ms": [
            23.965,
            23.525,
            23.857,
            23.826,
            22.808,
            23.116,
            22.953,
            23.944,
            23.659,
            23.274,
            23.324,
            23.85,
            23.623,
            24.623,
            24.886,
            25.403,
            24.198,
            24.893,
            24.711,
            23.619,
            24.964,
            24.146,
            23.548
          ],
          "e2e_ms": 587.054,
          "cache_tokens_before_release": 67
        }
      ],
      "median_ttft_ms": 36.399,
      "median_tpot_ms": 24.093
    },
    {
      "name": "long_cached",
      "samples": [
        {
          "prompt": "A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. Explain why a code assistant should stop generating when its user cancels.",
          "prompt_tokens": 263,
          "generated_token_ids": [
            4498,
            264,
            2038,
            17847,
            33906,
            264,
            1196,
            879,
            702,
            33446,
            279,
            10435,
            11,
            432,
            594,
            2989,
            311,
            3535,
            279,
            2266,
            323,
            279,
            7966,
            4815
          ],
          "generated_tokens": 24,
          "text": "When a code assistant encounters a user who has canceled the conversation, it's important to understand the context and the reasons behind",
          "use_cache": true,
          "chunk_size": null,
          "stop_reason": "length",
          "tokenization_ms": 0.758,
          "ttft_ms": 273.572,
          "tpot_ms": 26.82,
          "itl_ms": [
            27.356,
            27.6,
            26.837,
            26.249,
            27.147,
            27.567,
            27.574,
            26.044,
            26.353,
            26.533,
            26.771,
            26.266,
            25.951,
            26.33,
            26.194,
            26.868,
            26.187,
            27.931,
            27.892,
            26.783,
            27.818,
            25.98,
            26.633
          ],
          "e2e_ms": 890.437,
          "cache_tokens_before_release": 286
        },
        {
          "prompt": "A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. Explain why a code assistant should stop generating when its user cancels.",
          "prompt_tokens": 263,
          "generated_token_ids": [
            4498,
            264,
            2038,
            17847,
            33906,
            264,
            1196,
            879,
            702,
            33446,
            279,
            10435,
            11,
            432,
            594,
            2989,
            311,
            3535,
            279,
            2266,
            323,
            279,
            7966,
            4815
          ],
          "generated_tokens": 24,
          "text": "When a code assistant encounters a user who has canceled the conversation, it's important to understand the context and the reasons behind",
          "use_cache": true,
          "chunk_size": null,
          "stop_reason": "length",
          "tokenization_ms": 0.85,
          "ttft_ms": 86.29,
          "tpot_ms": 25.288,
          "itl_ms": [
            26.506,
            25.52,
            26.769,
            24.864,
            25.298,
            26.576,
            25.433,
            23.891,
            23.809,
            23.687,
            26.404,
            26.394,
            25.43,
            25.29,
            25.059,
            24.146,
            24.421,
            24.663,
            26.637,
            25.673,
            25.497,
            24.925,
            24.726
          ],
          "e2e_ms": 667.908,
          "cache_tokens_before_release": 286
        },
        {
          "prompt": "A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. Explain why a code assistant should stop generating when its user cancels.",
          "prompt_tokens": 263,
          "generated_token_ids": [
            4498,
            264,
            2038,
            17847,
            33906,
            264,
            1196,
            879,
            702,
            33446,
            279,
            10435,
            11,
            432,
            594,
            2989,
            311,
            3535,
            279,
            2266,
            323,
            279,
            7966,
            4815
          ],
          "generated_tokens": 24,
          "text": "When a code assistant encounters a user who has canceled the conversation, it's important to understand the context and the reasons behind",
          "use_cache": true,
          "chunk_size": null,
          "stop_reason": "length",
          "tokenization_ms": 0.694,
          "ttft_ms": 84.662,
          "tpot_ms": 24.334,
          "itl_ms": [
            25.428,
            25.259,
            24.846,
            24.946,
            25.267,
            23.982,
            24.173,
            24.283,
            24.333,
            23.722,
            23.755,
            23.511,
            24.927,
            24.42,
            24.321,
            24.326,
            24.096,
            24.343,
            23.912,
            24.211,
            24.209,
            23.855,
            23.554
          ],
          "e2e_ms": 644.34,
          "cache_tokens_before_release": 286
        }
      ],
      "median_ttft_ms": 86.29,
      "median_tpot_ms": 25.288
    },
    {
      "name": "short_without_cache",
      "samples": [
        {
          "prompt": "Explain why a code assistant should stop generating when its user cancels.",
          "prompt_tokens": 44,
          "generated_token_ids": [
            4498,
            264,
            2038,
            17847,
            33906,
            264,
            1196,
            879,
            702,
            33446,
            862,
            1681,
            11,
            432,
            374,
            16587,
            311,
            3535,
            279,
            2266,
            323,
            279,
            6993,
            315
          ],
          "generated_tokens": 24,
          "text": "When a code assistant encounters a user who has canceled their request, it is crucial to understand the context and the nature of",
          "use_cache": false,
          "chunk_size": null,
          "stop_reason": "length",
          "tokenization_ms": 0.37,
          "ttft_ms": 66.297,
          "tpot_ms": 67.008,
          "itl_ms": [
            79.186,
            134.713,
            39.694,
            226.116,
            201.879,
            41.316,
            41.58,
            41.005,
            40.472,
            41.45,
            81.137,
            42.528,
            40.705,
            40.632,
            41.435,
            41.415,
            42.948,
            40.251,
            41.818,
            41.779,
            109.786,
            44.586,
            44.763
          ],
          "e2e_ms": 1607.491,
          "cache_tokens_before_release": 0
        },
        {
          "prompt": "Explain why a code assistant should stop generating when its user cancels.",
          "prompt_tokens": 44,
          "generated_token_ids": [
            4498,
            264,
            2038,
            17847,
            33906,
            264,
            1196,
            879,
            702,
            33446,
            862,
            1681,
            11,
            432,
            374,
            16587,
            311,
            3535,
            279,
            2266,
            323,
            279,
            6993,
            315
          ],
          "generated_tokens": 24,
          "text": "When a code assistant encounters a user who has canceled their request, it is crucial to understand the context and the nature of",
          "use_cache": false,
          "chunk_size": null,
          "stop_reason": "length",
          "tokenization_ms": 0.388,
          "ttft_ms": 37.483,
          "tpot_ms": 39.405,
          "itl_ms": [
            36.29,
            37.237,
            37.341,
            39.518,
            38.716,
            39.582,
            38.704,
            39.39,
            39.481,
            39.121,
            39.194,
            40.456,
            40.368,
            39.749,
            40.942,
            39.908,
            39.032,
            38.143,
            39.892,
            36.976,
            41.664,
            41.7,
            42.915
          ],
          "e2e_ms": 943.802,
          "cache_tokens_before_release": 0
        },
        {
          "prompt": "Explain why a code assistant should stop generating when its user cancels.",
          "prompt_tokens": 44,
          "generated_token_ids": [
            4498,
            264,
            2038,
            17847,
            33906,
            264,
            1196,
            879,
            702,
            33446,
            862,
            1681,
            11,
            432,
            374,
            16587,
            311,
            3535,
            279,
            2266,
            323,
            279,
            6993,
            315
          ],
          "generated_tokens": 24,
          "text": "When a code assistant encounters a user who has canceled their request, it is crucial to understand the context and the nature of",
          "use_cache": false,
          "chunk_size": null,
          "stop_reason": "length",
          "tokenization_ms": 0.417,
          "ttft_ms": 38.318,
          "tpot_ms": 39.635,
          "itl_ms": [
            36.313,
            37.162,
            37.48,
            40.273,
            39.284,
            39.079,
            39.382,
            38.926,
            40.223,
            39.338,
            39.432,
            39.342,
            39.309,
            39.563,
            40.44,
            40.655,
            40.428,
            39.193,
            40.016,
            38.449,
            41.942,
            42.91,
            42.457
          ],
          "e2e_ms": 949.915,
          "cache_tokens_before_release": 0
        }
      ],
      "median_ttft_ms": 38.318,
      "median_tpot_ms": 39.635
    },
    {
      "name": "long_chunked",
      "samples": [
        {
          "prompt": "A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. Explain why a code assistant should stop generating when its user cancels.",
          "prompt_tokens": 263,
          "generated_token_ids": [
            4498,
            264,
            2038,
            17847,
            33906,
            264,
            1196,
            879,
            702,
            33446,
            279,
            10435,
            11,
            432,
            594,
            2989,
            311,
            3535,
            279,
            2266,
            323,
            279,
            7966,
            4815
          ],
          "generated_tokens": 24,
          "text": "When a code assistant encounters a user who has canceled the conversation, it's important to understand the context and the reasons behind",
          "use_cache": true,
          "chunk_size": 64,
          "stop_reason": "length",
          "tokenization_ms": 0.835,
          "ttft_ms": 702.895,
          "tpot_ms": 28.833,
          "itl_ms": [
            38.292,
            66.814,
            41.939,
            32.362,
            26.243,
            25.407,
            24.949,
            25.175,
            26.761,
            25.854,
            25.866,
            25.026,
            25.473,
            25.808,
            25.566,
            26.479,
            25.683,
            24.899,
            25.496,
            24.559,
            24.987,
            24.635,
            24.89
          ],
          "e2e_ms": 1366.057,
          "cache_tokens_before_release": 286
        },
        {
          "prompt": "A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. Explain why a code assistant should stop generating when its user cancels.",
          "prompt_tokens": 263,
          "generated_token_ids": [
            4498,
            264,
            2038,
            17847,
            33906,
            264,
            1196,
            879,
            702,
            33446,
            279,
            10435,
            11,
            432,
            594,
            2989,
            311,
            3535,
            279,
            2266,
            323,
            279,
            7966,
            4815
          ],
          "generated_tokens": 24,
          "text": "When a code assistant encounters a user who has canceled the conversation, it's important to understand the context and the reasons behind",
          "use_cache": true,
          "chunk_size": 64,
          "stop_reason": "length",
          "tokenization_ms": 0.797,
          "ttft_ms": 180.806,
          "tpot_ms": 28.243,
          "itl_ms": [
            24.767,
            25.971,
            28.707,
            28.913,
            29.586,
            32.806,
            33.812,
            32.961,
            28.86,
            30.909,
            27.018,
            25.968,
            25.662,
            26.752,
            26.08,
            25.695,
            26.85,
            25.259,
            26.738,
            29.582,
            31.276,
            28.427,
            26.987
          ],
          "e2e_ms": 830.391,
          "cache_tokens_before_release": 286
        },
        {
          "prompt": "A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. A retrieval service indexes source code and returns relevant functions. Explain why a code assistant should stop generating when its user cancels.",
          "prompt_tokens": 263,
          "generated_token_ids": [
            4498,
            264,
            2038,
            17847,
            33906,
            264,
            1196,
            879,
            702,
            33446,
            279,
            10435,
            11,
            432,
            594,
            2989,
            311,
            3535,
            279,
            2266,
            323,
            279,
            7966,
            4815
          ],
          "generated_tokens": 24,
          "text": "When a code assistant encounters a user who has canceled the conversation, it's important to understand the context and the reasons behind",
          "use_cache": true,
          "chunk_size": 64,
          "stop_reason": "length",
          "tokenization_ms": 0.763,
          "ttft_ms": 192.01,
          "tpot_ms": 25.814,
          "itl_ms": [
            25.66,
            25.885,
            25.866,
            26.904,
            26.668,
            26.756,
            26.449,
            27.375,
            25.687,
            25.242,
            25.962,
            25.179,
            24.328,
            24.858,
            24.397,
            24.939,
            24.364,
            24.276,
            26.038,
            27.134,
            27.045,
            26.886,
            25.831
          ],
          "e2e_ms": 785.739,
          "cache_tokens_before_release": 286
        }
      ],
      "median_ttft_ms": 192.01,
      "median_tpot_ms": 28.243
    },
    {
      "name": "cancelled",
      "samples": [
        {
          "prompt": "Explain why a code assistant should stop generating when its user cancels.",
          "prompt_tokens": 44,
          "generated_token_ids": [
            4498,
            264,
            2038,
            17847
          ],
          "generated_tokens": 4,
          "text": "When a code assistant",
          "use_cache": true,
          "chunk_size": null,
          "stop_reason": "cancelled",
          "tokenization_ms": 0.454,
          "ttft_ms": 39.63,
          "tpot_ms": 27.403,
          "itl_ms": [
            28.558,
            28.742,
            24.91
          ],
          "e2e_ms": 121.84,
          "cache_tokens_before_release": 47
        },
        {
          "prompt": "Explain why a code assistant should stop generating when its user cancels.",
          "prompt_tokens": 44,
          "generated_token_ids": [
            4498,
            264,
            2038,
            17847
          ],
          "generated_tokens": 4,
          "text": "When a code assistant",
          "use_cache": true,
          "chunk_size": null,
          "stop_reason": "cancelled",
          "tokenization_ms": 0.387,
          "ttft_ms": 33.383,
          "tpot_ms": 22.446,
          "itl_ms": [
            21.98,
            21.693,
            23.664
          ],
          "e2e_ms": 100.721,
          "cache_tokens_before_release": 47
        },
        {
          "prompt": "Explain why a code assistant should stop generating when its user cancels.",
          "prompt_tokens": 44,
          "generated_token_ids": [
            4498,
            264,
            2038,
            17847
          ],
          "generated_tokens": 4,
          "text": "When a code assistant",
          "use_cache": true,
          "chunk_size": null,
          "stop_reason": "cancelled",
          "tokenization_ms": 0.456,
          "ttft_ms": 34.266,
          "tpot_ms": 20.906,
          "itl_ms": [
            20.964,
            21.509,
            20.245
          ],
          "e2e_ms": 96.985,
          "cache_tokens_before_release": 47
        }
      ],
      "median_ttft_ms": 34.266,
      "median_tpot_ms": 22.446
    }
  ],
  "batch_sweep": [
    {
      "batch_size": 1,
      "elapsed_ms": 1144.114,
      "output_tokens": 24,
      "output_tokens_per_s": 20.977,
      "texts": [
        "When a code assistant encounters the user's cancellation of an operation, it is crucial to understand that this action signifies the end"
      ]
    },
    {
      "batch_size": 2,
      "elapsed_ms": 1490.669,
      "output_tokens": 48,
      "output_tokens_per_s": 32.2,
      "texts": [
        "When a code assistant encounters the user's cancellation of an operation, it is crucial to understand that this action signifies the end",
        "When a code assistant encounters the user's cancellation of an operation, it is crucial to understand that this action signifies the end"
      ]
    },
    {
      "batch_size": 4,
      "elapsed_ms": 1243.557,
      "output_tokens": 96,
      "output_tokens_per_s": 77.198,
      "texts": [
        "When a code assistant encounters the user's cancellation of an operation, it is crucial to understand that this action signifies the end",
        "When a code assistant encounters the user's cancellation of an operation, it is crucial to understand that this action signifies the end",
        "When a code assistant encounters the user's cancellation of an operation, it is crucial to understand that this action signifies the end",
        "When a code assistant encounters the user's cancellation of an operation, it is crucial to understand that this action signifies the end"
      ]
    }
  ],
  "prefix_reuse": {
    "total_tokens": 268,
    "cached_prefix_tokens": 134,
    "cold_prefill_ms": 90.296,
    "suffix_prefill_ms": 60.948,
    "max_abs_last_logit_diff": 4.506111145019531e-05,
    "same_greedy_next_token": true
  },
  "limits": "Three repeated development prompts per configuration; no held-out quality evaluation, HTTP latency, CUDA profiling, or production load test."
}