{
  "execution_mode": "offline analysis of fresh local inference record",
  "model": {
    "id": "Qwen/Qwen2.5-0.5B-Instruct",
    "revision": "7ae557604adf67be50417f59c2c2f167def9a775",
    "license": "Apache-2.0",
    "dtype": "float32",
    "attention_backend": "eager",
    "parameters": 494032768,
    "weight_bytes": 1976131072,
    "layers": 24,
    "kv_heads": 2,
    "head_dim": 64,
    "kv_bytes_per_token": 24576
  },
  "generation": {
    "strategy": "greedy argmax",
    "max_new_tokens": 24,
    "do_sample": false,
    "warmup_tokens": 4,
    "thinking": "not a configurable reasoning mode in this model"
  },
  "comparisons": {
    "short_cached": {
      "samples": 3,
      "median_ttft_ms": 36.399,
      "median_tpot_ms": 24.093
    },
    "long_cached": {
      "samples": 3,
      "median_ttft_ms": 86.29,
      "median_tpot_ms": 25.288
    },
    "short_without_cache": {
      "samples": 3,
      "median_ttft_ms": 38.318,
      "median_tpot_ms": 39.635
    },
    "long_chunked": {
      "samples": 3,
      "median_ttft_ms": 192.01,
      "median_tpot_ms": 28.243
    },
    "cancelled": {
      "samples": 3,
      "median_ttft_ms": 34.266,
      "median_tpot_ms": 22.446
    }
  },
  "batch_sweep": [
    {
      "batch_size": 1,
      "elapsed_ms": 1144.114,
      "output_tokens": 24,
      "output_tokens_per_s": 20.977,
      "texts": [
        "When a code assistant encounters the user's cancellation of an operation, it is crucial to understand that this action signifies the end"
      ]
    },
    {
      "batch_size": 2,
      "elapsed_ms": 1490.669,
      "output_tokens": 48,
      "output_tokens_per_s": 32.2,
      "texts": [
        "When a code assistant encounters the user's cancellation of an operation, it is crucial to understand that this action signifies the end",
        "When a code assistant encounters the user's cancellation of an operation, it is crucial to understand that this action signifies the end"
      ]
    },
    {
      "batch_size": 4,
      "elapsed_ms": 1243.557,
      "output_tokens": 96,
      "output_tokens_per_s": 77.198,
      "texts": [
        "When a code assistant encounters the user's cancellation of an operation, it is crucial to understand that this action signifies the end",
        "When a code assistant encounters the user's cancellation of an operation, it is crucial to understand that this action signifies the end",
        "When a code assistant encounters the user's cancellation of an operation, it is crucial to understand that this action signifies the end",
        "When a code assistant encounters the user's cancellation of an operation, it is crucial to understand that this action signifies the end"
      ]
    }
  ],
  "prefix_reuse": {
    "total_tokens": 268,
    "cached_prefix_tokens": 134,
    "cold_prefill_ms": 90.296,
    "suffix_prefill_ms": 60.948,
    "max_abs_last_logit_diff": 4.506111145019531e-05,
    "same_greedy_next_token": true
  }
}