{
  "execution_mode": "CPU algorithm with controlled inputs; any latency/capacity coefficients are assumptions",
  "model": "70B-GQA-80L",
  "context_len": 4096,
  "sla_tpot_ms": 25.0,
  "selected": {
    "name": "fp8-tp4-b128-nvlink",
    "precision": "fp8",
    "tp_degree": 4,
    "batch_size": 128,
    "interconnect": "nvlink",
    "fits_hbm": true,
    "memory_per_gpu_gb": 42.47,
    "weight_read_ms": 6.53,
    "kv_read_ms": 8.01,
    "compute_ms": 3.71,
    "allreduce_ms": 1.82,
    "tpot_ms": 17.27,
    "total_tokens_per_s": 7412.7,
    "tokens_per_s_per_gpu": 1853.2,
    "bottleneck": "kv_bandwidth"
  },
  "naive_low_latency_baseline": {
    "name": "bf16-tp8-b1-nvlink",
    "precision": "bf16",
    "tp_degree": 8,
    "batch_size": 1,
    "interconnect": "nvlink",
    "fits_hbm": true,
    "memory_per_gpu_gb": 21.17,
    "weight_read_ms": 6.53,
    "kv_read_ms": 0.06,
    "compute_ms": 0.03,
    "allreduce_ms": 0.57,
    "tpot_ms": 7.52,
    "total_tokens_per_s": 133.0,
    "tokens_per_s_per_gpu": 16.6,
    "bottleneck": "weight_bandwidth"
  },
  "pcie_tp4_b64_comparison": {
    "name": "fp8-tp4-b64-pcie",
    "precision": "fp8",
    "tp_degree": 4,
    "batch_size": 64,
    "interconnect": "pcie",
    "fits_hbm": true,
    "memory_per_gpu_gb": 31.74,
    "weight_read_ms": 6.53,
    "kv_read_ms": 4.01,
    "compute_ms": 1.86,
    "allreduce_ms": 19.73,
    "tpot_ms": 30.89,
    "total_tokens_per_s": 2071.6,
    "tokens_per_s_per_gpu": 517.9,
    "bottleneck": "allreduce_comm"
  },
  "tp2_batch_scaling": [
    {
      "name": "fp8-tp2-b64-nvlink",
      "precision": "fp8",
      "tp_degree": 2,
      "batch_size": 64,
      "interconnect": "nvlink",
      "fits_hbm": true,
      "memory_per_gpu_gb": 59.97,
      "weight_read_ms": 13.06,
      "kv_read_ms": 8.01,
      "compute_ms": 3.71,
      "allreduce_ms": 0.98,
      "tpot_ms": 22.96,
      "total_tokens_per_s": 2787.6,
      "tokens_per_s_per_gpu": 1393.8,
      "bottleneck": "weight_bandwidth"
    },
    {
      "name": "fp8-tp2-b128-nvlink",
      "precision": "fp8",
      "tp_degree": 2,
      "batch_size": 128,
      "interconnect": "nvlink",
      "fits_hbm": false,
      "memory_per_gpu_gb": 81.45,
      "weight_read_ms": 13.06,
      "kv_read_ms": 16.03,
      "compute_ms": 7.42,
      "allreduce_ms": 1.4,
      "tpot_ms": 31.95,
      "total_tokens_per_s": 4006.5,
      "tokens_per_s_per_gpu": 2003.3,
      "bottleneck": "kv_bandwidth"
    }
  ],
  "pareto_frontier": [
    {
      "name": "fp8-tp8-b1-nvlink",
      "precision": "fp8",
      "tp_degree": 8,
      "batch_size": 1,
      "interconnect": "nvlink",
      "fits_hbm": true,
      "memory_per_gpu_gb": 12.33,
      "weight_read_ms": 3.26,
      "kv_read_ms": 0.03,
      "compute_ms": 0.01,
      "allreduce_ms": 0.57,
      "tpot_ms": 4.22,
      "total_tokens_per_s": 237.0,
      "tokens_per_s_per_gpu": 29.6,
      "bottleneck": "weight_bandwidth"
    },
    {
      "name": "fp8-tp8-b8-nvlink",
      "precision": "fp8",
      "tp_degree": 8,
      "batch_size": 8,
      "interconnect": "nvlink",
      "fits_hbm": true,
      "memory_per_gpu_gb": 12.92,
      "weight_read_ms": 3.26,
      "kv_read_ms": 0.25,
      "compute_ms": 0.12,
      "allreduce_ms": 0.65,
      "tpot_ms": 4.53,
      "total_tokens_per_s": 1764.3,
      "tokens_per_s_per_gpu": 220.5,
      "bottleneck": "weight_bandwidth"
    },
    {
      "name": "fp8-tp8-b16-nvlink",
      "precision": "fp8",
      "tp_degree": 8,
      "batch_size": 16,
      "interconnect": "nvlink",
      "fits_hbm": true,
      "memory_per_gpu_gb": 13.59,
      "weight_read_ms": 3.26,
      "kv_read_ms": 0.5,
      "compute_ms": 0.23,
      "allreduce_ms": 0.74,
      "tpot_ms": 4.89,
      "total_tokens_per_s": 3269.3,
      "tokens_per_s_per_gpu": 408.7,
      "bottleneck": "weight_bandwidth"
    },
    {
      "name": "fp8-tp8-b32-nvlink",
      "precision": "fp8",
      "tp_degree": 8,
      "batch_size": 32,
      "interconnect": "nvlink",
      "fits_hbm": true,
      "memory_per_gpu_gb": 14.93,
      "weight_read_ms": 3.26,
      "kv_read_ms": 1.0,
      "compute_ms": 0.46,
      "allreduce_ms": 0.93,
      "tpot_ms": 5.61,
      "total_tokens_per_s": 5700.9,
      "tokens_per_s_per_gpu": 712.6,
      "bottleneck": "weight_bandwidth"
    },
    {
      "name": "fp8-tp8-b64-nvlink",
      "precision": "fp8",
      "tp_degree": 8,
      "batch_size": 64,
      "interconnect": "nvlink",
      "fits_hbm": true,
      "memory_per_gpu_gb": 17.62,
      "weight_read_ms": 3.26,
      "kv_read_ms": 2.0,
      "compute_ms": 0.93,
      "allreduce_ms": 1.29,
      "tpot_ms": 7.05,
      "total_tokens_per_s": 9076.3,
      "tokens_per_s_per_gpu": 1134.5,
      "bottleneck": "weight_bandwidth"
    },
    {
      "name": "fp8-tp8-b96-nvlink",
      "precision": "fp8",
      "tp_degree": 8,
      "batch_size": 96,
      "interconnect": "nvlink",
      "fits_hbm": true,
      "memory_per_gpu_gb": 20.3,
      "weight_read_ms": 3.26,
      "kv_read_ms": 3.0,
      "compute_ms": 1.39,
      "allreduce_ms": 1.66,
      "tpot_ms": 8.49,
      "total_tokens_per_s": 11308.1,
      "tokens_per_s_per_gpu": 1413.5,
      "bottleneck": "weight_bandwidth"
    },
    {
      "name": "fp8-tp8-b128-nvlink",
      "precision": "fp8",
      "tp_degree": 8,
      "batch_size": 128,
      "interconnect": "nvlink",
      "fits_hbm": true,
      "memory_per_gpu_gb": 22.99,
      "weight_read_ms": 3.26,
      "kv_read_ms": 4.01,
      "compute_ms": 1.86,
      "allreduce_ms": 2.03,
      "tpot_ms": 9.93,
      "total_tokens_per_s": 12893.2,
      "tokens_per_s_per_gpu": 1611.6,
      "bottleneck": "kv_bandwidth"
    },
    {
      "name": "fp8-tp4-b96-nvlink",
      "precision": "fp8",
      "tp_degree": 4,
      "batch_size": 96,
      "interconnect": "nvlink",
      "fits_hbm": true,
      "memory_per_gpu_gb": 37.11,
      "weight_read_ms": 6.53,
      "kv_read_ms": 6.01,
      "compute_ms": 2.78,
      "allreduce_ms": 1.5,
      "tpot_ms": 14.81,
      "total_tokens_per_s": 6481.8,
      "tokens_per_s_per_gpu": 1620.4,
      "bottleneck": "weight_bandwidth"
    },
    {
      "name": "fp8-tp4-b128-nvlink",
      "precision": "fp8",
      "tp_degree": 4,
      "batch_size": 128,
      "interconnect": "nvlink",
      "fits_hbm": true,
      "memory_per_gpu_gb": 42.47,
      "weight_read_ms": 6.53,
      "kv_read_ms": 8.01,
      "compute_ms": 3.71,
      "allreduce_ms": 1.82,
      "tpot_ms": 17.27,
      "total_tokens_per_s": 7412.7,
      "tokens_per_s_per_gpu": 1853.2,
      "bottleneck": "kv_bandwidth"
    }
  ]
}